From 2e6d64a5536b87d9847ff24dd692206ef232c779 Mon Sep 17 00:00:00 2001 From: Pastukhov Nikita Date: Thu, 18 May 2023 11:29:24 +0300 Subject: [PATCH] 0.1.1 Release * redis draft * update readme blank links * correct docs with redis examples * correct rabbit template * specify `propan create` command * add create redis app subcommand * correct tests * add create nats app subcommand * add redis examples * add redis examples * add redis ru docs * redis docs complete * release redis version * python3.7 comaptible annotation --- .github/workflows/documentation.yml | 2 - .github/workflows/tests.yml | 5 + README.md | 57 ++-- docs/docs.py | 15 +- docs/docs/en/4_nats/1_nats-index.md | 2 - docs/docs/en/4_nats/2_routing.md | 2 - docs/docs/en/4_nats/3_publishing.md | 2 - docs/docs/en/{9_CHANGELOG.md => CHANGELOG.md} | 23 +- .../en/{7_alternatives.md => alternatives.md} | 0 .../1_todo.md | 2 +- .../2_contributing-index.md | 0 .../3_docs.md | 0 .../4_adapters.md | 0 .../1_quick-start.md | 50 +++- .../2_cli.md | 7 +- .../3_app.md | 10 + .../4_broker/1_index.md | 41 ++- .../4_broker/2_routing.md | 4 +- .../4_broker/3_type-casting.md | 0 .../4_broker/4_publishing.md | 16 +- .../4_broker/5_rpc.md | 34 ++- .../5_dependency/1_di-index.md | 31 ++- .../5_dependency/2_context.md | 0 .../6_lifespans.md | 106 ++++++-- .../7_testing.md | 57 +++- .../8_logging.md | 0 docs/docs/en/{8_help.md => help.md} | 0 docs/docs/en/index.md | 46 ++-- .../1_integrations-index.md | 0 .../2_fastapi-plugin.md | 36 ++- docs/docs/en/nats/1_nats-index.md | 3 + docs/docs/en/nats/2_routing.md | 3 + docs/docs/en/nats/3_publishing.md | 3 + .../docs/en/{3_rabbit => rabbit}/1_routing.md | 0 .../en/{3_rabbit => rabbit}/2_exchanges.md | 0 docs/docs/en/{3_rabbit => rabbit}/3_queues.md | 0 .../en/{3_rabbit => rabbit}/4_publishing.md | 26 +- .../5_examples/1_direct.md | 16 +- .../5_examples/2_fanout.md | 14 +- .../5_examples/3_topic.md | 14 +- .../5_examples/4_header.md | 18 +- docs/docs/en/redis/1_redis-index.md | 50 ++++ docs/docs/en/redis/2_publishing.md | 55 ++++ docs/docs/en/redis/3_examples/1_direct.md | 56 ++++ docs/docs/en/redis/3_examples/2_pattern.md | 47 ++++ docs/docs/ru/{9_CHANGELOG.md => CHANGELOG.md} | 23 +- .../ru/{7_alternatives.md => alternatives.md} | 0 .../1_todo.md | 2 +- .../2_contributing-index.md | 0 .../3_docs.md | 0 .../4_adapters.md | 0 .../1_quick-start.md | 49 +++- .../2_cli.md | 7 +- .../3_app.md | 10 + .../4_broker/1_index.md | 41 ++- .../4_broker/2_routing.md | 4 +- .../4_broker/3_type-casting.md | 0 .../4_broker/4_publishing.md | 16 +- .../4_broker/5_rpc.md | 34 ++- .../5_dependency/1_di-index.md | 34 ++- .../5_dependency/2_context.md | 0 .../6_lifespans.md | 106 ++++++-- .../7_testing.md | 91 +++++-- .../8_logging.md | 0 docs/docs/ru/{8_help.md => help.md} | 0 docs/docs/ru/index.md | 44 +-- .../1_integrations-index.md | 0 .../2_fastapi-plugin.md | 36 ++- docs/docs/ru/{4_nats => nats}/1_nats-index.md | 0 docs/docs/ru/{4_nats => nats}/2_routing.md | 0 docs/docs/ru/{4_nats => nats}/3_publishing.md | 0 .../docs/ru/{3_rabbit => rabbit}/1_routing.md | 0 .../ru/{3_rabbit => rabbit}/2_exchanges.md | 0 docs/docs/ru/{3_rabbit => rabbit}/3_queues.md | 0 .../ru/{3_rabbit => rabbit}/4_publishing.md | 26 +- .../5_examples/1_direct.md | 24 +- .../5_examples/2_fanout.md | 22 +- .../5_examples/3_topic.md | 22 +- .../5_examples/4_header.md | 30 +-- docs/docs/ru/redis/1_redis-index.md | 50 ++++ docs/docs/ru/redis/2_publishing.md | 55 ++++ docs/docs/ru/redis/3_examples/1_direct.md | 57 ++++ docs/docs/ru/redis/3_examples/2_pattern.md | 47 ++++ docs/docs_src/index/01_redis_base.py | 9 + docs/docs_src/index/02_redis_type_casting.py | 12 + docs/docs_src/index/03_redis_dependencies.py | 14 + docs/docs_src/index/04_redis_context.py | 15 ++ docs/docs_src/index/05_redis_http_example.py | 17 ++ .../docs_src/index/06_redis_native_fastapi.py | 19 ++ .../integrations/fastapi_plugin_rabbit.py | 4 +- .../fastapi_plugin_rabbit_depends.py | 2 +- .../fastapi_plugin_rabbit_send.py | 2 +- .../integrations/fastapi_plugin_redis.py | 23 ++ .../fastapi_plugin_redis_depends.py | 18 ++ .../integrations/fastapi_plugin_redis_send.py | 13 + .../docs_src/quickstart/app/1_broker_redis.py | 4 + .../quickstart/app/2_set_broker_redis.py | 8 + .../broker/publishing/1_nats_inside_propan.py | 2 +- .../publishing/1_redis_inside_propan.py | 8 + .../broker/publishing/2_nats_context.py | 2 +- .../broker/publishing/2_redis_context.py | 2 + .../quickstart/broker/rpc/1_redis_handler.py | 7 + .../broker/rpc/2_redis_blocking_client.py | 11 + .../rpc/3_redis_blocking_client_timeout.py | 6 + .../4_redis_blocking_client_timeout_none.py | 6 + .../5_redis_blocking_client_timeout_error.py | 6 + .../broker/rpc/6_noblocking_client_rabbit.py | 2 +- .../broker/rpc/6_noblocking_client_redis.py | 24 ++ .../basic/1_propan_redis_depends.py | 11 + .../basic/2_propan_redis_depends.py | 17 ++ docs/docs_src/quickstart/lifespan/1_redis.py | 15 ++ .../docs_src/quickstart/lifespan/2_ml_nats.py | 2 +- .../quickstart/lifespan/2_ml_rabbit.py | 2 +- .../quickstart/lifespan/2_ml_redis.py | 26 ++ .../testing/1_main_rabbit.py} | 0 .../quickstart/testing/1_main_redis.py | 12 + .../testing/2_test_rabbit.py} | 0 .../quickstart/testing/2_test_redis.py | 9 + .../testing/3_conftest_rabbit.py} | 0 .../quickstart/testing/3_conftest_redis.py | 14 + .../testing/4_suppressed_exc_rabbit.py} | 0 .../testing/4_suppressed_exc_redis.py | 6 + .../testing/5_build_message_rabbit.py} | 0 .../testing/5_build_message_redis.py | 7 + .../testing/6_reraise_rabbit.py} | 0 .../quickstart/testing/6_reraise_redis.py | 11 + docs/docs_src/rabbit/{examples => }/direct.py | 6 +- docs/docs_src/rabbit/{examples => }/fanout.py | 6 +- docs/docs_src/rabbit/{examples => }/header.py | 12 +- docs/docs_src/rabbit/{examples => }/topic.py | 10 +- docs/docs_src/redis/direct.py | 26 ++ docs/docs_src/redis/pattern.py | 26 ++ docs/mkdocs.yml | 74 ++--- examples/rabbit/direct.py | 4 +- examples/rabbit/fanout.py | 4 +- examples/rabbit/header.py | 4 +- examples/rabbit/topic.py | 4 +- examples/redis/direct.py | 26 ++ examples/redis/pattern.py | 26 ++ propan/__about__.py | 2 +- propan/__init__.py | 10 +- propan/annotations.py | 9 + propan/brokers/model/broker_usecase.py | 28 +- propan/brokers/model/schemas.py | 35 ++- propan/brokers/nats/nats_broker.py | 16 +- propan/brokers/nats/schemas.py | 5 +- propan/brokers/rabbit/rabbit_broker.py | 10 +- propan/brokers/rabbit/schemas.py | 10 +- propan/brokers/redis/__init__.py | 3 + propan/brokers/redis/redis_broker.py | 254 ++++++++++++++++++ propan/brokers/redis/redis_broker.pyi | 108 ++++++++ propan/brokers/redis/schemas.py | 25 ++ propan/cli/app.py | 38 ++- propan/cli/main.py | 27 +- propan/cli/startproject.py | 225 ---------------- propan/cli/startproject/__init__.py | 3 + propan/cli/startproject/app.py | 8 + propan/cli/startproject/async_app/__init__.py | 3 + propan/cli/startproject/async_app/app.py | 30 +++ propan/cli/startproject/async_app/core.py | 37 +++ propan/cli/startproject/async_app/nats.py | 82 ++++++ propan/cli/startproject/async_app/rabbit.py | 93 +++++++ propan/cli/startproject/async_app/redis.py | 87 ++++++ propan/cli/startproject/core.py | 143 ++++++++++ propan/cli/startproject/sync_app/__init__.py | 3 + propan/cli/startproject/sync_app/app.py | 3 + propan/cli/startproject/utils.py | 18 ++ propan/fastapi/__init__.py | 12 +- propan/fastapi/core/route.py | 2 +- propan/fastapi/rabbit/__init__.py | 2 +- .../rabbit/{rabbit_router.py => router.py} | 0 .../rabbit/{rabbit_router.pyi => router.pyi} | 0 propan/fastapi/redis/__init__.py | 3 + propan/fastapi/redis/router.py | 7 + propan/fastapi/redis/router.pyi | 87 ++++++ propan/test/__init__.py | 10 +- propan/test/rabbit.py | 22 +- propan/test/redis.py | 86 ++++++ propan/test/utils.py | 23 ++ propan/types.py | 4 +- propan/utils/context/main.py | 1 + pyproject.toml | 8 + tests/brokers/rabbit/conftest.py | 2 +- tests/brokers/rabbit/test_publish.py | 1 + tests/brokers/redis/conftest.py | 44 +++ tests/brokers/redis/test_acc.py | 115 ++++++++ tests/brokers/redis/test_connect.py | 26 ++ tests/brokers/redis/test_publish.py | 70 +++++ tests/brokers/redis/test_test_client.py | 88 ++++++ tests/cli/conftest.py | 6 +- tests/cli/test_app.py | 6 +- tests/cli/test_creation.py | 4 +- tests/cli/test_run.py | 4 +- tests/fastapi/test_app.py | 36 ++- 194 files changed, 3518 insertions(+), 745 deletions(-) delete mode 100644 docs/docs/en/4_nats/1_nats-index.md delete mode 100644 docs/docs/en/4_nats/2_routing.md delete mode 100644 docs/docs/en/4_nats/3_publishing.md rename docs/docs/en/{9_CHANGELOG.md => CHANGELOG.md} (76%) rename docs/docs/en/{7_alternatives.md => alternatives.md} (100%) rename docs/docs/en/{6_contributing => contributing}/1_todo.md (99%) rename docs/docs/en/{6_contributing => contributing}/2_contributing-index.md (100%) rename docs/docs/en/{6_contributing => contributing}/3_docs.md (100%) rename docs/docs/en/{6_contributing => contributing}/4_adapters.md (100%) rename docs/docs/en/{2_getting_started => getting_started}/1_quick-start.md (79%) rename docs/docs/en/{2_getting_started => getting_started}/2_cli.md (95%) rename docs/docs/en/{2_getting_started => getting_started}/3_app.md (87%) rename docs/docs/en/{2_getting_started => getting_started}/4_broker/1_index.md (72%) rename docs/docs/en/{2_getting_started => getting_started}/4_broker/2_routing.md (94%) rename docs/docs/en/{2_getting_started => getting_started}/4_broker/3_type-casting.md (100%) rename docs/docs/en/{2_getting_started => getting_started}/4_broker/4_publishing.md (80%) rename docs/docs/en/{2_getting_started => getting_started}/4_broker/5_rpc.md (77%) rename docs/docs/en/{2_getting_started => getting_started}/5_dependency/1_di-index.md (88%) rename docs/docs/en/{2_getting_started => getting_started}/5_dependency/2_context.md (100%) rename docs/docs/en/{2_getting_started => getting_started}/6_lifespans.md (67%) rename docs/docs/en/{2_getting_started => getting_started}/7_testing.md (60%) rename docs/docs/en/{2_getting_started => getting_started}/8_logging.md (100%) rename docs/docs/en/{8_help.md => help.md} (100%) rename docs/docs/en/{5_integrations => integrations}/1_integrations-index.md (100%) rename docs/docs/en/{5_integrations => integrations}/2_fastapi-plugin.md (61%) create mode 100644 docs/docs/en/nats/1_nats-index.md create mode 100644 docs/docs/en/nats/2_routing.md create mode 100644 docs/docs/en/nats/3_publishing.md rename docs/docs/en/{3_rabbit => rabbit}/1_routing.md (100%) rename docs/docs/en/{3_rabbit => rabbit}/2_exchanges.md (100%) rename docs/docs/en/{3_rabbit => rabbit}/3_queues.md (100%) rename docs/docs/en/{3_rabbit => rabbit}/4_publishing.md (80%) rename docs/docs/en/{3_rabbit => rabbit}/5_examples/1_direct.md (87%) rename docs/docs/en/{3_rabbit => rabbit}/5_examples/2_fanout.md (82%) rename docs/docs/en/{3_rabbit => rabbit}/5_examples/3_topic.md (83%) rename docs/docs/en/{3_rabbit => rabbit}/5_examples/4_header.md (85%) create mode 100644 docs/docs/en/redis/1_redis-index.md create mode 100644 docs/docs/en/redis/2_publishing.md create mode 100644 docs/docs/en/redis/3_examples/1_direct.md create mode 100644 docs/docs/en/redis/3_examples/2_pattern.md rename docs/docs/ru/{9_CHANGELOG.md => CHANGELOG.md} (77%) rename docs/docs/ru/{7_alternatives.md => alternatives.md} (100%) rename docs/docs/ru/{6_contributing => contributing}/1_todo.md (97%) rename docs/docs/ru/{6_contributing => contributing}/2_contributing-index.md (100%) rename docs/docs/ru/{6_contributing => contributing}/3_docs.md (100%) rename docs/docs/ru/{6_contributing => contributing}/4_adapters.md (100%) rename docs/docs/ru/{2_getting_started => getting_started}/1_quick-start.md (84%) rename docs/docs/ru/{2_getting_started => getting_started}/2_cli.md (96%) rename docs/docs/ru/{2_getting_started => getting_started}/3_app.md (91%) rename docs/docs/ru/{2_getting_started => getting_started}/4_broker/1_index.md (80%) rename docs/docs/ru/{2_getting_started => getting_started}/4_broker/2_routing.md (96%) rename docs/docs/ru/{2_getting_started => getting_started}/4_broker/3_type-casting.md (100%) rename docs/docs/ru/{2_getting_started => getting_started}/4_broker/4_publishing.md (88%) rename docs/docs/ru/{2_getting_started => getting_started}/4_broker/5_rpc.md (83%) rename docs/docs/ru/{2_getting_started => getting_started}/5_dependency/1_di-index.md (90%) rename docs/docs/ru/{2_getting_started => getting_started}/5_dependency/2_context.md (100%) rename docs/docs/ru/{2_getting_started => getting_started}/6_lifespans.md (78%) rename docs/docs/ru/{2_getting_started => getting_started}/7_testing.md (61%) rename docs/docs/ru/{2_getting_started => getting_started}/8_logging.md (100%) rename docs/docs/ru/{8_help.md => help.md} (100%) rename docs/docs/ru/{5_integrations => integrations}/1_integrations-index.md (100%) rename docs/docs/ru/{5_integrations => integrations}/2_fastapi-plugin.md (75%) rename docs/docs/ru/{4_nats => nats}/1_nats-index.md (100%) rename docs/docs/ru/{4_nats => nats}/2_routing.md (100%) rename docs/docs/ru/{4_nats => nats}/3_publishing.md (100%) rename docs/docs/ru/{3_rabbit => rabbit}/1_routing.md (100%) rename docs/docs/ru/{3_rabbit => rabbit}/2_exchanges.md (100%) rename docs/docs/ru/{3_rabbit => rabbit}/3_queues.md (100%) rename docs/docs/ru/{3_rabbit => rabbit}/4_publishing.md (87%) rename docs/docs/ru/{3_rabbit => rabbit}/5_examples/1_direct.md (85%) rename docs/docs/ru/{3_rabbit => rabbit}/5_examples/2_fanout.md (79%) rename docs/docs/ru/{3_rabbit => rabbit}/5_examples/3_topic.md (81%) rename docs/docs/ru/{3_rabbit => rabbit}/5_examples/4_header.md (82%) create mode 100644 docs/docs/ru/redis/1_redis-index.md create mode 100644 docs/docs/ru/redis/2_publishing.md create mode 100644 docs/docs/ru/redis/3_examples/1_direct.md create mode 100644 docs/docs/ru/redis/3_examples/2_pattern.md create mode 100644 docs/docs_src/index/01_redis_base.py create mode 100644 docs/docs_src/index/02_redis_type_casting.py create mode 100644 docs/docs_src/index/03_redis_dependencies.py create mode 100644 docs/docs_src/index/04_redis_context.py create mode 100644 docs/docs_src/index/05_redis_http_example.py create mode 100644 docs/docs_src/index/06_redis_native_fastapi.py create mode 100644 docs/docs_src/integrations/fastapi_plugin_redis.py create mode 100644 docs/docs_src/integrations/fastapi_plugin_redis_depends.py create mode 100644 docs/docs_src/integrations/fastapi_plugin_redis_send.py create mode 100644 docs/docs_src/quickstart/app/1_broker_redis.py create mode 100644 docs/docs_src/quickstart/app/2_set_broker_redis.py create mode 100644 docs/docs_src/quickstart/broker/publishing/1_redis_inside_propan.py create mode 100644 docs/docs_src/quickstart/broker/publishing/2_redis_context.py create mode 100644 docs/docs_src/quickstart/broker/rpc/1_redis_handler.py create mode 100644 docs/docs_src/quickstart/broker/rpc/2_redis_blocking_client.py create mode 100644 docs/docs_src/quickstart/broker/rpc/3_redis_blocking_client_timeout.py create mode 100644 docs/docs_src/quickstart/broker/rpc/4_redis_blocking_client_timeout_none.py create mode 100644 docs/docs_src/quickstart/broker/rpc/5_redis_blocking_client_timeout_error.py create mode 100644 docs/docs_src/quickstart/broker/rpc/6_noblocking_client_redis.py create mode 100644 docs/docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py create mode 100644 docs/docs_src/quickstart/dependencies/basic/2_propan_redis_depends.py create mode 100644 docs/docs_src/quickstart/lifespan/1_redis.py create mode 100644 docs/docs_src/quickstart/lifespan/2_ml_redis.py rename docs/docs_src/{rabbit/testing/1_main.py => quickstart/testing/1_main_rabbit.py} (100%) create mode 100644 docs/docs_src/quickstart/testing/1_main_redis.py rename docs/docs_src/{rabbit/testing/2_test.py => quickstart/testing/2_test_rabbit.py} (100%) create mode 100644 docs/docs_src/quickstart/testing/2_test_redis.py rename docs/docs_src/{rabbit/testing/3_conftest.py => quickstart/testing/3_conftest_rabbit.py} (100%) create mode 100644 docs/docs_src/quickstart/testing/3_conftest_redis.py rename docs/docs_src/{rabbit/testing/4_suppressed_exc.py => quickstart/testing/4_suppressed_exc_rabbit.py} (100%) create mode 100644 docs/docs_src/quickstart/testing/4_suppressed_exc_redis.py rename docs/docs_src/{rabbit/testing/5_build_message.py => quickstart/testing/5_build_message_rabbit.py} (100%) create mode 100644 docs/docs_src/quickstart/testing/5_build_message_redis.py rename docs/docs_src/{rabbit/testing/6_reraise.py => quickstart/testing/6_reraise_rabbit.py} (100%) create mode 100644 docs/docs_src/quickstart/testing/6_reraise_redis.py rename docs/docs_src/rabbit/{examples => }/direct.py (90%) rename docs/docs_src/rabbit/{examples => }/fanout.py (82%) rename docs/docs_src/rabbit/{examples => }/header.py (82%) rename docs/docs_src/rabbit/{examples => }/topic.py (75%) create mode 100644 docs/docs_src/redis/direct.py create mode 100644 docs/docs_src/redis/pattern.py create mode 100644 examples/redis/direct.py create mode 100644 examples/redis/pattern.py create mode 100644 propan/brokers/redis/__init__.py create mode 100644 propan/brokers/redis/redis_broker.py create mode 100644 propan/brokers/redis/redis_broker.pyi create mode 100644 propan/brokers/redis/schemas.py delete mode 100644 propan/cli/startproject.py create mode 100644 propan/cli/startproject/__init__.py create mode 100644 propan/cli/startproject/app.py create mode 100644 propan/cli/startproject/async_app/__init__.py create mode 100644 propan/cli/startproject/async_app/app.py create mode 100644 propan/cli/startproject/async_app/core.py create mode 100644 propan/cli/startproject/async_app/nats.py create mode 100644 propan/cli/startproject/async_app/rabbit.py create mode 100644 propan/cli/startproject/async_app/redis.py create mode 100644 propan/cli/startproject/core.py create mode 100644 propan/cli/startproject/sync_app/__init__.py create mode 100644 propan/cli/startproject/sync_app/app.py create mode 100644 propan/cli/startproject/utils.py rename propan/fastapi/rabbit/{rabbit_router.py => router.py} (100%) rename propan/fastapi/rabbit/{rabbit_router.pyi => router.pyi} (100%) create mode 100644 propan/fastapi/redis/__init__.py create mode 100644 propan/fastapi/redis/router.py create mode 100644 propan/fastapi/redis/router.pyi create mode 100644 propan/test/redis.py create mode 100644 propan/test/utils.py create mode 100644 tests/brokers/redis/conftest.py create mode 100644 tests/brokers/redis/test_acc.py create mode 100644 tests/brokers/redis/test_connect.py create mode 100644 tests/brokers/redis/test_publish.py create mode 100644 tests/brokers/redis/test_test_client.py diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 06cf2a28..24fa2f38 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -6,8 +6,6 @@ on: - main paths: - docs/** - pull_request: - types: [opened, synchronize] permissions: contents: write diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 460354b8..eb0cf8a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,6 +20,11 @@ jobs: image: rabbitmq ports: - 5672:5672 + + redis: + image: redis + ports: + - 6379:6379 steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 2ed8ce8a..ba453eec 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,11 @@ # Propan -**Propan** - just *~~an another one HTTP~~* a **declarative Python MQ framework**. It's following by [*fastapi*](https://fastapi.tiangolo.com/ru/), -simplify Message Brokers around code writing and provides a helpful development toolkit, which existed only in HTTP-frameworks world until now. +**Propan** - just *~~an another one HTTP~~* a **declarative Python MQ framework**. It's following by *fastapi*, simplify Message Brokers around code writing and provides a helpful development toolkit, which existed only in HTTP-frameworks world until now. It's designed to create reactive microservices around Messaging Architecture. -It is a modern, high-level framework on top of popular specific Python brokers libraries, based on [*pydantic*](https://docs.pydantic.dev/) and [*fastapi*](https://fastapi.tiangolo.com/ru/), [*pytest*](https://docs.pytest.org/en/7.3.x/) concepts. +It is a modern, high-level framework on top of popular specific Python brokers libraries, based on *pydantic* and *fastapi*, *pytest* concepts. --- @@ -43,30 +42,34 @@ It is a modern, high-level framework on top of popular specific Python brokers l ### The key features are -* **Easy**: Designed to be easy to use and learn. +* **Simple**: Designed to be easy to use and learn. * **Intuitive**: Great editor support. Autocompletion everywhere. -* [**Dependencies management**](#dependencies): Minimize code duplication. Multiple features from each argument and parameter declaration. -* [**Integrations**](#http-frameworks-integrations): **Propan** is ready to use in pair with [any HTTP framework](https://lancetnik.github.io/Propan/5_integrations/1_integrations-index/) you want +* [**Dependencies management**](#dependencies): Minimization of code duplication. Access to dependencies at any level of the call stack. +* [**Integrations**](#http-frameworks-integrations): **Propan** is fully compatible with any HTTP framework you want * **MQ independent**: Single interface to popular MQ: - * **NATS** (based on [nats-py](https://github.com/nats-io/nats.py)) - * **RabbitMQ** (based on [aio-pika](https://aio-pika.readthedocs.io/en/latest/)) -* [**RPC**](https://lancetnik.github.io/Propan/2_getting_started/4_broker/5_rpc/): The framework supports RPC requests over MQ, which will allow performing long operations on remote services asynchronously. + * **Redis** (based on redis-py) + * **RabbitMQ** (based on aio-pika) + * **NATS** (based on nats-py) +* **RPC**: The framework supports RPC requests over MQ, which will allow performing long operations on remote services asynchronously. * [**Greate to develop**](#cli-power): CLI tool provides great development experience: - * framework-independent way to rule application environment - * application code hot reloading -* [**Testability**](https://lancetnik.github.io/Propan/2_getting_started/7_testing): **Propan** allows you to test your app without external dependencies: you shouldn't suit up a Message Broker, use a virtual one! + * framework-independent way to manage the project environment + * application code *hot reload* + * robust application templates +* **Testability**: **Propan** allows you to test your app without external dependencies: you do not have to set up a Message Broker, you can use a virtual one! ### Supported MQ brokers -| | async | sync | -|--------------|:-------------------------------------------------------:|:--------------------:| -| **RabbitMQ** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | -| **Nats** | :warning: **beta** :warning: | :mag: planning :mag: | -| **NatsJS** | :hammer_and_wrench: **in progress** :hammer_and_wrench: | :mag: planning :mag: | -| **MQTT** | :mag: planning :mag: | :mag: planning :mag: | -| **REDIS** | :mag: planning :mag: | :mag: planning :mag: | -| **Kafka** | :mag: planning :mag: | :mag: planning :mag: | -| **SQS** | :mag: planning :mag: | :mag: planning :mag: | +| | async | sync | +|-------------------|:-------------------------------------------------------:|:--------------------:| +| **RabbitMQ** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | +| **Redis** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | +| **Nats** | :warning: **beta** :warning: | :mag: planning :mag: | +| **NatsJS** | :hammer_and_wrench: **in progress** :hammer_and_wrench: | :mag: planning :mag: | +| **MQTT** | :mag: planning :mag: | :mag: planning :mag: | +| **Kafka** | :mag: planning :mag: | :mag: planning :mag: | +| **Redis Streams** | :mag: planning :mag: | :mag: planning :mag: | +| **Pulsar** | :mag: planning :mag: | :mag: planning :mag: | +| **SQS** | :mag: planning :mag: | :mag: planning :mag: | ### Community @@ -78,9 +81,9 @@ If you have any questions or ideas about features to implement, welcome to [disc ## Declarative? -With declarative tools you should define **what you need to get**. With traditional imperative tools you should write **what you need to do**. +With declarative tools you can define **what you need to get**. With traditional imperative tools you must write **what you need to do**. -Take a look at classic imperative tools, such as [aio-pika](https://aio-pika.readthedocs.io/en/latest/), [pika](https://pika.readthedocs.io/en/stable/), [nats-py](https://github.com/nats-io/nats.py), etc. +Take a look at classic imperative tools, such as aio-pika, pika, redis-py, nats-py, etc. This is the **Quickstart** with the *aio-pika*: @@ -108,9 +111,9 @@ async def main(): asyncio.run(main()) ``` -**aio-pika** is a really great tool with a really easy learning curve. But it's still imperative. You need to *connect*, declare *channel*, *queues*, *exchanges* by yourself. Also, you need to manage *connection*, *message*, *queue* context to avoid any troubles. +**aio-pika** is a great tool with a really easy learning curve. But it's still imperative. You need to *connect*, declare *channel*, *queues*, *exchanges* by yourself. Also, you need to manage *connection*, *message*, *queue* context to avoid any troubles. -It is not a bad way, but it can be easy. +It is not a bad way, but it can be much easier. ```python from propan import PropanApp, RabbitBroker @@ -145,10 +148,12 @@ Create an application with the following code at `serve.py`: ```python from propan import PropanApp from propan import RabbitBroker +# from propan import RedisBroker # from propan import NatsBroker broker = RabbitBroker("amqp://guest:guest@localhost:5672/") # broker = NatsBroker("nats://localhost:4222") +# broker = RedisBroker("redis://localhost:6379") app = PropanApp(broker) @@ -261,7 +266,7 @@ async def setup(env: str, context: ContextRepo): Also, **Propan CLI** is able to generate a production-ready application template: ```bash -propan create [projectname] +propan create async rabbit [projectname] ``` *Notice: project template require* `pydantic[dotenv]` *installation.* diff --git a/docs/docs.py b/docs/docs.py index 33354372..1198f9ec 100644 --- a/docs/docs.py +++ b/docs/docs.py @@ -6,6 +6,7 @@ import subprocess from yaml import load + try: from yaml import CLoader as Loader except ImportError: @@ -21,9 +22,9 @@ BASE_DIR = Path(__file__).resolve().parent CONFIG = BASE_DIR / "mkdocs.yml" DOCS_DIR = BASE_DIR / "docs" -LANGUAGES_DIRS = tuple(filter( - lambda f: f.is_dir() and f.name not in IGNORE_DIRS, DOCS_DIR.iterdir() -)) +LANGUAGES_DIRS = tuple( + filter(lambda f: f.is_dir() and f.name not in IGNORE_DIRS, DOCS_DIR.iterdir()) +) BUILD_DIR = BASE_DIR / "site" with CONFIG.open("r") as f: @@ -31,6 +32,7 @@ DEV_SERVER = config.get("dev_addr", "0.0.0.0:8000") + def get_missing_translation(lng: str) -> str: return str(Path(DOCS_DIR.name) / lng / "helpful" / "missing-translation.md") @@ -43,7 +45,7 @@ def get_in_progress(lng: str) -> str: def get_default_title(file: Path) -> str: - title = file.stem.upper().replace('-', ' ') + title = file.stem.upper().replace("-", " ") if title == "INDEX": title = get_default_title(file.parent) return title @@ -92,7 +94,7 @@ def build(): @app.command() -def add(path = typer.Argument(...)): +def add(path=typer.Argument(...)): title = "" exists = [] @@ -156,10 +158,9 @@ def mv(path: str = typer.Argument(...), new_path: str = typer.Argument(...)): typer.echo(f"{i / new_path} moved") - - def _build(): subprocess.run(["mkdocs", "build", "--site-dir", BUILD_DIR], check=True) + if __name__ == "__main__": app() diff --git a/docs/docs/en/4_nats/1_nats-index.md b/docs/docs/en/4_nats/1_nats-index.md deleted file mode 100644 index f63f9e99..00000000 --- a/docs/docs/en/4_nats/1_nats-index.md +++ /dev/null @@ -1,2 +0,0 @@ -# NATS -{! docs/en/helpful/in-progress.md !} \ No newline at end of file diff --git a/docs/docs/en/4_nats/2_routing.md b/docs/docs/en/4_nats/2_routing.md deleted file mode 100644 index c912d239..00000000 --- a/docs/docs/en/4_nats/2_routing.md +++ /dev/null @@ -1,2 +0,0 @@ -# NATS Routing -{! docs/en/helpful/in-progress.md !} \ No newline at end of file diff --git a/docs/docs/en/4_nats/3_publishing.md b/docs/docs/en/4_nats/3_publishing.md deleted file mode 100644 index 6a53ea5b..00000000 --- a/docs/docs/en/4_nats/3_publishing.md +++ /dev/null @@ -1,2 +0,0 @@ -# NATS Publishing -{! docs/en/helpful/in-progress.md !} \ No newline at end of file diff --git a/docs/docs/en/9_CHANGELOG.md b/docs/docs/en/CHANGELOG.md similarity index 76% rename from docs/docs/en/9_CHANGELOG.md rename to docs/docs/en/CHANGELOG.md index 8a783b9f..f49d9ded 100644 --- a/docs/docs/en/9_CHANGELOG.md +++ b/docs/docs/en/CHANGELOG.md @@ -1,5 +1,22 @@ # CHANGELOG +## 2023-05-18 **0.1.1.0** REDIS + +**Propan** added support for *Redis Pub/Sub* as a message broker. This functionality is fully tested and described in the documentation. + +*RedisBroker* supports: + +* message delivery by key or pattern +* test client, without the need to run *Redis* +* **RPC** requests over *Redis Pub/Sub* +* *FastAPI* Plugin + +Also, **Propan CLI** is able to generate templates to any supported broker + +```bash +propan create async [broker] [APPNAME] +``` + ## 2023-05-15 **0.1.0.0** STABLE Stable and fully documented **Propan** release! @@ -28,9 +45,9 @@ async def hello(m: dict) -> dict: app.include_router(router) ``` -You can find a complete example in [documentation](../5_integrations/2_fastapi-plugin) +You can find a complete example in [documentation](../integrations/2_fastapi-plugin) -Also, added the ability to [test](../2_getting_started/7_testing) your application without running external dependencies as a broker (for now only for RabbitMQ)! +Also, added the ability to [test](../getting_started/7_testing) your application without running external dependencies as a broker (for now only for RabbitMQ)! ```python from propan import RabbitBroker @@ -49,7 +66,7 @@ def test_publish(): assert r == "pong" ``` -Also added support for [RPC over MQ](../2_getting_started/4_broker/5_rpc) (RabbitMQ only for now): `return` of your handler function will be sent in response to a message if a response is expected. +Also added support for [RPC over MQ](../getting_started/4_broker/5_rpc) (RabbitMQ only for now): `return` of your handler function will be sent in response to a message if a response is expected.

Breaking changes:

diff --git a/docs/docs/en/7_alternatives.md b/docs/docs/en/alternatives.md similarity index 100% rename from docs/docs/en/7_alternatives.md rename to docs/docs/en/alternatives.md diff --git a/docs/docs/en/6_contributing/1_todo.md b/docs/docs/en/contributing/1_todo.md similarity index 99% rename from docs/docs/en/6_contributing/1_todo.md rename to docs/docs/en/contributing/1_todo.md index 69b3562b..13b6c63c 100644 --- a/docs/docs/en/6_contributing/1_todo.md +++ b/docs/docs/en/contributing/1_todo.md @@ -28,4 +28,4 @@ this will allow you to keep a common counter for all consumers. To start developing the project, go to the following [section](../2_contributing-index/). -If you want to develop your own adapter for any broker, you will find a lot of useful information in [this](../4_adapters/) section. \ No newline at end of file +If you want to develop your own adapter for any broker, you will find a lot of useful information in [this](../4_adapters/) section. diff --git a/docs/docs/en/6_contributing/2_contributing-index.md b/docs/docs/en/contributing/2_contributing-index.md similarity index 100% rename from docs/docs/en/6_contributing/2_contributing-index.md rename to docs/docs/en/contributing/2_contributing-index.md diff --git a/docs/docs/en/6_contributing/3_docs.md b/docs/docs/en/contributing/3_docs.md similarity index 100% rename from docs/docs/en/6_contributing/3_docs.md rename to docs/docs/en/contributing/3_docs.md diff --git a/docs/docs/en/6_contributing/4_adapters.md b/docs/docs/en/contributing/4_adapters.md similarity index 100% rename from docs/docs/en/6_contributing/4_adapters.md rename to docs/docs/en/contributing/4_adapters.md diff --git a/docs/docs/en/2_getting_started/1_quick-start.md b/docs/docs/en/getting_started/1_quick-start.md similarity index 79% rename from docs/docs/en/2_getting_started/1_quick-start.md rename to docs/docs/en/getting_started/1_quick-start.md index 1d272465..d1bd6b20 100644 --- a/docs/docs/en/2_getting_started/1_quick-start.md +++ b/docs/docs/en/getting_started/1_quick-start.md @@ -1,7 +1,20 @@ -# QUICK START +# QUICK START Install using `pip`: +=== "Redis" +
+ ```console + $ pip install "propan[async-redis]" + ---> 100% + ``` +
+ !!! tip + To working with project start a test broker container + ```bash + docker run -d --rm -p 6379:6379 --name test-mq redis + ``` + === "RabbitMQ"
```console @@ -32,6 +45,11 @@ Install using `pip`: Create an application with the following code at `serve.py`: +=== "Redis" + ```python linenums="1" + {!> docs_src/index/01_redis_base.py!} + ``` + === "RabbitMQ" ```python linenums="1" {!> docs_src/index/01_rabbit_base.py!} @@ -60,6 +78,11 @@ $ propan run serve:app Propan uses `pydantic` to cast incoming function arguments to types according to their annotation. +=== "Redis" + ```python linenums="1" hl_lines="12" + {!> docs_src/index/02_redis_type_casting.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="12" {!> docs_src/index/02_rabbit_type_casting.py!} @@ -83,6 +106,10 @@ If you call a non-existent field, raises *pydantic.error_wrappers.ValidationErro But you can specify your own dependencies, call dependencies functions (like `Fastapi Depends`) and [more](../5_dependency/1_di-index). +=== "Redis" + ```python linenums="1" hl_lines="11-12" + {!> docs_src/index/03_redis_dependencies.py!} + ``` === "RabbitMQ" ```python linenums="1" hl_lines="11-12" @@ -102,7 +129,7 @@ Also, **Propan CLI** is able to generate a production-ready application template
```console -$ propan create [projectname] +$ propan create async rabbit [projectname] Create Propan project template at: /home/user/projectname ```
@@ -138,6 +165,11 @@ Now you can enjoy a new development experience! You can use **Propan** `MQBrokers` without `PropanApp`. Just *start* and *stop* them according to your application lifespan. +=== "Redis" + ```python linenums="1" hl_lines="5 11-13 16-17" + {!> docs_src/index/05_redis_http_example.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="5 11-13 16-17" {!> docs_src/index/05_rabbit_http_example.py!} @@ -159,12 +191,18 @@ using the `@event` decorator. This decorator is similar to the decorator `@handl When used this way, **Propan** does not utilize its own dependency system, but integrates into **FastAPI**. That is, you can use `Depends`, `Background Tasks` and other tools **Facet API** as if it were a regular HTTP endpoint. -```python linenums="1" hl_lines="7 15 19" -{!> docs_src/index/06_rabbit_native_fastapi.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="7 15 19" + {!> docs_src/index/06_redis_native_fastapi.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="7 15 19" + {!> docs_src/index/06_rabbit_native_fastapi.py!} + ``` !!! note - More integration examples you can find [here](../../5_integrations/1_integrations-index/) + More integration examples you can find [here](../../integrations/1_integrations-index/) ??? tip "Don't forget to stop test broker container" ```bash diff --git a/docs/docs/en/2_getting_started/2_cli.md b/docs/docs/en/getting_started/2_cli.md similarity index 95% rename from docs/docs/en/2_getting_started/2_cli.md rename to docs/docs/en/getting_started/2_cli.md index 3c9c49ca..f310902f 100644 --- a/docs/docs/en/2_getting_started/2_cli.md +++ b/docs/docs/en/getting_started/2_cli.md @@ -33,7 +33,7 @@ To start a new project not from scratch, you can use the standard template **Pro
```console -$ propan create app +$ propan create async rabbit app Create Propan project template at: ./app ``` @@ -79,6 +79,11 @@ $ propan run serve:app --env=.env.dev ```
+=== "Redis" + ```python linenums="1" hl_lines="3 14-15" + {!> docs_src/index/04_redis_context.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="3 14-15" {!> docs_src/index/04_rabbit_context.py!} diff --git a/docs/docs/en/2_getting_started/3_app.md b/docs/docs/en/getting_started/3_app.md similarity index 87% rename from docs/docs/en/2_getting_started/3_app.md rename to docs/docs/en/getting_started/3_app.md index cffb5616..91edfc76 100644 --- a/docs/docs/en/2_getting_started/3_app.md +++ b/docs/docs/en/getting_started/3_app.md @@ -16,6 +16,11 @@ In order for `PropanApp` to launch your broker, you need to put it in the applic This is usually done when declaring the application itself +=== "Redis" + ```python + {!> docs_src/quickstart/app/1_broker_redis.py!} + ``` + === "RabbitMQ" ```python {!> docs_src/quickstart/app/1_broker_rabbit.py!} @@ -28,6 +33,11 @@ This is usually done when declaring the application itself But, sometimes you may need to initialize the broker elsewhere. In this case, you can use the `app.set_broker` method +=== "Redis" + ```python + {!> docs_src/quickstart/app/2_set_broker_redis.py!} + ``` + === "RabbitMQ" ```python {!> docs_src/quickstart/app/2_set_broker_rabbit.py!} diff --git a/docs/docs/en/2_getting_started/4_broker/1_index.md b/docs/docs/en/getting_started/4_broker/1_index.md similarity index 72% rename from docs/docs/en/2_getting_started/4_broker/1_index.md rename to docs/docs/en/getting_started/4_broker/1_index.md index 4a12ea0f..abde3f7d 100644 --- a/docs/docs/en/2_getting_started/4_broker/1_index.md +++ b/docs/docs/en/getting_started/4_broker/1_index.md @@ -4,6 +4,11 @@ **Propan** supports various message brokers using special classes +=== "Redis" + ```python + from propan import RedisBroker + ``` + === "RabbitMQ" ```python from propan import RabbitBroker @@ -18,6 +23,11 @@ Be careful! Different brokers require different dependencies. If you have not in To install **Propan** with the necessary dependencies for your broker, select one of the installation options +=== "Redis" + ```bash + pip install "propan[async-redis]" + ``` + === "RabbitMQ" ```bash pip install "propan[async-rabbit]" @@ -32,16 +42,35 @@ To install **Propan** with the necessary dependencies for your broker, select on Data for connecting **Propan Broker** to your message broker can be transmitted in 2 ways: +=== "Redis" + 1. In the broker constructor + + ```python + from propan import RedisBroker + broker = RedisBroker("redis://localhost:6379/") + ``` + + 2. In the `connect` method + ```python + + from propan import RedisBroker + broker = RedisBroker() + ... + await broker.connect("redis://localhost:6379/") + ``` + === "RabbitMQ" 1. In the broker constructor + ```python - from propan.brokers.rabbit import RabbitBroker + from propan import RabbitBroker broker = RabbitBroker("amqp://guest:guest@localhost:5672/") ``` 2. In the `connect` method + ```python - from propan.brokers.rabbit import RabbitBroker + from propan import RabbitBroker broker = RabbitBroker() ... await broker.connect("amqp://guest:guest@localhost:5672/") @@ -49,14 +78,16 @@ Data for connecting **Propan Broker** to your message broker can be transmitted === "NATS" 1. In the broker constructor + ```python - from propan.brokers.nats import NatsBroker + from propan import NatsBroker broker = NatsBroker("nats://localhost:4222") ``` 2. In the `connect` method + ```python - from propan.brokers.nats import NatsBroker + from propan import NatsBroker broker = NatsBroker() ... await broker.connect("nats://localhost:4222") @@ -70,4 +101,4 @@ However, in more complex scenarios: for example, when configuring a project via The parameters passed to `connect` overrides the parameters passed to the constructor. Be careful with this. In addition, calling `connect` again has no effect. Therefore, you may not worry that `broker.start()` call - (used inside 'PropanApp` to run the broker) will cause any errors. + (used inside `PropanApp` to run the broker) will cause any errors. diff --git a/docs/docs/en/2_getting_started/4_broker/2_routing.md b/docs/docs/en/getting_started/4_broker/2_routing.md similarity index 94% rename from docs/docs/en/2_getting_started/4_broker/2_routing.md rename to docs/docs/en/getting_started/4_broker/2_routing.md index c5812cca..93db5a9e 100644 --- a/docs/docs/en/2_getting_started/4_broker/2_routing.md +++ b/docs/docs/en/getting_started/4_broker/2_routing.md @@ -14,8 +14,8 @@ This behavior is similar for all brokers, however, the parameters passed to `@br To learn more about the behavior of specialized brokers, go to the following sections: -* [RabbitBroker](../../../3_rabbit/1_routing) -* [NatsBroker](../../../4_nats/2_routing) +* [RabbitBroker](../../../rabbit/1_routing) +* [NatsBroker](../../../nats/2_routing) ## Error handling diff --git a/docs/docs/en/2_getting_started/4_broker/3_type-casting.md b/docs/docs/en/getting_started/4_broker/3_type-casting.md similarity index 100% rename from docs/docs/en/2_getting_started/4_broker/3_type-casting.md rename to docs/docs/en/getting_started/4_broker/3_type-casting.md diff --git a/docs/docs/en/2_getting_started/4_broker/4_publishing.md b/docs/docs/en/getting_started/4_broker/4_publishing.md similarity index 80% rename from docs/docs/en/2_getting_started/4_broker/4_publishing.md rename to docs/docs/en/getting_started/4_broker/4_publishing.md index 5a70a39e..0c1f7495 100644 --- a/docs/docs/en/2_getting_started/4_broker/4_publishing.md +++ b/docs/docs/en/getting_started/4_broker/4_publishing.md @@ -11,14 +11,16 @@ specific to different brokers. You can get acquainted with all the features specific to your broker here: -* [RabbitBroker](../../../3_rabbit/4_publishing) -* [NatsBroker](../../../4_nats/3_publishing) +* [Redis](../../../redis/2_publishing) +* [RabbitBroker](../../../rabbit/4_publishing) +* [NatsBroker](../../../nats/3_publishing) ## Valid types to submit | Type | Send header | Method of casting to bytes | | -------------------- | --------------------- | ---------------------------------- | | `dict` | application/json | json.dumps(message).encode() | +| `Sequence` | application/json | json.dumps(message).encode() | | `pydantic.BaseModel` | application/json | message.json().encode() | | `str` | text/plain | message.encode() | | `bytes` | | message | @@ -32,6 +34,11 @@ To send a message to a queue, you must first connect to it. If you are inside a running **Propan** application, you don't need to do anything: the broker is already running. Just access it and send a message. +=== "Redis" + ```python linenums="1" hl_lines="8" + {!> docs_src/quickstart/broker/publishing/1_redis_inside_propan.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="8" {!> docs_src/quickstart/broker/publishing/1_rabbit_inside_propan.py !} @@ -45,6 +52,11 @@ Just access it and send a message. If you are only using **Propan** to send asynchronous messages within another framework, you can use broker as context manager to send. +=== "Redis" + ```python + {!> docs_src/quickstart/broker/publishing/2_redis_context.py !} + ``` + === "RabbitMQ" ```python {!> docs_src/quickstart/broker/publishing/2_rabbit_context.py !} diff --git a/docs/docs/en/2_getting_started/4_broker/5_rpc.md b/docs/docs/en/getting_started/4_broker/5_rpc.md similarity index 77% rename from docs/docs/en/2_getting_started/4_broker/5_rpc.md rename to docs/docs/en/getting_started/4_broker/5_rpc.md index 0bcf9e7e..fc113e1b 100644 --- a/docs/docs/en/2_getting_started/4_broker/5_rpc.md +++ b/docs/docs/en/getting_started/4_broker/5_rpc.md @@ -22,7 +22,12 @@ From the server side (the receiving side), you do not need to change the code: ` !!! note The result of your function must match the valid types of the `message` parameter of the `broker.publish` function. - Acceptable types are `str`, `dict`, `pydantic.BaseModel`, `bytes` and a native message of the library used for the broker. + Acceptable types are `str`, `dict`, `Sequence`, `pydantic.BaseModel`, `bytes` and a native message of the library used for the broker. + +=== "Redis" + ```python linenums="1" hl_lines="7" + {!> docs_src/quickstart/broker/rpc/1_redis_handler.py !} + ``` === "RabbitMQ" ```python linenums="1" hl_lines="7" @@ -35,6 +40,11 @@ From the server side (the receiving side), you do not need to change the code: ` To wait for the result of executing the request "right here" (as if it were an HTTP request), you just need to specify the parameter `callback=True` when sending the message. +=== "Redis" + ```python linenums="1" hl_lines="8" + {!> docs_src/quickstart/broker/rpc/2_redis_blocking_client.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="8" {!> docs_src/quickstart/broker/rpc/2_rabbit_blocking_client.py !} @@ -42,6 +52,13 @@ To wait for the result of executing the request "right here" (as if it were an H To set the time that the client is ready to wait for a response from the server, use the`callback_timeout` parameter (by default - **30** seconds) +=== "Redis" + ```python linenums="1" hl_lines="5" + {!> docs_src/quickstart/broker/rpc/3_redis_blocking_client_timeout.py !} + ``` + + 1. Waits for the result for 3 seconds + === "RabbitMQ" ```python linenums="1" hl_lines="5" {!> docs_src/quickstart/broker/rpc/3_rabbit_blocking_client_timeout.py !} @@ -51,6 +68,11 @@ To set the time that the client is ready to wait for a response from the server, If you are ready to wait for a response as long as it takes, you can set `callback_timeout=None` +=== "Redis" + ```python linenums="1" hl_lines="5" + {!> docs_src/quickstart/broker/rpc/4_redis_blocking_client_timeout_none.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="5" {!> docs_src/quickstart/broker/rpc/4_rabbit_blocking_client_timeout_none.py !} @@ -61,6 +83,11 @@ If you are ready to wait for a response as long as it takes, you can set `callba By default, if **Propan** did not wait for the server response, the function will return `None`. If you want to explicitly process `asyncio.TimeoutError`, use the `raise_timeout` parameter. +=== "Redis" + ```python linenums="1" hl_lines="5" + {!> docs_src/quickstart/broker/rpc/5_redis_blocking_client_timeout_error.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="5" {!> docs_src/quickstart/broker/rpc/5_rabbit_blocking_client_timeout_error.py !} @@ -70,6 +97,11 @@ By default, if **Propan** did not wait for the server response, the function wil To process the response outside of the main execution loop, you can initialize a handler and then pass its queue as the `reply_to` argument of the request. +=== "Redis" + ```python linenums="1" hl_lines="6 16" + {!> docs_src/quickstart/broker/rpc/6_noblocking_client_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="6 16" {!> docs_src/quickstart/broker/rpc/6_noblocking_client_rabbit.py !} diff --git a/docs/docs/en/2_getting_started/5_dependency/1_di-index.md b/docs/docs/en/getting_started/5_dependency/1_di-index.md similarity index 88% rename from docs/docs/en/2_getting_started/5_dependency/1_di-index.md rename to docs/docs/en/getting_started/5_dependency/1_di-index.md index 8a9e757e..044c8ac4 100644 --- a/docs/docs/en/2_getting_started/5_dependency/1_di-index.md +++ b/docs/docs/en/getting_started/5_dependency/1_di-index.md @@ -11,6 +11,12 @@ The key function in the dependency management and type conversion system in *Pro By default, it applies to all event handlers, unless you disabled the same option at a broker creation. +=== "Redis" + ```python + from propan import RedisBroker + broker = RedisBroker(..., apply_types=False) + ``` + === "RabbitMQ" ```python from propan import RabbitBroker @@ -33,6 +39,11 @@ a native dependency system. To implement dependencies in **Propan**, a special class **Depends** is used +=== "Redis" + ```python linenums="1" hl_lines="6-7" + {!> docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="6-7" {!> docs_src/quickstart/dependencies/basic/1_propan_rabbit_depends.py !} @@ -50,6 +61,11 @@ To implement dependencies in **Propan**, a special class **Depends** is used In other words: if you can write such code `my_object()` - `my_object` will be `Callable` +=== "Redis" + ```python hl_lines="1" linenums="10" + {!> docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py [ln:10-11]!} + ``` + === "RabbitMQ" ```python hl_lines="1" linenums="10" {!> docs_src/quickstart/dependencies/basic/1_propan_rabbit_depends.py [ln:10-11]!} @@ -62,6 +78,11 @@ To implement dependencies in **Propan**, a special class **Depends** is used **Second step**: Declare which dependencies you need using `Depends` +=== "Redis" + ```python hl_lines="2" linenums="10" + {!> docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py [ln:10-11]!} + ``` + === "RabbitMQ" ```python hl_lines="2" linenums="10" {!> docs_src/quickstart/dependencies/basic/1_propan_rabbit_depends.py [ln:10-11]!} @@ -78,13 +99,20 @@ It's easy, isn't it? !!! tip "Auto @apply_types" In the code above, we didn't use this decorator for our dependencies. However, it still applies - to all functions used as dependencies. Keep this in your mind. + to all functions used as dependencies. Keep this in your mind. ## Nested dependencies Dependencies can also contain other dependencies. This works in a very predictable way: just declare `Depends` in the dependent function. +=== "Redis" + ```python linenums="1" hl_lines="6-7 9-10 15-16" + {!> docs_src/quickstart/dependencies/basic/2_propan_redis_depends.py !} + ``` + + 1. A nested dependency is called here + === "RabbitMQ" ```python linenums="1" hl_lines="6-7 9-10 15-16" {!> docs_src/quickstart/dependencies/basic/2_propan_rabbit_depends.py !} @@ -108,7 +136,6 @@ Dependencies can also contain other dependencies. This works in a very predictab To prevent this behavior, just use `Depends(..., cache=False)`. In this case, the dependency will be used for each function in the call stack where it is used. - ## Use with regular functions You can use the decorator `@apply_types` not only together with your `@broker.hanle', but also with the usual functions: both synchronous and asynchronous. diff --git a/docs/docs/en/2_getting_started/5_dependency/2_context.md b/docs/docs/en/getting_started/5_dependency/2_context.md similarity index 100% rename from docs/docs/en/2_getting_started/5_dependency/2_context.md rename to docs/docs/en/getting_started/5_dependency/2_context.md diff --git a/docs/docs/en/2_getting_started/6_lifespans.md b/docs/docs/en/getting_started/6_lifespans.md similarity index 67% rename from docs/docs/en/2_getting_started/6_lifespans.md rename to docs/docs/en/getting_started/6_lifespans.md index adc0ae35..25bf76dd 100644 --- a/docs/docs/en/2_getting_started/6_lifespans.md +++ b/docs/docs/en/getting_started/6_lifespans.md @@ -1,4 +1,4 @@ -# LIFESPANS +# LIFESPANS Sometimes you need to define the logic that should be executed before launching the application. This means that the code will be executed once - even before your application starts receiving messages. @@ -10,7 +10,6 @@ Since this code is executed before the application starts and after it stops, it This can be very useful for initializing your application settings at startup, raising a pool of connections to a database, or running machine learning models. - ## Usage example Let's imagine that your application uses **pydantic** as your settings manager. @@ -28,6 +27,11 @@ By [passing optional arguments with the command line](../2_cli/#environment-mana Let's write some code for our example +=== "Redis" + ```python linenums="1" hl_lines="12-16" + {!> docs_src/quickstart/lifespan/1_redis.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="12-16" {!> docs_src/quickstart/lifespan/1_rabbit.py!} @@ -39,38 +43,89 @@ Let's write some code for our example ``` Now this application can be run using the following command to manage the environment: + ```bash -$ propan run serve:app --env .env.test +propan run serve:app --env .env.test ``` ### Details Now let's look into a little more detail -To begin with, we used a decorator -```python linenums="12" hl_lines="1" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` +To begin with, we used a decorator + +=== "Redis" + ```python linenums="12" hl_lines="1" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="1" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="1" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` + to declare a function that should run when our application starts The next step is to declare the arguments that our function will receive -```python linenums="12" hl_lines="2" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` + +=== "Redis" + ```python linenums="12" hl_lines="2" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="2" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="2" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` + In this case, the `env` field will be passed to the `setup` function from the arguments with the command line !!! tip The default lifecycle functions are used with the decorator `@apply_types', therefore, all [context fields](../5_dependency/2_context) and [dependencies](../5_dependency/1_di-index) are available in them Then, we initialized the settings of our application using the file passed to us from the command line -```python linenums="12" hl_lines="3" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` + +=== "Redis" + ```python linenums="12" hl_lines="3" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="3" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="3" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` And put these settings in a global context -```python linenums="12" hl_lines="4" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` + +=== "Redis" + ```python linenums="12" hl_lines="4" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="4" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="4" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` ??? note Now we can access our settings anywhere in the application right from the context @@ -82,6 +137,12 @@ And put these settings in a global context ``` The last step we initialized our broker: now, when the application starts, it will be ready to receive messages + +=== "Redis" + ```python linenums="12" hl_lines="5" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + === "RabbitMQ" ```python linenums="12" hl_lines="5" {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} @@ -105,6 +166,11 @@ Therefore, it is worth initializing the model in the `@app.on_startup` hook. Also, we don't want the model to finish its work incorrectly when the application is stopped. To avoid this, we need the hook `@app.on_shutdown` +=== "Redis" + ```python linenums="1" hl_lines="12 18" + {!> docs_src/quickstart/lifespan/2_ml_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="12 18" {!> docs_src/quickstart/lifespan/2_ml_rabbit.py !} @@ -126,12 +192,14 @@ If you want to declare multiple lifecycle hooks, they will be used in the order ## Some more details ### Async or not async + In the asynchronous version of the application, both asynchronous and synchronous methods can be used as hooks. In the synchronous version, only synchronous methods are available. ### Command line arguments -Command line arguments are available in all `@app.on_startup` hooks. To use them in other parts of the application -, put them in the `ContextRepo'. + +Command line arguments are available in all `@app.on_startup` hooks. To use them in other parts of the application, put them in the `ContextRepo`. ### Broker initialization -The '@app.on_startup` hooks are called **BEFORE** the broker is launched by the application. The '@app.on_shutdown` hooks are triggered **AFTER** stopping the broker. \ No newline at end of file + +The `@app.on_startup` hooks are called **BEFORE** the broker is launched by the application. The `@app.after_shutdown` hooks are triggered **AFTER** stopping the broker. diff --git a/docs/docs/en/2_getting_started/7_testing.md b/docs/docs/en/getting_started/7_testing.md similarity index 60% rename from docs/docs/en/2_getting_started/7_testing.md rename to docs/docs/en/getting_started/7_testing.md index 9790dcb1..aac17702 100644 --- a/docs/docs/en/2_getting_started/7_testing.md +++ b/docs/docs/en/getting_started/7_testing.md @@ -11,38 +11,64 @@ Also, the absence of dependencies helps to avoid tests failure, based on an erro Let's image we have an application like so: +=== "Redis" + ```python linenums="1" title="main.py" + {!> docs_src/quickstart/testing/1_main_redis.py!} + ``` + + In order to test it without running *Redis*, you need to modify the broker with `propan.test.TestRedisBroker`: + + ```python linenums="1" title="test_ping.py" hl_lines="1 6" + {!> docs_src/quickstart/testing/2_test_redis.py!} + ``` + === "RabbitMQ" ```python linenums="1" title="main.py" - {!> docs_src/rabbit/testing/1_main.py!} + {!> docs_src/quickstart/testing/1_main_rabbit.py!} ``` In order to test it without running *RabbitMQ*, you need to modify the broker with `propan.test.TestRabbitBroker`: ```python linenums="1" title="test_ping.py" hl_lines="1 6" - {!> docs_src/rabbit/testing/2_test.py!} + {!> docs_src/quickstart/testing/2_test_rabbit.py!} ``` This is done with the `start` method: +=== "Redis" + ```python hl_lines="2" + {!> docs_src/quickstart/testing/2_test_redis.py [ln:6-8]!} + ``` + === "RabbitMQ" ```python hl_lines="2" - {!> docs_src/rabbit/testing/2_test.py [ln:6-8]!} + {!> docs_src/quickstart/testing/2_test_rabbit.py [ln:6-8]!} ``` Then make an *RPC* request to check the result of the execution: +=== "Redis" + ```python hl_lines="3" + {!> docs_src/quickstart/testing/2_test_redis.py [ln:6-8]!} + ``` + === "RabbitMQ" ```python hl_lines="3" - {!> docs_src/rabbit/testing/2_test.py [ln:6-8]!} + {!> docs_src/quickstart/testing/2_test_rabbit.py [ln:6-8]!} ``` ## Using fixtures For large applications to reuse the test broker, you can use the following fixture: +=== "Redis" + ```python linenums="1" title="test_broker.py" hl_lines="6-10 12" + {!> docs_src/quickstart/testing/3_conftest_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" title="test_broker.py" hl_lines="6-10 12" - {!> docs_src/rabbit/testing/3_conftest.py !} + {!> docs_src/quickstart/testing/3_conftest_rabbit.py !} ``` ## Regular function calling @@ -52,9 +78,14 @@ For large applications to reuse the test broker, you can use the following fixtu For example, the following test will return `None` and inside the handler, a `pydantic.ValidationError` will be raised: +=== "Redis" + ```python hl_lines="4 6" + {!> docs_src/quickstart/testing/4_suppressed_exc_redis.py !} + ``` + === "RabbitMQ" ```python hl_lines="4 6" - {!> docs_src/rabbit/testing/4_suppressed_exc.py !} + {!> docs_src/quickstart/testing/4_suppressed_exc_rabbit.py !} ``` Also this test will be blocked for `callback_timeout` (default **30** seconds), which can be very annoying when a handler development error occures, and your tests fail with a long timeout of `None`. @@ -63,16 +94,26 @@ Therefore, **Propan** provides the ability to run handler functions as if they w To do this, you need to construct a message using the `build_message`, if it was `pubslih` (same method signatures), and passe this message to your handler as the single function argument. +=== "Redis" + ```python linenums="1" hl_lines="6-7" title="test_ping.py" + {!> docs_src/quickstart/testing/5_build_message_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="6-7" title="test_ping.py" - {!> docs_src/rabbit/testing/5_build_message.py !} + {!> docs_src/quickstart/testing/5_build_message_rabbit.py !} ``` That being said, if you want to catch handler exceptions, you need to use the `reraise_exc=True` calling flag: +=== "Redis" + ```python linenums="1" hl_lines="9-10" title="test_ping.py" + {!> docs_src/quickstart/testing/6_reraise_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="9-10" title="test_ping.py" - {!> docs_src/rabbit/testing/6_reraise.py !} + {!> docs_src/quickstart/testing/6_reraise_rabbit.py !} ``` Thus, **Propan** provides you with a complete toolkit for testing your handlers, from checking *RPC* responses to correctly executing body functions. diff --git a/docs/docs/en/2_getting_started/8_logging.md b/docs/docs/en/getting_started/8_logging.md similarity index 100% rename from docs/docs/en/2_getting_started/8_logging.md rename to docs/docs/en/getting_started/8_logging.md diff --git a/docs/docs/en/8_help.md b/docs/docs/en/help.md similarity index 100% rename from docs/docs/en/8_help.md rename to docs/docs/en/help.md diff --git a/docs/docs/en/index.md b/docs/docs/en/index.md index f9d8d159..d7427211 100644 --- a/docs/docs/en/index.md +++ b/docs/docs/en/index.md @@ -24,7 +24,6 @@

- # Propan **Propan** - just *another one HTTP* a **declarative Python MQ framework**. It it inspired by [*fastapi*](https://fastapi.tiangolo.com/ru/){target="_blank"}, aims to simplify writing code that works with Message Brokers and provides a helpful development toolkit, which existed only in HTTP-frameworks world until now. @@ -39,16 +38,18 @@ It is a modern, high-level framework on top of popular Python libraries for vari * **Simple**: Designed to be easy to use and learn. * **Intuitive**: Great editor support. Autocompletion everywhere. -* [**Dependencies management**](2_getting_started/1_quick-start/#dependencies): Minimization of code duplication. Access to dependencies at any level of the call stack. -* [**Integrations**](2_getting_started/1_quick-start/#http-frameworks-integrations): **Propan** is fully compatible with [any HTTP framework](5_integrations/1_integrations-index/) you want +* [**Dependencies management**](getting_started/1_quick-start/#dependencies): Minimization of code duplication. Access to dependencies at any level of the call stack. +* [**Integrations**](getting_started/1_quick-start/#http-frameworks-integrations): **Propan** is fully compatible with [any HTTP framework](integrations/1_integrations-index/) you want * **MQ independent**: Single interface to popular MQ: - * **NATS** (based on [nats-py](https://github.com/nats-io/nats.py)) - * **RabbitMQ** (based on [aio-pika](https://aio-pika.readthedocs.io/en/latest/)) -* [**RPC**](2_getting_started/4_broker/5_rpc/): The framework supports RPC requests on top of message brokers, which will allow performing long operations on remote services asynchronously. -* [**Greate to develop**](2_getting_started/2_cli/): CLI tool provides great development experience: + * **Redis** (based on [redis-py]("https://redis.readthedocs.io/en/stable/index.html"){target="_blank"}) + * **RabbitMQ** (based on [aio-pika](https://aio-pika.readthedocs.io/en/latest/){target="_blank"}) + * **NATS** (based on [nats-py](https://github.com/nats-io/nats.py){target="_blank"}) +* [**RPC**](getting_started/4_broker/5_rpc/): The framework supports RPC requests on top of message brokers, which will allow performing long operations on remote services asynchronously. +* [**Greate to develop**](getting_started/2_cli/): CLI tool provides great development experience: * framework-independent way to manage the project environment - * application code hot reload -* [**Testability**](2_getting_started/7_testing): **Propan** allows you to test your app without external dependencies: you do not have to set up a Message Broker, you can use a virtual one! + * application code *hot reload* + * robust application templates +* [**Testability**](getting_started/7_testing): **Propan** allows you to test your app without external dependencies: you do not have to set up a Message Broker, you can use a virtual one! --- @@ -56,7 +57,7 @@ It is a modern, high-level framework on top of popular Python libraries for vari With declarative tools you can define **what you need to get**. With traditional imperative tools you must write **what you need to do**. -Take a look at classic imperative tools, such as [aio-pika](https://aio-pika.readthedocs.io/en/latest/){target="_blank"}, [pika](https://pika.readthedocs.io/en/stable/){target="_blank"}, [nats-py](https://github.com/nats-io/nats.py){target="_blank"}, etc. +Take a look at classic imperative tools, such as [aio-pika](https://aio-pika.readthedocs.io/en/latest/){target="_blank"}, [pika](https://pika.readthedocs.io/en/stable/){target="_blank"}, [redis-py]("https://redis.readthedocs.io/en/stable/index.html"){target="_blank"}, [nats-py](https://github.com/nats-io/nats.py){target="_blank"}, etc. This is the **Quickstart** with the *aio-pika*: @@ -107,15 +108,16 @@ This is the **Propan** declarative way to write the same code. That is so much e ## Supported MQ brokers !!! note "Need your help" - The framework is now in active development. We have a very long list of what has yet to be implemented and various brokers are only part of it. If you want to implement something from this list or help in any other way, take a look [here](6_contributing/1_todo/) - -| | async | sync | -|--------------|:-------------------------------------------------------:|:--------------------:| -| **RabbitMQ** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | -| **Nats** | :warning: **beta** :warning: | :mag: planning :mag: | -| **NatsJS** | :hammer_and_wrench: **in progress** :hammer_and_wrench: | :mag: planning :mag: | -| **MQTT** | :mag: planning :mag: | :mag: planning :mag: | -| **Redis** | :mag: planning :mag: | :mag: planning :mag: | -| **Kafka** | :mag: planning :mag: | :mag: planning :mag: | -| **Pulsar** | :mag: planning :mag: | :mag: planning :mag: | -| **SQS** | :mag: planning :mag: | :mag: planning :mag: | + The framework is now in active development. We have a very long list of what has yet to be implemented and various brokers are only part of it. If you want to implement something from this list or help in any other way, take a look [here](contributing/1_todo/) + +| | async | sync | +|-------------------|:-------------------------------------------------------:|:--------------------:| +| **RabbitMQ** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | +| **Redis** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | +| **Nats** | :warning: **beta** :warning: | :mag: planning :mag: | +| **NatsJS** | :hammer_and_wrench: **in progress** :hammer_and_wrench: | :mag: planning :mag: | +| **MQTT** | :mag: planning :mag: | :mag: planning :mag: | +| **Kafka** | :mag: planning :mag: | :mag: planning :mag: | +| **Redis Streams** | :mag: planning :mag: | :mag: planning :mag: | +| **Pulsar** | :mag: planning :mag: | :mag: planning :mag: | +| **SQS** | :mag: planning :mag: | :mag: planning :mag: | diff --git a/docs/docs/en/5_integrations/1_integrations-index.md b/docs/docs/en/integrations/1_integrations-index.md similarity index 100% rename from docs/docs/en/5_integrations/1_integrations-index.md rename to docs/docs/en/integrations/1_integrations-index.md diff --git a/docs/docs/en/5_integrations/2_fastapi-plugin.md b/docs/docs/en/integrations/2_fastapi-plugin.md similarity index 61% rename from docs/docs/en/5_integrations/2_fastapi-plugin.md rename to docs/docs/en/integrations/2_fastapi-plugin.md index 038086fd..f1c45c98 100644 --- a/docs/docs/en/5_integrations/2_fastapi-plugin.md +++ b/docs/docs/en/integrations/2_fastapi-plugin.md @@ -11,9 +11,15 @@ using the `@event` decorator. This decorator is similar to the decorator `@handl Note that the code below uses `fastapi.Depends`, not `propan.Depends`. -```python linenums="1" hl_lines="1 3 7 15 19 23" -{!> docs_src/integrations/fastapi_plugin_rabbit.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="1 3 7 15 19 23" + {!> docs_src/integrations/fastapi_plugin_redis.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="1 3 7 15 19 23" + {!> docs_src/integrations/fastapi_plugin_rabbit.py!} + ``` When processing a message from a broker, the entire message body is placed simultaneously in both the `body` and `path` request parameters: you can get access to them in any way convenient for you. The message header is placed in `headers`. @@ -25,12 +31,24 @@ use it to declare any `get`, `post`, `put` and other HTTP methods. For example, Inside each router there is a broker. You can easily access it if you need to send a message to MQ. -```python linenums="1" hl_lines="6 10" -{!> docs_src/integrations/fastapi_plugin_rabbit_send.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="6 10" + {!> docs_src/integrations/fastapi_plugin_redis_send.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="6 10" + {!> docs_src/integrations/fastapi_plugin_rabbit_send.py!} + ``` You can use the following `Depends` to access the broker if you want to use it at different parts of your program. -```python linenums="1" hl_lines="8 14-15" -{!> docs_src/integrations/fastapi_plugin_rabbit_depends.py!} -``` \ No newline at end of file +=== "Redis" + ```python linenums="1" hl_lines="8 14-15" + {!> docs_src/integrations/fastapi_plugin_redis_depends.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="8 14-15" + {!> docs_src/integrations/fastapi_plugin_rabbit_depends.py!} + ``` \ No newline at end of file diff --git a/docs/docs/en/nats/1_nats-index.md b/docs/docs/en/nats/1_nats-index.md new file mode 100644 index 00000000..98f143c8 --- /dev/null +++ b/docs/docs/en/nats/1_nats-index.md @@ -0,0 +1,3 @@ +# NATS + +{! docs/en/helpful/in-progress.md !} diff --git a/docs/docs/en/nats/2_routing.md b/docs/docs/en/nats/2_routing.md new file mode 100644 index 00000000..286ee419 --- /dev/null +++ b/docs/docs/en/nats/2_routing.md @@ -0,0 +1,3 @@ +# NATS Routing + +{! docs/en/helpful/in-progress.md !} diff --git a/docs/docs/en/nats/3_publishing.md b/docs/docs/en/nats/3_publishing.md new file mode 100644 index 00000000..d3245745 --- /dev/null +++ b/docs/docs/en/nats/3_publishing.md @@ -0,0 +1,3 @@ +# NATS Publishing + +{! docs/en/helpful/in-progress.md !} diff --git a/docs/docs/en/3_rabbit/1_routing.md b/docs/docs/en/rabbit/1_routing.md similarity index 100% rename from docs/docs/en/3_rabbit/1_routing.md rename to docs/docs/en/rabbit/1_routing.md diff --git a/docs/docs/en/3_rabbit/2_exchanges.md b/docs/docs/en/rabbit/2_exchanges.md similarity index 100% rename from docs/docs/en/3_rabbit/2_exchanges.md rename to docs/docs/en/rabbit/2_exchanges.md diff --git a/docs/docs/en/3_rabbit/3_queues.md b/docs/docs/en/rabbit/3_queues.md similarity index 100% rename from docs/docs/en/3_rabbit/3_queues.md rename to docs/docs/en/rabbit/3_queues.md diff --git a/docs/docs/en/3_rabbit/4_publishing.md b/docs/docs/en/rabbit/4_publishing.md similarity index 80% rename from docs/docs/en/3_rabbit/4_publishing.md rename to docs/docs/en/rabbit/4_publishing.md index 085e3afe..a4ffd6a1 100644 --- a/docs/docs/en/3_rabbit/4_publishing.md +++ b/docs/docs/en/rabbit/4_publishing.md @@ -4,24 +4,26 @@ However, in this case, an object of the `aio_pika.Message` class (if necessary) can act as a message (in addition to `str`, `bytes`, `dict`, `pydatic.BaseModel`). ```python -from propan.brokers.rabbit import RabbitBroker, RabbitQueue +import asyncio +from propan import RabbitBroker -broker = RabbitBroker() +async def pub(): + async with RabbitBroker() as broker: + await broker.publish("Hi!", queue="test", exhcange="test") -... - await broker.publish("Hi!", queue="test", exhcange="test") +asyncio.run(pub()) ``` -### Basic arguments +## Basic arguments -The `pubslish` method takes the following arguments: +The `publish` method takes the following arguments: -* `message`: bytes | str | dict | pydatic.BaseModel | aio_pika.Message = "" - message to send +* `message`: bytes | str | dict | Sequence[Any] | pydatic.BaseModel | aio_pika.Message = "" - message to send * `exchange`: str | RabbitExchange | None = None - the exchange where the message will be sent to. If not specified - *default* is used * `queue`: str | RabbitQueue = "" - the queue where the message will be sent (since most queues use their name as the routing key, this is a human-readable version of `routing_key`) * `routing_key`: str = "" - also a message routing key, if not specified, the `queue` argument is used -### Message parameters +## Message parameters You can read more about all the flags in the [RabbitMQ documentation](https://www.rabbitmq.com/consumers.html){.external-link target="_blank"} @@ -39,7 +41,7 @@ You can read more about all the flags in the [RabbitMQ documentation](https://ww * `user_id`: str | None - ID of the *RabbitMQ* user who sent the message * `app_id`: str | None - ID of the application that sent the message (used by consumers) -### Send flags +## Send flags Arguments for sending a message: @@ -47,12 +49,12 @@ Arguments for sending a message: * `immediate`: bool = False - the client expects that there is a consumer ready to take the message to work "right now" (if there is no consumer, return it to the sender) * `timeout`: int | float | None = None - send confirmation time from *RabbitMQ* -### RPC arguments +## RPC arguments -Also `publish` supports generic arguments for making [*RPC* requests](../../2_getting_started/4_broker/5_rpc/#client): +Also `publish` supports common arguments for making [*RPC* requests](../../getting_started/4_broker/5_rpc/#client): * `callback`: bool = False - whether to wait for a response to the message -* `callbakc_timeout`: float | None = 30.0 - response waiting timeout. In case of `None` - waits indefinitely +* `callback_timeout`: float | None = 30.0 - response waiting timeout. In case of `None` - waits indefinitely * `raise_timeout`: bool = False * `False` - return None on timeout * `True` - `asyncio.TimeoutError` error in case of timeout \ No newline at end of file diff --git a/docs/docs/en/3_rabbit/5_examples/1_direct.md b/docs/docs/en/rabbit/5_examples/1_direct.md similarity index 87% rename from docs/docs/en/3_rabbit/5_examples/1_direct.md rename to docs/docs/en/rabbit/5_examples/1_direct.md index fdc40f96..b84553c9 100644 --- a/docs/docs/en/3_rabbit/5_examples/1_direct.md +++ b/docs/docs/en/rabbit/5_examples/1_direct.md @@ -5,7 +5,7 @@ !!! note **Default** Exchange, to which all queues in *RabbitMQ* are subscribed, has the **Direct** type by default -## Message distribution +## Scaling If several consumers are listening to the same queue, messages will go to the one of them (round-robin). This behavior is common for all types of `exchange`, because it refers to the queue itself. The type of `exchange` affects which queues the message gets into. @@ -25,7 +25,7 @@ async def handler(): The argument `auto_delete=True` in this and subsequent examples is used only to clear the state of *RabbitMQ* after example runs ```python linenums="1" -{!> docs_src/rabbit/examples/direct.py !} +{!> docs_src/rabbit/direct.py !} ``` ### Consumer Announcement @@ -33,13 +33,13 @@ The argument `auto_delete=True` in this and subsequent examples is used only to To begin with, we announced our **Direct** exchange and several queues that will listen to it: ```python linenums="8" -{!> docs_src/rabbit/examples/direct.py [ln:8-11]!} +{!> docs_src/rabbit/direct.py [ln:8-11]!} ``` Then we signed up several consumers using the advertised queues to the `exchange` we created ```python linenums="13" hl_lines="1 5 9" -{!> docs_src/rabbit/examples/direct.py [ln:13-23]!} +{!> docs_src/rabbit/direct.py [ln:13-23]!} ``` !!! note @@ -52,7 +52,7 @@ Then we signed up several consumers using the advertised queues to the `exchange Now the distribution of messages between these consumers will look like this: ```python -{!> docs_src/rabbit/examples/direct.py [ln:29]!} +{!> docs_src/rabbit/direct.py [ln:27]!} ``` Messages `1` will be sent to `handler1` because it listens to `exchange` using a queue with the routing key `test-q-1` @@ -60,7 +60,7 @@ Messages `1` will be sent to `handler1` because it listens to `exchange` using a --- ```python -{!> docs_src/rabbit/examples/direct.py [ln:30]!} +{!> docs_src/rabbit/direct.py [ln:28]!} ``` Messages `2` will be sent to `handler2` because it listens to `exchange` using the same queue, but `handler1` is busy @@ -68,7 +68,7 @@ Messages `2` will be sent to `handler2` because it listens to `exchange` using t --- ```python -{!> docs_src/rabbit/examples/direct.py [ln:31]!} +{!> docs_src/rabbit/direct.py [ln:29]!} ``` Messages `3` will be sent to `handler1` again, because it is currently free @@ -76,7 +76,7 @@ Messages `3` will be sent to `handler1` again, because it is currently free --- ```python -{!> docs_src/rabbit/examples/direct.py [ln:32]!} +{!> docs_src/rabbit/direct.py [ln:30]!} ``` Messages `4` will be sent to `handler3`, because it is the only one listening to `exchange` using a queue with the routing key `test-q-2` \ No newline at end of file diff --git a/docs/docs/en/3_rabbit/5_examples/2_fanout.md b/docs/docs/en/rabbit/5_examples/2_fanout.md similarity index 82% rename from docs/docs/en/3_rabbit/5_examples/2_fanout.md rename to docs/docs/en/rabbit/5_examples/2_fanout.md index 710231c1..1f6c7a73 100644 --- a/docs/docs/en/3_rabbit/5_examples/2_fanout.md +++ b/docs/docs/en/rabbit/5_examples/2_fanout.md @@ -8,7 +8,7 @@ At the same time, if the queue listens to several consumers, messages will also ## Example ```python linenums="1" -{!> docs_src/rabbit/examples/fanout.py !} +{!> docs_src/rabbit/fanout.py !} ``` ### Consumer Announcement @@ -16,13 +16,13 @@ At the same time, if the queue listens to several consumers, messages will also To begin with, we announced our **Fanout** exchange and several queues that will listen to it: ```python linenums="8" hl_lines="1" -{!> docs_src/rabbit/examples/fanout.py [ln:8-11]!} +{!> docs_src/rabbit/fanout.py [ln:8-11]!} ``` Then we signed up several consumers using the advertised queues to the `exchange` we created ```python linenums="13" hl_lines="1 5 9" -{!> docs_src/rabbit/examples/fanout.py [ln:13-23]!} +{!> docs_src/rabbit/fanout.py [ln:13-23]!} ``` !!! note @@ -35,7 +35,7 @@ Then we signed up several consumers using the advertised queues to the `exchange Now the distribution of messages between these consumers will look like this: ```python -{!> docs_src/rabbit/examples/fanout.py [ln:29]!} +{!> docs_src/rabbit/fanout.py [ln:27]!} ``` Messages `1` will be sent to `handler1` and `handler3`, because they listen to `exchange` using different queues @@ -43,7 +43,7 @@ Messages `1` will be sent to `handler1` and `handler3`, because they listen to ` --- ```python -{!> docs_src/rabbit/examples/fanout.py [ln:30]!} +{!> docs_src/rabbit/fanout.py [ln:28]!} ``` Messages `2` will be sent to `handler2` and `handler3`, because `handler2` listens to `exchange` using the same queue as `handler1` @@ -51,7 +51,7 @@ Messages `2` will be sent to `handler2` and `handler3`, because `handler2` liste --- ```python -{!> docs_src/rabbit/examples/fanout.py [ln:31]!} +{!> docs_src/rabbit/fanout.py [ln:29]!} ``` Messages `3` will be sent to `handler1` and `handler3` @@ -59,7 +59,7 @@ Messages `3` will be sent to `handler1` and `handler3` --- ```python -{!> docs_src/rabbit/examples/fanout.py [ln:32]!} +{!> docs_src/rabbit/fanout.py [ln:30]!} ``` Messages `4` will be sent to `handler3` and `handler3` diff --git a/docs/docs/en/3_rabbit/5_examples/3_topic.md b/docs/docs/en/rabbit/5_examples/3_topic.md similarity index 83% rename from docs/docs/en/3_rabbit/5_examples/3_topic.md rename to docs/docs/en/rabbit/5_examples/3_topic.md index b054dfbd..94c4589b 100644 --- a/docs/docs/en/3_rabbit/5_examples/3_topic.md +++ b/docs/docs/en/rabbit/5_examples/3_topic.md @@ -7,7 +7,7 @@ At the same time, if the queue listens to several consumers, messages will also ## Example ```python linenums="1" -{!> docs_src/rabbit/examples/topic.py !} +{!> docs_src/rabbit/topic.py !} ``` ### Consumer Announcement @@ -15,7 +15,7 @@ At the same time, if the queue listens to several consumers, messages will also To begin with, we announced our **Topic** exchange and several queues that will listen to it: ```python linenums="8" hl_lines="1 3-4" -{!> docs_src/rabbit/examples/topic.py [ln:8-11]!} +{!> docs_src/rabbit/topic.py [ln:8-11]!} ``` At the same time, in the `routing_key` of our queues, we specify the *pattern* of routing keys that will be processed by this queue. @@ -23,7 +23,7 @@ At the same time, in the `routing_key` of our queues, we specify the *pattern* o Then we signed up several consumers using the advertised queues to the `exchange` we created ```python linenums="13" hl_lines="1 5 9" -{!> docs_src/rabbit/examples/topic.py [ln:13-23]!} +{!> docs_src/rabbit/topic.py [ln:13-23]!} ``` !!! note @@ -36,7 +36,7 @@ Then we signed up several consumers using the advertised queues to the `exchange Now the distribution of messages between these consumers will look like this: ```python -{!> docs_src/rabbit/examples/topic.py [ln:29]!} +{!> docs_src/rabbit/topic.py [ln:27]!} ``` Messages `1` will be sent to `handler1` because it listens to `exchange` using a queue with the routing key `*.info` @@ -44,7 +44,7 @@ Messages `1` will be sent to `handler1` because it listens to `exchange` using a --- ```python -{!> docs_src/rabbit/examples/topic.py [ln:30]!} +{!> docs_src/rabbit/topic.py [ln:28]!} ``` Messages `2` will be sent to `handler2` because it listens to `exchange` using the same queue, but `handler1` is busy @@ -52,7 +52,7 @@ Messages `2` will be sent to `handler2` because it listens to `exchange` using t --- ```python -{!> docs_src/rabbit/examples/topic.py [ln:31]!} +{!> docs_src/rabbit/topic.py [ln:29]!} ``` Messages `3` will be sent to `handler1` again, because it is currently free @@ -60,7 +60,7 @@ Messages `3` will be sent to `handler1` again, because it is currently free --- ```python -{!> docs_src/rabbit/examples/topic.py [ln:32]!} +{!> docs_src/rabbit/topic.py [ln:30]!} ``` Messages `4` will be sent to `handler3`, because it is the only one listening to `exchange` using a queue with the routing key `*.debug` \ No newline at end of file diff --git a/docs/docs/en/3_rabbit/5_examples/4_header.md b/docs/docs/en/rabbit/5_examples/4_header.md similarity index 85% rename from docs/docs/en/3_rabbit/5_examples/4_header.md rename to docs/docs/en/rabbit/5_examples/4_header.md index 95605951..5ca88fd8 100644 --- a/docs/docs/en/3_rabbit/5_examples/4_header.md +++ b/docs/docs/en/rabbit/5_examples/4_header.md @@ -8,7 +8,7 @@ At the same time, if the queue listens to several consumers, messages will also ## Example ```python linenums="1" -{!> docs_src/rabbit/examples/header.py !} +{!> docs_src/rabbit/header.py !} ``` ### Consumer Announcement @@ -16,7 +16,7 @@ At the same time, if the queue listens to several consumers, messages will also To begin with, we announced our **Fanout** exchange and several queues that will listen to it: ```python linenums="8" hl_lines="1 5 9 13" -{!> docs_src/rabbit/examples/header.py [ln:8-21]!} +{!> docs_src/rabbit/header.py [ln:8-21]!} ``` The `x-match` argument indicates whether the arguments should match the message headers in whole or in part. @@ -24,7 +24,7 @@ The `x-match` argument indicates whether the arguments should match the message Then we signed up several consumers using the advertised queues to the `exchange` we created ```python linenums="23" hl_lines="1 5 9 13" -{!> docs_src/rabbit/examples/header.py [ln:23-37]!} +{!> docs_src/rabbit/header.py [ln:23-37]!} ``` !!! note @@ -37,7 +37,7 @@ Then we signed up several consumers using the advertised queues to the `exchange Now the distribution of messages between these consumers will look like this: ```python -{!> docs_src/rabbit/examples/header.py [ln:43]!} +{!> docs_src/rabbit/header.py [ln:41]!} ``` Messages `1` will be sent to `handler1`, because it listens to a queue whose `key` header matches the `key` header of the message @@ -45,7 +45,7 @@ Messages `1` will be sent to `handler1`, because it listens to a queue whose `ke --- ```python -{!> docs_src/rabbit/examples/header.py [ln:44]!} +{!> docs_src/rabbit/header.py [ln:42]!} ``` Messages `2` will be sent to `handler2` because it listens to `exchange` using the same queue, but `handler1` is busy @@ -53,7 +53,7 @@ Messages `2` will be sent to `handler2` because it listens to `exchange` using t --- ```python -{!> docs_src/rabbit/examples/header.py [ln:45]!} +{!> docs_src/rabbit/header.py [ln:43]!} ``` Messages `3` will be sent to `handler1` again, because it is currently free @@ -61,7 +61,7 @@ Messages `3` will be sent to `handler1` again, because it is currently free --- ```python -{!> docs_src/rabbit/examples/header.py [ln:46]!} +{!> docs_src/rabbit/header.py [ln:44]!} ``` Messages `4` will be sent to `handler3`, because it listens to a queue whose `key` header coincided with the `key` header of the message @@ -69,7 +69,7 @@ Messages `4` will be sent to `handler3`, because it listens to a queue whose `ke --- ```python -{!> docs_src/rabbit/examples/header.py [ln:47]!} +{!> docs_src/rabbit/header.py [ln:45]!} ``` Messages `5` will be sent to `handler3`, because it listens to a queue whose header `key2` coincided with the header `key2` of the message @@ -77,7 +77,7 @@ Messages `5` will be sent to `handler3`, because it listens to a queue whose hea --- ```python -{!> docs_src/rabbit/examples/header.py [ln:48-49]!} +{!> docs_src/rabbit/header.py [ln:46-47]!} ``` Messages `6` will be sent to `handler3` and `handler4`, because the message headers completely match the queue keys diff --git a/docs/docs/en/redis/1_redis-index.md b/docs/docs/en/redis/1_redis-index.md new file mode 100644 index 00000000..a6324f50 --- /dev/null +++ b/docs/docs/en/redis/1_redis-index.md @@ -0,0 +1,50 @@ +# Redis Pub/Sub + +## Advantages and disadvantages + +Most likely, your project already uses *Redis*. If you want to use asynchronous messaging, but do not want +to include a new heavy dependency (*Kafka*, *RabbitMQ*, *Nats*, etc.) to your infrastructure, you should use it as a message broker. + +*Redis* works fast, does not degrade with a large number of messages, and most importantly - you already have it! + +!!! note +More information about the documentation of *Redis Pub/Sub* can be found at [official site](https://redis.io/docs/manual/pubsub/#messages-matching-both-a-pattern-and-a-channel-subscription){.external- link target="_blank"} + +However, *Redis* as a message broker has some important disadvantages: + +* Messages are not persistent. If the message is published while your consumer is disconnected, it will be lost. +* There is no possibility of horizontal scaling of consumers. +* There are no complex routing mechanisms. +* There are no mechanisms for confirming receipt and processing of messages from the consumer. +* Messages are represented by raw bytes without meta-information. + +Not all of these features are necessary for your project, but you should keep them in mind when choosing *Redis* as a message broker. + +In any case, since the **Propan** application code is weakly dependent on the message broker used, you can build a prototype of your system based on *Redis* and if necessary quickly adapt it to use another broker. + +Also, *Redis 5.0+* contains the *Streams* mechanism, which can also serve as a message broker and covers the main disadvantages of *Redis Pub/Sub*: message persistence and consumer scaling. + +## Routing rules + +*Redis* cannot configure complex routing rules. The only entity in *Redis Pub/Sub* is `channel`, which can be subscribed to either directly by name or by regular expression pattern. + +Both examples are described [a little further](../3_examples/1_direct). + +## Features **Propan** + +Since *Redis* uses just a set of bytes without headers and other meta information as a message, **Propan** uses encoded *json* with the following structure as a message: + +```json +{ +"data": "", +"headers": {}, +"reply_to": "" +} +``` + +This is necessary for the correct recognition of the *content-type* of the incoming message (necessary for valid decoding) and support for **RPC** requests. + +If **Propan** receives a message sent using another library or framework (or just a message in a different format), +the entire body of this message will be perceived as the `data` field of the received message, and the `content-type` will be recognized automatically. + +At the same time, **RPC** requests will not work, since there is no `reply_to` field in the incoming message. \ No newline at end of file diff --git a/docs/docs/en/redis/2_publishing.md b/docs/docs/en/redis/2_publishing.md new file mode 100644 index 00000000..094efb58 --- /dev/null +++ b/docs/docs/en/redis/2_publishing.md @@ -0,0 +1,55 @@ +# Redis Publishing + +To send messages, `RedisBroker` also uses the unified `publish` method. + +```python +import asyncio +from propan import RedisBroker + +async def pub(): + async with RedisBroker() as broker: + await broker.publish("Hi!", channel="test") + +asyncio.run(pub()) +``` + +## Basic arguments + +The `publish` method accepts the following arguments: + +* `message`: bytes | str | dict | Sequence[Any] | pydatic.BaseModel - message to send +* `channel`: str = "" - *channel* to which the message will be sent. + +## Message Parameters + +*Redis* by default sends a message in the form of raw `bytes'. So **Propan** uses its own message transmission format: +when calling the `publish` method, *json* is sent to *Redis* with the following fields: + +```json +{ + "data": "", + "headers": {}, + "reply_to": "" +} +``` + +Independently, you can set and use the headers of the sent message within your application (the `content-type` is automatically set there, according to which **Propan** determines how to decode the received message) + +* `headers`: dict[str, Any] | None = None - headers of the message being sent (used by consumers) + +!!! note "" + If **Propan** receives a message sent using another library or framework (or just a message in a different format), + the entire body of this message will be perceived as the `data` field of the received message, and the `content-type` will be recognized automatically. + + At the same time, **RPC** requests will not work, since there is no `reply_to` field in the incoming message. + +## RPC arguments + +Also `publish` supports common arguments for making [*RPC* requests](../../getting_started/4_broker/5_rpc/#client): + +* `reply_to`: str = "" - which *channel* to send the response to (used for asynchronous RPC requests) +* `callback`: bool = False - whether to expect a response to the message +* `callback_timeout`: float | None = 30.0 - timeout waiting for a response. In the case of `None` - waits indefinitely +* `raise_timeout`: bool = False + * `False` - return None in case of timeout + * `True` - error `asyncio.TimeoutError` in case of timeout diff --git a/docs/docs/en/redis/3_examples/1_direct.md b/docs/docs/en/redis/3_examples/1_direct.md new file mode 100644 index 00000000..771af738 --- /dev/null +++ b/docs/docs/en/redis/3_examples/1_direct.md @@ -0,0 +1,56 @@ +# Direct + +**Direct** Channel is the basic way to route messages in *Redis*. Its core is very simple: `channel` sends messages to all consumers subscribed to it. + +## Scaling + +If one channel is listened by several consumers, the message will be received by **all** consumers of this channel. +Thus, horizontal scaling by increasing the number of consumer services is not possible only using *Redis Pub/Sub*. + +If you need similar functionality, look for *Redis Streams* or other brokers (for example, *Nats* or *RabbitMQ*). + +## Example + +**Direct** Channel is the default type used in **Propan**: you can simply declare it as follows + +```python +@broker.handler("test_channel") +async def handler(): + ... +``` + +Full example: + +```python linenums="1" +{!> docs_src/redis/direct.py !} +``` + +### Consumer Announcement + +To begin with, we have announced several consumers for the two channels `test` and `test2`: + +```python linenums="8" hl_lines="1 6 11" +{!> docs_src/redis/direct.py [ln:8-20]!} +``` + +!!! note + Note that `handler1` and `handler2` are subscribed to the same `channel`: + both of these handlers will receive messages. + +### Message distribution + +Now the distribution of messages between these consumers will look like this: + +```python +{!> docs_src/redis/direct.py [ln:25]!} +``` + +The message `1` will be sent to `handler1` and `handler2` because they are listening to `channel` with the name `test` + +--- + +```python +{!> docs_src/redis/direct.py [ln:26]!} +``` + +The message `2` will be sent to `handler3` because it listens to `channel` with the name `test2` \ No newline at end of file diff --git a/docs/docs/en/redis/3_examples/2_pattern.md b/docs/docs/en/redis/3_examples/2_pattern.md new file mode 100644 index 00000000..85629ce3 --- /dev/null +++ b/docs/docs/en/redis/3_examples/2_pattern.md @@ -0,0 +1,47 @@ +# Pattern + +**Pattern** Channel is a powerful *Redis* routing engine. This type of `channel` sends messages to consumers by the *pattern* +specified when they connect to the `channel` and the message key. + +## Scaling + +If the message key matches the pattern of several consumers, it will be sent to **all** them. +Thus, horizontal scaling by increasing the number of consumer services is not possible only using *Redis Pub/Sub*. + +If you need similar functionality, look towards *Redis Streams* or other brokers (for example, *Nats* or *RabbitMQ*). + +## Example + +```python linenums="1" +{!> docs_src/redis/pattern.py !} +``` + +### Consumer Announcement + +To begin with, we have announced several consumers for two channels `*.info*` and `*.error`: + +```python linenums="8" hl_lines="1 6 11" +{!> docs_src/redis/pattern.py [ln:8-20]!} +``` + +!!! note + Note that `handler1` and `handler2` are subscribed to the same `channel`: + both of these handlers will receive messages. + +### Message distribution + +Now the distribution of messages between these consumers will look like this: + +```python +{!> docs_src/redis/pattern.py [ln:25]!} +``` + +The message `1` will be sent to `handler1` and `handler2` because they are listening to `channel` with the pattern `*.info*` + +--- + +```python +{!> docs_src/redis/pattern.py [ln:26]!} +``` + +The message `2` will be sent to `handler3` because it listens to `channel` with the pattern `*.error` diff --git a/docs/docs/ru/9_CHANGELOG.md b/docs/docs/ru/CHANGELOG.md similarity index 77% rename from docs/docs/ru/9_CHANGELOG.md rename to docs/docs/ru/CHANGELOG.md index 816a221c..44aed75b 100644 --- a/docs/docs/ru/9_CHANGELOG.md +++ b/docs/docs/ru/CHANGELOG.md @@ -1,5 +1,22 @@ # CHANGELOG +## 2023-05-18 **0.1.1.0** REDIS + +В **Propan** добавлена поддержка *Redis Pub/Sub* в качестве брокера сообщений. Данный функционал полностью протестирован и описан в документации. + +*RedisBroker* поддерживает: + +* доставку сообщений по ключу или паттерну +* тестовый клиент, без необходимости запуска *Redis* +* **RPC** запросы поверх *Redis Pub/Sub* +* В качестве плагина *FastAPI* + +Также, **Propan CLI** теперь позволяет генерировать шаблоны проектов для использования с различными брокерами сообщений + +```bash +propan create async [broker] [APPNAME] +``` + ## 2023-05-15 **0.1.0.0** STABLE Стабильный и полностью задокументированный релиз **Propan**! @@ -28,9 +45,9 @@ async def hello(m: dict) -> dict: app.include_router(router) ``` -Полный пример вы можете найти в [документации](../5_integrations/2_fastapi-plugin) +Полный пример вы можете найти в [документации](../integrations/2_fastapi-plugin) -Также, добавлена возможность [тестировать](../2_getting_started/7_testing) свое приложение без запуска внешних зависимостей в виде брокера (пока только для RabbitMQ)! +Также, добавлена возможность [тестировать](../getting_started/7_testing) свое приложение без запуска внешних зависимостей в виде брокера (пока только для RabbitMQ)! ```python from propan import RabbitBroker @@ -49,7 +66,7 @@ def test_publish(): assert r == "pong" ``` -Также добавлена поддержка [RPC over MQ](../2_getting_started/4_broker/5_rpc) (пока только для RabbitMQ): `return` вашей функции-обработчика будет отправлен в ответ на сообщение, если ответ ожидается. +Также добавлена поддержка [RPC over MQ](../getting_started/4_broker/5_rpc) (пока только для RabbitMQ): `return` вашей функции-обработчика будет отправлен в ответ на сообщение, если ответ ожидается.

Breaking changes:

diff --git a/docs/docs/ru/7_alternatives.md b/docs/docs/ru/alternatives.md similarity index 100% rename from docs/docs/ru/7_alternatives.md rename to docs/docs/ru/alternatives.md diff --git a/docs/docs/ru/6_contributing/1_todo.md b/docs/docs/ru/contributing/1_todo.md similarity index 97% rename from docs/docs/ru/6_contributing/1_todo.md rename to docs/docs/ru/contributing/1_todo.md index 61b946cb..60a8fd24 100644 --- a/docs/docs/ru/6_contributing/1_todo.md +++ b/docs/docs/ru/contributing/1_todo.md @@ -29,4 +29,4 @@ Для того, чтобы приступить к разработке проекта, перейдите в следующий [раздел](../2_contributing-index/). -Если же вы хотите разработать собственный адаптер для какого-либо брокера, вы найдете много полезной информации в [этом](../4_adapters/) разделе. \ No newline at end of file +Если же вы хотите разработать собственный адаптер для какого-либо брокера, вы найдете много полезной информации в [этом](../4_adapters/) разделе. diff --git a/docs/docs/ru/6_contributing/2_contributing-index.md b/docs/docs/ru/contributing/2_contributing-index.md similarity index 100% rename from docs/docs/ru/6_contributing/2_contributing-index.md rename to docs/docs/ru/contributing/2_contributing-index.md diff --git a/docs/docs/ru/6_contributing/3_docs.md b/docs/docs/ru/contributing/3_docs.md similarity index 100% rename from docs/docs/ru/6_contributing/3_docs.md rename to docs/docs/ru/contributing/3_docs.md diff --git a/docs/docs/ru/6_contributing/4_adapters.md b/docs/docs/ru/contributing/4_adapters.md similarity index 100% rename from docs/docs/ru/6_contributing/4_adapters.md rename to docs/docs/ru/contributing/4_adapters.md diff --git a/docs/docs/ru/2_getting_started/1_quick-start.md b/docs/docs/ru/getting_started/1_quick-start.md similarity index 84% rename from docs/docs/ru/2_getting_started/1_quick-start.md rename to docs/docs/ru/getting_started/1_quick-start.md index 5cf8fca6..ed6c5ed7 100644 --- a/docs/docs/ru/2_getting_started/1_quick-start.md +++ b/docs/docs/ru/getting_started/1_quick-start.md @@ -2,6 +2,19 @@ Для начала, установите фрейморк через `pip`: +=== "Redis" +
+ ```console + $ pip install "propan[async-redis]" + ---> 100% + ``` +
+ !!! tip + Для работы проекта запустите тестовый контейнер с брокером + ```bash + docker run -d --rm -p 6379:6379 --name test-mq redis + ``` + === "RabbitMQ"
```console @@ -32,6 +45,11 @@ Создайте приложение со следующим кодом в `serve.py` файле: +=== "Redis" + ```python linenums="1" + {!> docs_src/index/01_redis_base.py!} + ``` + === "RabbitMQ" ```python linenums="1" {!> docs_src/index/01_rabbit_base.py!} @@ -60,6 +78,11 @@ $ propan run serve:app Propan использует `pydantic` для приведения типов входящих аргументов в соответсвии с их аннотацией. +=== "Redis" + ```python linenums="1" hl_lines="12" + {!> docs_src/index/02_redis_type_casting.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="12" {!> docs_src/index/02_rabbit_type_casting.py!} @@ -86,6 +109,11 @@ Propan имеет систему управления зависимостями Подробнее будет [чуть дальше](../5_dependency/1_di-index). +=== "Redis" + ```python linenums="1" hl_lines="11-12" + {!> docs_src/index/03_redis_dependencies.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="11-12" {!> docs_src/index/03_rabbit_dependencies.py!} @@ -104,7 +132,7 @@ Propan имеет систему управления зависимостями
```console -$ propan create [projectname] +$ propan create async rabbit [projectname] Create Propan project template at: /home/user/projectname ```
@@ -140,6 +168,11 @@ $ propan run [projectname].app.serve:app --env=.env --reload Вы можете использовать брокеры Propan без самого Propan приложения. Просто *запустите* и *остановите* его вместе с вашим HTTP приложением. +=== "Redis" + ```python linenums="1" hl_lines="5 11-13 16-17" + {!> docs_src/index/05_redis_http_example.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="5 11-13 16-17" {!> docs_src/index/05_rabbit_http_example.py!} @@ -161,12 +194,18 @@ $ propan run [projectname].app.serve:app --env=.env --reload При использовании таким образом **Propan** не использует собственную систему зависимостей, а интегрируется в **FastAPI**. Т.е. вы можете использовать `Depends`, `BackgroundTasks` и прочие инструменты **FastAPI** так, если бы это был обычный HTTP-endpoint. -```python linenums="1" hl_lines="7 15 19" -{!> docs_src/index/06_rabbit_native_fastapi.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="7 15 19" + {!> docs_src/index/06_redis_native_fastapi.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="7 15 19" + {!> docs_src/index/06_rabbit_native_fastapi.py!} + ``` !!! note - Больше примеров использования с другими фреймворками вы найдете [здесь](../../5_integrations/1_integrations-index/) + Больше примеров использования с другими фреймворками вы найдете [здесь](../../integrations/1_integrations-index/) ??? tip "Не забудьте остановить тестовый контейнер" ```bash diff --git a/docs/docs/ru/2_getting_started/2_cli.md b/docs/docs/ru/getting_started/2_cli.md similarity index 96% rename from docs/docs/ru/2_getting_started/2_cli.md rename to docs/docs/ru/getting_started/2_cli.md index 4c90a56e..1ef23d38 100644 --- a/docs/docs/ru/2_getting_started/2_cli.md +++ b/docs/docs/ru/getting_started/2_cli.md @@ -33,7 +33,7 @@ Commands:
```console -$ propan create app +$ propan create async rabbit app Create Propan project template at: ./app ``` @@ -79,6 +79,11 @@ $ propan run serve:app --env=.env.dev ```
+=== "Redis" + ```python linenums="1" hl_lines="3 14-15" + {!> docs_src/index/04_redis_context.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="3 14-15" {!> docs_src/index/04_rabbit_context.py!} diff --git a/docs/docs/ru/2_getting_started/3_app.md b/docs/docs/ru/getting_started/3_app.md similarity index 91% rename from docs/docs/ru/2_getting_started/3_app.md rename to docs/docs/ru/getting_started/3_app.md index d3dbf569..7f2f1ec3 100644 --- a/docs/docs/ru/2_getting_started/3_app.md +++ b/docs/docs/ru/getting_started/3_app.md @@ -16,6 +16,11 @@ app = PropanApp() Обычно это делается при объявлении самого приложения +=== "Redis" + ```python + {!> docs_src/quickstart/app/1_broker_redis.py!} + ``` + === "RabbitMQ" ```python {!> docs_src/quickstart/app/1_broker_rabbit.py!} @@ -28,6 +33,11 @@ app = PropanApp() Но, иногда вам может понадобиться инициализировать брокера в другом месте. В таком случае, вы можете использовать метод `app.set_broker` +=== "Redis" + ```python + {!> docs_src/quickstart/app/2_set_broker_redis.py!} + ``` + === "RabbitMQ" ```python {!> docs_src/quickstart/app/2_set_broker_rabbit.py!} diff --git a/docs/docs/ru/2_getting_started/4_broker/1_index.md b/docs/docs/ru/getting_started/4_broker/1_index.md similarity index 80% rename from docs/docs/ru/2_getting_started/4_broker/1_index.md rename to docs/docs/ru/getting_started/4_broker/1_index.md index 3d3d5a3c..1761733b 100644 --- a/docs/docs/ru/2_getting_started/4_broker/1_index.md +++ b/docs/docs/ru/getting_started/4_broker/1_index.md @@ -4,6 +4,11 @@ **Propan** поддерживает различные брокеры сообщений используя для этого специальные классы +=== "Redis" + ```python + from propan import RedisBroker + ``` + === "RabbitMQ" ```python from propan import RabbitBroker @@ -18,6 +23,11 @@ Для установки **Propan** с необходимыми для вашего брокера зависимостями, выберите один из вариантов установки +=== "Redis" + ```bash + pip install "propan[async-redis]" + ``` + === "RabbitMQ" ```bash pip install "propan[async-rabbit]" @@ -32,18 +42,35 @@ Данные для подключения **Propan Broker** к вашему брокеру сообщений могут быть переданы 2мя способами: +=== "Redis" + 1. В конструкторе брокера + + ```python + from propan import RedisBroker + broker = RedisBroker("redis://localhost:6379/") + ``` + + 2. В методе `connect` + ```python + + from propan import RedisBroker + broker = RedisBroker() + ... + await broker.connect("redis://localhost:6379/") + ``` + === "RabbitMQ" 1. В конструкторе брокера ```python - from propan.brokers.rabbit import RabbitBroker + from propan import RabbitBroker broker = RabbitBroker("amqp://guest:guest@localhost:5672/") ``` 2. В методе `connect` - + ```python - from propan.brokers.rabbit import RabbitBroker + from propan import RabbitBroker broker = RabbitBroker() ... await broker.connect("amqp://guest:guest@localhost:5672/") @@ -53,14 +80,14 @@ 1. В конструкторе брокера ```python - from propan.brokers.nats import NatsBroker + from propan import NatsBroker broker = NatsBroker("nats://localhost:4222") ``` 2. В методе `connect` - + ```python - from propan.brokers.nats import NatsBroker + from propan import NatsBroker broker = NatsBroker() ... await broker.connect("nats://localhost:4222") @@ -74,4 +101,4 @@ Параметры, переданные в `connect` имеют приоритет над параметрами, переданными в конструктор. Будьте с этим аккуратны. Кроме этого, повторный вызов `connect` не приведет ни к какому эффекту. Поэтому вы можете не опосаться, что вызов `broker.start()` - (используется внутри `PropanApp` для запуска брокера) вызовет какие-либо ошибки. \ No newline at end of file + (используется внутри `PropanApp` для запуска брокера) вызовет какие-либо ошибки. diff --git a/docs/docs/ru/2_getting_started/4_broker/2_routing.md b/docs/docs/ru/getting_started/4_broker/2_routing.md similarity index 96% rename from docs/docs/ru/2_getting_started/4_broker/2_routing.md rename to docs/docs/ru/getting_started/4_broker/2_routing.md index 51f84999..8de061ae 100644 --- a/docs/docs/ru/2_getting_started/4_broker/2_routing.md +++ b/docs/docs/ru/getting_started/4_broker/2_routing.md @@ -14,8 +14,8 @@ async def base_handler(body: str): Чтобы узнать подробнее о поведении специализированных брокеров перейдите в следующие разделы: -* [RabbitBroker](../../../3_rabbit/1_routing) -* [NatsBroker](../../../4_nats/2_routing) +* [RabbitBroker](../../../rabbit/1_routing) +* [NatsBroker](../../../nats/2_routing) ## Обработка ошибок diff --git a/docs/docs/ru/2_getting_started/4_broker/3_type-casting.md b/docs/docs/ru/getting_started/4_broker/3_type-casting.md similarity index 100% rename from docs/docs/ru/2_getting_started/4_broker/3_type-casting.md rename to docs/docs/ru/getting_started/4_broker/3_type-casting.md diff --git a/docs/docs/ru/2_getting_started/4_broker/4_publishing.md b/docs/docs/ru/getting_started/4_broker/4_publishing.md similarity index 88% rename from docs/docs/ru/2_getting_started/4_broker/4_publishing.md rename to docs/docs/ru/getting_started/4_broker/4_publishing.md index a21f1a02..2c278a38 100644 --- a/docs/docs/ru/2_getting_started/4_broker/4_publishing.md +++ b/docs/docs/ru/getting_started/4_broker/4_publishing.md @@ -11,14 +11,16 @@ await broker.pubslih(message, ...) Ознакомиться со всеми особенностями, специфичными для вашего брокеры, вы можете здесь: -* [RabbitBroker](../../../3_rabbit/4_publishing) -* [NatsBroker](../../../4_nats/3_publishing) +* [Redis](../../../redis/2_publishing) +* [RabbitBroker](../../../rabbit/4_publishing) +* [NatsBroker](../../../nats/3_publishing) ## Допустимые типы для отправки | Тип | Заголовок при отправке | Способ приведения к bytes | | -------------------- | --------------------------- | ---------------------------- | | `dict` | application/json | json.dumps(message).encode() | +| `Sequence` | application/json | json.dumps(message).encode() | | `pydantic.BaseModel` | application/json | message.json().encode() | | `str` | text/plain | message.encode() | | `bytes` | | message | @@ -32,6 +34,11 @@ await broker.pubslih(message, ...) Если вы находитесь внутри запущенного **Propan** приложения, вам не нужно ничего делать: брокер уже запущен. Просто получите к нему доступ и отправьте сообщение. +=== "Redis" + ```python linenums="1" hl_lines="8" + {!> docs_src/quickstart/broker/publishing/1_redis_inside_propan.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="8" {!> docs_src/quickstart/broker/publishing/1_rabbit_inside_propan.py !} @@ -45,6 +52,11 @@ await broker.pubslih(message, ...) Если же вы используете **Propan** только для отправки асинхронных сообщений в рамках другого фреймворка, вы можете использовать брокер в качестве контекстного менеджера для отправки. +=== "Redis" + ```python + {!> docs_src/quickstart/broker/publishing/2_redis_context.py !} + ``` + === "RabbitMQ" ```python {!> docs_src/quickstart/broker/publishing/2_rabbit_context.py !} diff --git a/docs/docs/ru/2_getting_started/4_broker/5_rpc.md b/docs/docs/ru/getting_started/4_broker/5_rpc.md similarity index 83% rename from docs/docs/ru/2_getting_started/4_broker/5_rpc.md rename to docs/docs/ru/getting_started/4_broker/5_rpc.md index ba760a8d..b1ea88aa 100644 --- a/docs/docs/ru/2_getting_started/4_broker/5_rpc.md +++ b/docs/docs/ru/getting_started/4_broker/5_rpc.md @@ -22,7 +22,12 @@ !!! note Результат вашей функции должен соответствовать допустимым типам параметра `message` функции `broker.publish`. - Допустимые типы: `str`, `dict`, `pydantic.BaseModel`, `bytes`, а также нативные сообщения используемой для брокера библиотеки. + Допустимые типы: `str`, `dict`, `Sequence`, `pydantic.BaseModel`, `bytes`, а также нативные сообщения используемой для брокера библиотеки. + +=== "Redis" + ```python linenums="1" hl_lines="7" + {!> docs_src/quickstart/broker/rpc/1_redis_handler.py !} + ``` === "RabbitMQ" ```python linenums="1" hl_lines="7" @@ -36,6 +41,11 @@ Для того, чтобы дождаться результата выполнения запроса "прямо здесь" (как если бы это был HTTP запрос) необходимо просто указать параметер `callback=True` при отправке сообщения. +=== "Redis" + ```python linenums="1" hl_lines="8" + {!> docs_src/quickstart/broker/rpc/2_redis_blocking_client.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="8" {!> docs_src/quickstart/broker/rpc/2_rabbit_blocking_client.py !} @@ -43,6 +53,13 @@ Для установки времени, которое клиент готов ожидать ответ от сервера, используйте параметр `callback_timeout` (по умолчанию - **30** секунд) +=== "Redis" + ```python linenums="1" hl_lines="5" + {!> docs_src/quickstart/broker/rpc/3_redis_blocking_client_timeout.py !} + ``` + + 1. Ожидает результат выполнения 3 секунды + === "RabbitMQ" ```python linenums="1" hl_lines="5" {!> docs_src/quickstart/broker/rpc/3_rabbit_blocking_client_timeout.py !} @@ -52,6 +69,11 @@ Если вы готовы ждать ответ столько, сколько это понадобится вы можете выставить `callback_timeout=None` +=== "Redis" + ```python linenums="1" hl_lines="5" + {!> docs_src/quickstart/broker/rpc/4_redis_blocking_client_timeout_none.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="5" {!> docs_src/quickstart/broker/rpc/4_rabbit_blocking_client_timeout_none.py !} @@ -63,6 +85,11 @@ По умолчанию, если **Propan** не дождался ответа сервера, функция вернет `None`. Если же вы хотите явным образом обработать `asyncio.TimeoutError`, используйте параметр `raise_timeout`. +=== "Redis" + ```python linenums="1" hl_lines="5" + {!> docs_src/quickstart/broker/rpc/5_redis_blocking_client_timeout_error.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="5" {!> docs_src/quickstart/broker/rpc/5_rabbit_blocking_client_timeout_error.py !} @@ -72,6 +99,11 @@ Для того, чтобы обработать ответ вне основного потока выполнения, вы можете просто инициализировать обработчик, а затем передать его адрес в качестве `reply_to` аргумента запроса. +=== "Redis" + ```python linenums="1" hl_lines="6 16" + {!> docs_src/quickstart/broker/rpc/6_noblocking_client_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="6 16" {!> docs_src/quickstart/broker/rpc/6_noblocking_client_rabbit.py !} diff --git a/docs/docs/ru/2_getting_started/5_dependency/1_di-index.md b/docs/docs/ru/getting_started/5_dependency/1_di-index.md similarity index 90% rename from docs/docs/ru/2_getting_started/5_dependency/1_di-index.md rename to docs/docs/ru/getting_started/5_dependency/1_di-index.md index 3f122dc1..718667ea 100644 --- a/docs/docs/ru/2_getting_started/5_dependency/1_di-index.md +++ b/docs/docs/ru/getting_started/5_dependency/1_di-index.md @@ -1,4 +1,4 @@ -# Зависимости +# Зависимости **Propan** использует второстепенную библиотеку [**FastDepends**](https://lancetnik.github.io/FastDepends/){target="_blank"} ддя управления зависимостями. Эта система зависимостей буквально позаимствована у **FastAPI**, так что, если вы умеет работать с этим фреймворков - вы умеете работать с зависимостями в Propan. @@ -11,6 +11,12 @@ По умолчанию он применяется ко всем обработчикам событий, если только вы не отключили соответсвующую опцию при создании брокера. +=== "Redis" + ```python + from propan import RedisBroker + broker = RedisBroker(..., apply_types=False) + ``` + === "RabbitMQ" ```python from propan import RabbitBroker @@ -33,6 +39,11 @@ Для внедрения зависимостей в **Propan** используется специальный класс **Depends** +=== "Redis" + ```python linenums="1" hl_lines="6-7" + {!> docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="6-7" {!> docs_src/quickstart/dependencies/basic/1_propan_rabbit_depends.py !} @@ -50,6 +61,11 @@ Другими словами: если вы можете написать такой код `my_object()` - `my_object` будет `Callable` +=== "Redis" + ```python hl_lines="1" linenums="10" + {!> docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py [ln:10-11]!} + ``` + === "RabbitMQ" ```python hl_lines="1" linenums="10" {!> docs_src/quickstart/dependencies/basic/1_propan_rabbit_depends.py [ln:10-11]!} @@ -62,6 +78,11 @@ **Вторым шагом**: объявите, какие зависимости вам нужны с помощью `Depends` +=== "Redis" + ```python hl_lines="2" linenums="10" + {!> docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py [ln:10-11]!} + ``` + === "RabbitMQ" ```python hl_lines="2" linenums="10" {!> docs_src/quickstart/dependencies/basic/1_propan_rabbit_depends.py [ln:10-11]!} @@ -85,6 +106,13 @@ Зависимости также могут содержать другие зависимости. Это работает очень предсказуемым образом: просто объявите `Depends` в зависимой функции. +=== "Redis" + ```python linenums="1" hl_lines="6-7 9-10 15-16" + {!> docs_src/quickstart/dependencies/basic/2_propan_redis_depends.py !} + ``` + + 1. Здесь вызывается вложенная зависимость + === "RabbitMQ" ```python linenums="1" hl_lines="6-7 9-10 15-16" {!> docs_src/quickstart/dependencies/basic/2_propan_rabbit_depends.py !} @@ -108,7 +136,6 @@ Чтобы предотвратить это поведение, просто используйте `Depends(..., cache=False)`. В этом случае зависимость будет испольняться для каждой функции в стеке вызова, где она используется. - ## Использование с обычными функциями Вы можете использовать декоратор `@apply_types` не только вместе с вашими `@broker.hanle`'ми, но и с обычными функциями: как синхронными, так и асинхронными. @@ -150,5 +177,4 @@ assert method("1") == 5 Также, результат выполнения зависимости кешируется. Если вы используете эту зависимости в `N` функциях, этот закешированный результат будет приводится к типу `N` раз (на входе в используемую функцию). -Для избежания проблем с этим, используйте [mypy](https://www.mypy-lang.org){target="_blank"} или просто будьте аккуратны с аннотацией -типов в вашем проекте. \ No newline at end of file +Для избежания проблем с этим, используйте [mypy](https://www.mypy-lang.org){target="_blank"} или просто будьте аккуратны с аннотацией типов в вашем проекте. diff --git a/docs/docs/ru/2_getting_started/5_dependency/2_context.md b/docs/docs/ru/getting_started/5_dependency/2_context.md similarity index 100% rename from docs/docs/ru/2_getting_started/5_dependency/2_context.md rename to docs/docs/ru/getting_started/5_dependency/2_context.md diff --git a/docs/docs/ru/2_getting_started/6_lifespans.md b/docs/docs/ru/getting_started/6_lifespans.md similarity index 78% rename from docs/docs/ru/2_getting_started/6_lifespans.md rename to docs/docs/ru/getting_started/6_lifespans.md index 0aabf6a5..229032e8 100644 --- a/docs/docs/ru/2_getting_started/6_lifespans.md +++ b/docs/docs/ru/getting_started/6_lifespans.md @@ -1,4 +1,4 @@ -# LIFESPANS +# LIFESPANS Иногда вам нужно определить логику, которая должна исполняться перед запуском приложения. Это означает, что код будет исполнен один раз - еще до того, как ваше приложение начнет принимать сообщения. @@ -6,11 +6,10 @@ Также, у вас может возникнуть необходимость завершить некоторые процессы после остановки приложения. В этом случае, ваш код также будет выполнен ровно один раз: но уже после завершения работы основного приложения. -Поскольку этот код исполняется перед запуском приложения и после его остановки, он покрывает весь жизненный цикл *( lifespan )* приложения. +Поскольку этот код исполняется перед запуском приложения и после его остановки, он покрывает весь жизненный цикл *(lifespan)* приложения. Это может быть очень полезно для инициализации настроек вашего приложения при старте, поднятия пула соединений к базе данных или запуска моделей машинного обучения. - ## Пример использования Давайте представим, что ваше приложение использует **pydantic** в качестве менеджера ваших настроек. @@ -28,6 +27,11 @@ Давайте напишем немного кода для нашего примера +=== "Redis" + ```python linenums="1" hl_lines="12-16" + {!> docs_src/quickstart/lifespan/1_redis.py!} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="12-16" {!> docs_src/quickstart/lifespan/1_rabbit.py!} @@ -39,38 +43,89 @@ ``` Теперь это приложение можно запускать с помощью следующей команды для управления окружением: + ```bash -$ propan run serve:app --env .env.test +propan run serve:app --env .env.test ``` ### Детали Теперь разберемся немного детальнее -Для начала, мы использовали декоратор -```python linenums="12" hl_lines="1" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` +Для начала, мы использовали декоратор + +=== "Redis" + ```python linenums="12" hl_lines="1" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="1" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="1" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` + для объявления функции, которая должна запускаться при старте нашего приложения Следующим шагом мы объявили аргументы, которые будет получать наша функция -```python linenums="12" hl_lines="2" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` + +=== "Redis" + ```python linenums="12" hl_lines="2" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="2" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="2" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` + При этом поле `env` будет передано в функцию `setup` из аргументов командой строки !!! tip Функции жизненного цикла по умолчанию используются с декоратором `@apply_types`, поэтому в них доступны все [поля контекста](../5_dependency/2_context) и [зависимости](../5_dependency/1_di-index) Затем, мы инициализировали настройки нашего приложения с использованием переданного нам из командой строки файла -```python linenums="12" hl_lines="3" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` + +=== "Redis" + ```python linenums="12" hl_lines="3" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="3" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="3" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` И поместили эти настройки в глобальный контекст -```python linenums="12" hl_lines="4" -{!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} -``` + +=== "Redis" + ```python linenums="12" hl_lines="4" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + +=== "RabbitMQ" + ```python linenums="12" hl_lines="4" + {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} + ``` + +=== "NATS" + ```python linenums="12" hl_lines="4" + {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} + ``` ??? note Теперь мы можем получить доступ к нашим настройкам в любом месте приложения прямо из контекста @@ -82,6 +137,12 @@ $ propan run serve:app --env .env.test ``` Последним шагом мы инициализировали нашего брокера: теперь, при старте приложения он будет готов принимать сообщения + +=== "Redis" + ```python linenums="12" hl_lines="5" + {!> docs_src/quickstart/lifespan/1_redis.py [ln:11-15] !} + ``` + === "RabbitMQ" ```python linenums="12" hl_lines="5" {!> docs_src/quickstart/lifespan/1_rabbit.py [ln:11-15] !} @@ -92,7 +153,6 @@ $ propan run serve:app --env .env.test {!> docs_src/quickstart/lifespan/1_nats.py [ln:11-15] !} ``` - ## Другой пример Теперь давайте представим, что у нас есть модель машинного обучения, которая должна обрабатывать сообщения из какого-либо брокера. @@ -106,6 +166,11 @@ $ propan run serve:app --env .env.test Также, мы не хотим, чтобы модель окончила свою работу при остановке приложения некорректно. Чтобы избежать этого, нам понадобиться хук `@app.on_shutdown` +=== "Redis" + ```python linenums="1" hl_lines="12 18" + {!> docs_src/quickstart/lifespan/2_ml_redis.py !} + ``` + === "RabbitMQ" ```python linenums="1" hl_lines="12 18" {!> docs_src/quickstart/lifespan/2_ml_rabbit.py !} @@ -127,12 +192,15 @@ $ propan run serve:app --env .env.test ## Еще немного деталей ### Async или не async + В асинхронной версии приложения в качестве хуков могут использоваться как асинхронные, так и синхронные методы. В синхронной версии доступны только синхронные методы. ### Аргументы командной строки + Аргументы командной строки доступно во всех `@app.on_startup` хуках. Для использования их в других частях приложения поместите их в `ContextRepo`. ### Инициализация брокера -Хуки `@app.on_startup` вызываются **ДО** запуска брокера приложением. Хуки `@app.on_shutdown` запускаются **ПОСЛЕ** остановки брокера. + +Хуки `@app.on_startup` вызываются **ДО** запуска брокера приложением. Хуки `@app.after_shutdown` запускаются **ПОСЛЕ** остановки брокера. diff --git a/docs/docs/ru/2_getting_started/7_testing.md b/docs/docs/ru/getting_started/7_testing.md similarity index 61% rename from docs/docs/ru/2_getting_started/7_testing.md rename to docs/docs/ru/getting_started/7_testing.md index 58531324..515b47a6 100644 --- a/docs/docs/ru/2_getting_started/7_testing.md +++ b/docs/docs/ru/getting_started/7_testing.md @@ -11,39 +11,65 @@ Допустим, у нас есть приложение со следующим содержанием: +=== "Redis" + ```python linenums="1" title="main.py" + {!> docs_src/quickstart/testing/1_main_redis.py!} + ``` + + Для того, чтобы протестировать его без запуска *Redis*, необходимо модифицировать брокера с помощью `propan.test.TestRedisBroker`: + + ```python linenums="1" title="test_ping.py" hl_lines="1 6" + {!> docs_src/quickstart/testing/2_test_redis.py!} + ``` + === "RabbitMQ" - ```python linenums="1" title="main.py" - {!> docs_src/rabbit/testing/1_main.py!} - ``` + ```python linenums="1" title="main.py" + {!> docs_src/quickstart/testing/1_main_rabbit.py!} + ``` - Для того, чтобы протестировать его без запуска *RabbitMQ*, необходимо модифицировать брокера с помощью `propan.test.TestRabbitBroker`: + Для того, чтобы протестировать его без запуска *RabbitMQ*, необходимо модифицировать брокера с помощью `propan.test.TestRabbitBroker`: - ```python linenums="1" title="test_ping.py" hl_lines="1 6" - {!> docs_src/rabbit/testing/2_test.py!} - ``` + ```python linenums="1" title="test_ping.py" hl_lines="1 6" + {!> docs_src/quickstart/testing/2_test_rabbit.py!} + ``` Для тестирования мы сначала должны запустить обработчики наших сообщений: это делается с помощью метода `start`: +=== "Redis" + ```python hl_lines="2" + {!> docs_src/quickstart/testing/2_test_redis.py [ln:6-8]!} + ``` + === "RabbitMQ" - ```python hl_lines="2" - {!> docs_src/rabbit/testing/2_test.py [ln:6-8]!} - ``` + ```python hl_lines="2" + {!> docs_src/quickstart/testing/2_test_rabbit.py [ln:6-8]!} + ``` А затем мы делает *RPC* запрос для того, чтобы проверить результат выполнения: +=== "Redis" + ```python hl_lines="3" + {!> docs_src/quickstart/testing/2_test_redis.py [ln:6-8]!} + ``` + === "RabbitMQ" - ```python hl_lines="3" - {!> docs_src/rabbit/testing/2_test.py [ln:6-8]!} - ``` + ```python hl_lines="3" + {!> docs_src/quickstart/testing/2_test_rabbit.py [ln:6-8]!} + ``` ## Использование фикстур Для больших приложений для переиспользования тестового брокера вы можете использовать фикстуру следующего содержания: +=== "Redis" + ```python linenums="1" title="test_broker.py" hl_lines="6-10 12" + {!> docs_src/quickstart/testing/3_conftest_redis.py !} + ``` + === "RabbitMQ" - ```python linenums="1" title="test_broker.py" hl_lines="6-10 12" - {!> docs_src/rabbit/testing/3_conftest.py !} - ``` + ```python linenums="1" title="test_broker.py" hl_lines="6-10 12" + {!> docs_src/quickstart/testing/3_conftest_rabbit.py !} + ``` ## Прямой вызов функций @@ -52,10 +78,15 @@ Например, следующий тест вернет `None`, а внутри обработчика - возникнет `pydantic.ValidationError`: +=== "Redis" + ```python hl_lines="4 6" + {!> docs_src/quickstart/testing/4_suppressed_exc_redis.py !} + ``` + === "RabbitMQ" - ```python hl_lines="4 6" - {!> docs_src/rabbit/testing/4_suppressed_exc.py !} - ``` + ```python hl_lines="4 6" + {!> docs_src/quickstart/testing/4_suppressed_exc_rabbit.py !} + ``` Также этот тест будет заблокировать на `callback_timeout` (по умолчанию **30** секунд), что может может сильно раздражать, когда внутри разрабатываемого обработчика возникают ошибки, а ваши тесты отваливаются по длительному таймауту с `None`. @@ -65,16 +96,26 @@ Для этого вам нужно сконструироваться сообщение с помощью метода `build_message` так, если бы это был `pubslih` (сигнатуры методов совпадают), а затем передать это сообщение в ваш обработчик в качестве единственного аргумента функции. +=== "Redis" + ```python linenums="1" hl_lines="6-7" title="test_ping.py" + {!> docs_src/quickstart/testing/5_build_message_redis.py !} + ``` + === "RabbitMQ" - ```python linenums="1" hl_lines="6-7" title="test_ping.py" - {!> docs_src/rabbit/testing/5_build_message.py !} - ``` + ```python linenums="1" hl_lines="6-7" title="test_ping.py" + {!> docs_src/quickstart/testing/5_build_message_rabbit.py !} + ``` При этом, если вы хотите, чтобы захватывать исключения обработчика, вам нужно использовать флаг `reraise_exc=True` при вызове: +=== "Redis" + ```python linenums="1" hl_lines="9-10" title="test_ping.py" + {!> docs_src/quickstart/testing/6_reraise_redis.py !} + ``` + === "RabbitMQ" - ```python linenums="1" hl_lines="9-10" title="test_ping.py" - {!> docs_src/rabbit/testing/6_reraise.py !} - ``` + ```python linenums="1" hl_lines="9-10" title="test_ping.py" + {!> docs_src/quickstart/testing/6_reraise_rabbit.py !} + ``` Таким образом, **Propan** предоставляет вам полный инструментарий для тестирования ваших обработчиков: от валидации *RPC* ответов до корректно выполнения тела функций. \ No newline at end of file diff --git a/docs/docs/ru/2_getting_started/8_logging.md b/docs/docs/ru/getting_started/8_logging.md similarity index 100% rename from docs/docs/ru/2_getting_started/8_logging.md rename to docs/docs/ru/getting_started/8_logging.md diff --git a/docs/docs/ru/8_help.md b/docs/docs/ru/help.md similarity index 100% rename from docs/docs/ru/8_help.md rename to docs/docs/ru/help.md diff --git a/docs/docs/ru/index.md b/docs/docs/ru/index.md index 8ace6b4b..ee604093 100644 --- a/docs/docs/ru/index.md +++ b/docs/docs/ru/index.md @@ -38,17 +38,18 @@ * **Простота**: спроектирован для максимальной простоты изучения и использования. * **Интуитивность**: Отличная поддержка IDE, автодополнение даже в *vim*`е. -* [**Управление зависимостями**](2_getting_started/1_quick-start/#_4): Эффективное переиспользование за счет аннотации типов. Доступ к зависимостями во всем стеке вызова. -* [**Интeграция**](2_getting_started/1_quick-start/#http): Propan полностью совместим с [любыми HTTP фреймворками](5_integrations/1_integrations-index/) +* [**Управление зависимостями**](getting_started/1_quick-start/#_4): Эффективное переиспользование за счет аннотации типов. Доступ к зависимостями во всем стеке вызова. +* [**Интeграция**](getting_started/1_quick-start/#http): Propan полностью совместим с [любыми HTTP фреймворками](integrations/1_integrations-index/) * **Независимость от брокеров**: Единый интерфейс для популярных брокеров: - * **NATS** (основан на [nats-py](https://github.com/nats-io/nats.py)) - * **RabbitMQ** (основан на [aio-pika](https://aio-pika.readthedocs.io/en/latest/)) -* [**RPC**](2_getting_started/4_broker/5_rpc/): Фреймворк поддерживает RPC запросы поверх брокеров сообщений, что позволит выполнять длительные операции на удаленных сервисах асинхронно. -* [**Скорость разработки**](2_getting_started/2_cli/): собственный *CLI* инструмент предоставляет отличный опыт разработки: + * **Redis** (основан на [redis-py]("https://redis.readthedocs.io/en/stable/index.html"){target="_blank"}) + * **NATS** (основан на [nats-py](https://github.com/nats-io/nats.py){target="_blank"}) + * **RabbitMQ** (основан на [aio-pika](https://aio-pika.readthedocs.io/en/latest/){target="_blank"}) +* [**RPC**](getting_started/4_broker/5_rpc/): Фреймворк поддерживает RPC запросы поверх брокеров сообщений, что позволит выполнять длительные операции на удаленных сервисах асинхронно. +* [**Скорость разработки**](getting_started/2_cli/): собственный *CLI* инструмент предоставляет отличный опыт разработки: * Полностью совместимый с любым фреймворком способ управлять окружением проекта - * *hotreloading* при изменениях в коде + * *hot reloading* при изменениях в коде * Готовые шаблоны проекта -* [**Тестируемость**](2_getting_started/7_testing): **Propan** позволяет тестировать ваше приложение без внешних зависимостей: вам не нужно поднимать брокер сообщений, используйте виртуального! +* [**Тестируемость**](getting_started/7_testing): **Propan** позволяет тестировать ваше приложение без внешних зависимостей: вам не нужно поднимать брокер сообщений, используйте виртуального! --- @@ -57,7 +58,7 @@ Декларативные иснтрументы позволяют нам описывать **что мы хотим получить**, в то время как традиционные императивные инструменты заставляют нас писать **что мы хотим сделать**. -К традиционным императивным библиотекам относятся [aio-pika](https://aio-pika.readthedocs.io/en/latest/){target="_blank"}, [pika](https://pika.readthedocs.io/en/stable/){target="_blank"}, [nats-py](https://github.com/nats-io/nats.py){target="_blank"} и подобные. +К традиционным императивным библиотекам относятся [aio-pika](https://aio-pika.readthedocs.io/en/latest/){target="_blank"}, [pika](https://pika.readthedocs.io/en/stable/){target="_blank"}, [redis-py]("https://redis.readthedocs.io/en/stable/index.html"){target="_blank"}, [nats-py](https://github.com/nats-io/nats.py){target="_blank"} и подобные. Например, это **Quickstart** из библиотеки *aio-pika*: @@ -110,15 +111,16 @@ async def base_handler(body): ## Поддерживаемые брокеры !!! note "Нужна ваша помощь" - Фреймоворк сейчас активно развивается. У нас очень длинный список того, что еще предстоит реализовать и различные брокеры - только его часть. Если вы хотите реализовать что-то из этого списка или помочь любым другим способом - загляните [сюда](6_contributing/1_todo/) - -| | async | sync | -|--------------|:-------------------------------------------------------:|:--------------------:| -| **RabbitMQ** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | -| **Nats** | :warning: **beta** :warning: | :mag: planning :mag: | -| **NatsJS** | :hammer_and_wrench: **in progress** :hammer_and_wrench: | :mag: planning :mag: | -| **MQTT** | :mag: planning :mag: | :mag: planning :mag: | -| **Redis** | :mag: planning :mag: | :mag: planning :mag: | -| **Kafka** | :mag: planning :mag: | :mag: planning :mag: | -| **Pulsar** | :mag: planning :mag: | :mag: planning :mag: | -| **SQS** | :mag: planning :mag: | :mag: planning :mag: | \ No newline at end of file + Фреймоворк сейчас активно развивается. У нас очень длинный список того, что еще предстоит реализовать и различные брокеры - только его часть. Если вы хотите реализовать что-то из этого списка или помочь любым другим способом - загляните [сюда](contributing/1_todo/) + +| | async | sync | +|-------------------|:-------------------------------------------------------:|:--------------------:| +| **RabbitMQ** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | +| **Redis** | :heavy_check_mark: **stable** :heavy_check_mark: | :mag: planning :mag: | +| **Nats** | :warning: **beta** :warning: | :mag: planning :mag: | +| **NatsJS** | :hammer_and_wrench: **in progress** :hammer_and_wrench: | :mag: planning :mag: | +| **MQTT** | :mag: planning :mag: | :mag: planning :mag: | +| **Kafka** | :mag: planning :mag: | :mag: planning :mag: | +| **Redis Streams** | :mag: planning :mag: | :mag: planning :mag: | +| **Pulsar** | :mag: planning :mag: | :mag: planning :mag: | +| **SQS** | :mag: planning :mag: | :mag: planning :mag: | \ No newline at end of file diff --git a/docs/docs/ru/5_integrations/1_integrations-index.md b/docs/docs/ru/integrations/1_integrations-index.md similarity index 100% rename from docs/docs/ru/5_integrations/1_integrations-index.md rename to docs/docs/ru/integrations/1_integrations-index.md diff --git a/docs/docs/ru/5_integrations/2_fastapi-plugin.md b/docs/docs/ru/integrations/2_fastapi-plugin.md similarity index 75% rename from docs/docs/ru/5_integrations/2_fastapi-plugin.md rename to docs/docs/ru/integrations/2_fastapi-plugin.md index 617e66d5..64ee57f4 100644 --- a/docs/docs/ru/5_integrations/2_fastapi-plugin.md +++ b/docs/docs/ru/integrations/2_fastapi-plugin.md @@ -13,9 +13,15 @@ Обратите внимание, что в коде ниже используется `fastapi.Depends`, а не `propan.Depends`. -```python linenums="1" hl_lines="1 3 7 15 19 23" -{!> docs_src/integrations/fastapi_plugin_rabbit.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="1 3 7 15 19 23" + {!> docs_src/integrations/fastapi_plugin_redis.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="1 3 7 15 19 23" + {!> docs_src/integrations/fastapi_plugin_rabbit.py!} + ``` При обработке сообщения из брокера все тело сообщения помещается одновременно и в `body`, и в `path` параметры запроса: вы можете достать получить к ним доступ любым удобным для вас способом. Заголовок сообщения помещается в `headers`. @@ -26,12 +32,24 @@ Внутри каждого роутера есть соответсвующий брокер. Вы можете легко получить к нему доступ, если вам необходимо отправить сообщение в MQ. -```python linenums="1" hl_lines="6 10" -{!> docs_src/integrations/fastapi_plugin_rabbit_send.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="6 10" + {!> docs_src/integrations/fastapi_plugin_redis_send.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="6 10" + {!> docs_src/integrations/fastapi_plugin_rabbit_send.py!} + ``` Вы можете оформить доступ к брокеру в виде `Depends`, если хотите использовать его в разных частях вашей программы. -```python linenums="1" hl_lines="8 14-15" -{!> docs_src/integrations/fastapi_plugin_rabbit_depends.py!} -``` +=== "Redis" + ```python linenums="1" hl_lines="8 14-15" + {!> docs_src/integrations/fastapi_plugin_redis_depends.py!} + ``` + +=== "RabbitMQ" + ```python linenums="1" hl_lines="8 14-15" + {!> docs_src/integrations/fastapi_plugin_rabbit_depends.py!} + ``` diff --git a/docs/docs/ru/4_nats/1_nats-index.md b/docs/docs/ru/nats/1_nats-index.md similarity index 100% rename from docs/docs/ru/4_nats/1_nats-index.md rename to docs/docs/ru/nats/1_nats-index.md diff --git a/docs/docs/ru/4_nats/2_routing.md b/docs/docs/ru/nats/2_routing.md similarity index 100% rename from docs/docs/ru/4_nats/2_routing.md rename to docs/docs/ru/nats/2_routing.md diff --git a/docs/docs/ru/4_nats/3_publishing.md b/docs/docs/ru/nats/3_publishing.md similarity index 100% rename from docs/docs/ru/4_nats/3_publishing.md rename to docs/docs/ru/nats/3_publishing.md diff --git a/docs/docs/ru/3_rabbit/1_routing.md b/docs/docs/ru/rabbit/1_routing.md similarity index 100% rename from docs/docs/ru/3_rabbit/1_routing.md rename to docs/docs/ru/rabbit/1_routing.md diff --git a/docs/docs/ru/3_rabbit/2_exchanges.md b/docs/docs/ru/rabbit/2_exchanges.md similarity index 100% rename from docs/docs/ru/3_rabbit/2_exchanges.md rename to docs/docs/ru/rabbit/2_exchanges.md diff --git a/docs/docs/ru/3_rabbit/3_queues.md b/docs/docs/ru/rabbit/3_queues.md similarity index 100% rename from docs/docs/ru/3_rabbit/3_queues.md rename to docs/docs/ru/rabbit/3_queues.md diff --git a/docs/docs/ru/3_rabbit/4_publishing.md b/docs/docs/ru/rabbit/4_publishing.md similarity index 87% rename from docs/docs/ru/3_rabbit/4_publishing.md rename to docs/docs/ru/rabbit/4_publishing.md index efaa2584..7384b2af 100644 --- a/docs/docs/ru/3_rabbit/4_publishing.md +++ b/docs/docs/ru/rabbit/4_publishing.md @@ -4,24 +4,26 @@ Однако, в данном случае в качестве сообщения (помимо `str`, `bytes`, `dict`, `pydatic.BaseModel`) может выступать объект класса `aio_pika.Message` (при необходимости). ```python -from propan.brokers.rabbit import RabbitBroker, RabbitQueue +import asyncio +from propan import RabbitBroker -broker = RabbitBroker() +async def pub(): + async with RabbitBroker() as broker: + await broker.publish("Hi!", queue="test", exhcange="test") -... - await broker.publish("Hi!", queue="test", exhcange="test") +asyncio.run(pub()) ``` -### Базовые аргументы +## Базовые аргументы -Метод `pubslish` принимает следующие аргументы: +Метод `publish` принимает следующие аргументы: -* `message`: bytes | str | dict | pydatic.BaseModel | aio_pika.Message = "" - сообщение для отправки +* `message`: bytes | str | dict | Sequence[Any] | pydatic.BaseModel | aio_pika.Message = "" - сообщение для отправки * `exchange`: str | RabbitExchange | None = None - exchange, куда будет отправлено сообщение. Если не указан - используется *default* * `queue`: str | RabbitQueue = "" - очередь, куда будет отправлено сообщение (т.к. большинство очередей используют свое название в качестве ключа маршрутизации, это человекочитаемый вариант `routing_key`) * `routing_key`: str = "" - тоже ключ маршрутизации сообщения, если не указан - используется аргумент `queue` -### Параметры сообщения +## Параметры сообщения Подробнее обо всех флагах вы можете прочитать в [документации RabbitMQ](https://www.rabbitmq.com/consumers.html){.external-link target="_blank"} @@ -39,7 +41,7 @@ broker = RabbitBroker() * `user_id`: str | None - идентификатор пользователя *RabbitMQ*, отправившего сообщение * `app_id`: str | None - идентификатор приложения, отправившего сообщение (используется потребителями) -### Флаги отправки +## Флаги отправки Аргументы отправки сообщения: @@ -47,12 +49,12 @@ broker = RabbitBroker() * `immediate`: bool = False - клиент ожидает, что есть потребитель, готовый взять сообщение в работу "прямо сейчас" (если потребителя нет - вернуть отправителю) * `timeout`: int | float | None = None - время подтверждения отправки от *RabbitMQ* -### RPC аргументы +## RPC аргументы -Также `publish` поддерживает общие аргументы для создания [*RPC* запросов](../../2_getting_started/4_broker/5_rpc/#_3): +Также `publish` поддерживает общие аргументы для создания [*RPC* запросов](../../getting_started/4_broker/5_rpc/#_3): * `callback`: bool = False - ожидать ли ответ на сообщение -* `callbakc_timeout`: float | None = 30.0 - таймаут ожидания ответа. В случае `None` - ждет бесконечно +* `callback_timeout`: float | None = 30.0 - таймаут ожидания ответа. В случае `None` - ждет бесконечно * `raise_timeout`: bool = False * `False` - возвращать None в случае таймаута * `True` - ошибка `asyncio.TimeoutError` в случае таймаута diff --git a/docs/docs/ru/3_rabbit/5_examples/1_direct.md b/docs/docs/ru/rabbit/5_examples/1_direct.md similarity index 85% rename from docs/docs/ru/3_rabbit/5_examples/1_direct.md rename to docs/docs/ru/rabbit/5_examples/1_direct.md index a9e4d711..a833bcee 100644 --- a/docs/docs/ru/3_rabbit/5_examples/1_direct.md +++ b/docs/docs/ru/rabbit/5_examples/1_direct.md @@ -6,7 +6,7 @@ !!! note **Default** Exchange, на который подписаны все очереди в *RabbitMQ* по умолчанию имеет тип **Direct** -## Распределение сообщений +## Масштабирование Если одну очередь слушает несколько потребителей, сообщение будет уходить каждый раз новому потребителю. Это поведение общее для всех типов `exchange`, т.к. оно относится к самой очереди. Тип `exchange` влияет на то, в какие очереди попадет сообщение. @@ -26,7 +26,7 @@ async def handler(): Аргумент `auto_delete=True` в этом и последующих примерах используется только для того, чтобы очистить состояние *RabbitMQ* после примера ```python linenums="1" -{!> docs_src/rabbit/examples/direct.py !} +{!> docs_src/rabbit/direct.py !} ``` ### Объявление потребителей @@ -34,13 +34,13 @@ async def handler(): Для начала мы объявили наш **Direct** exchange и несколько очередей, которые будут его слушать: ```python linenums="8" -{!> docs_src/rabbit/examples/direct.py [ln:8-11]!} +{!> docs_src/rabbit/direct.py [ln:8-11]!} ``` Затем мы подписали несколько потребителей с помощью объявленных очередей на созданный нами `exchange` ```python linenums="13" hl_lines="1 5 9" -{!> docs_src/rabbit/examples/direct.py [ln:13-23]!} +{!> docs_src/rabbit/direct.py [ln:13-23]!} ``` !!! note @@ -53,31 +53,31 @@ async def handler(): Теперь распределение сообщений между этими потребителями будет выглядеть следующим образом: ```python -{!> docs_src/rabbit/examples/direct.py [ln:29]!} +{!> docs_src/rabbit/direct.py [ln:27]!} ``` -Сообщений `1` будет отправлено в `handler1`, т.к. он слушает `exchange` с помощью очереди с ключом маршрутизации `test-q-1` +Сообщение `1` будет отправлено в `handler1`, т.к. он слушает `exchange` с помощью очереди с ключом маршрутизации `test-q-1` --- ```python -{!> docs_src/rabbit/examples/direct.py [ln:30]!} +{!> docs_src/rabbit/direct.py [ln:28]!} ``` -Сообщений `2` будет отправлено в `handler2`, т.к. он слушает `exchange` с помощью той же очереди, но `handler1` занят +Сообщение `2` будет отправлено в `handler2`, т.к. он слушает `exchange` с помощью той же очереди, но `handler1` занят --- ```python -{!> docs_src/rabbit/examples/direct.py [ln:31]!} +{!> docs_src/rabbit/direct.py [ln:29]!} ``` -Сообщений `3` снова будет отправлено в `handler1`, т.к. он освободился на данный момент +Сообщение `3` снова будет отправлено в `handler1`, т.к. он освободился на данный момент --- ```python -{!> docs_src/rabbit/examples/direct.py [ln:32]!} +{!> docs_src/rabbit/direct.py [ln:30]!} ``` -Сообщений `4` будет отправлено в `handler3`, т.к. он единственный слушает `exchange` с помощью очереди с ключом маршрутизации `test-q-2` \ No newline at end of file +Сообщение `4` будет отправлено в `handler3`, т.к. он единственный слушает `exchange` с помощью очереди с ключом маршрутизации `test-q-2` \ No newline at end of file diff --git a/docs/docs/ru/3_rabbit/5_examples/2_fanout.md b/docs/docs/ru/rabbit/5_examples/2_fanout.md similarity index 79% rename from docs/docs/ru/3_rabbit/5_examples/2_fanout.md rename to docs/docs/ru/rabbit/5_examples/2_fanout.md index 22e515a6..acc4a6c3 100644 --- a/docs/docs/ru/3_rabbit/5_examples/2_fanout.md +++ b/docs/docs/ru/rabbit/5_examples/2_fanout.md @@ -8,7 +8,7 @@ ## Пример ```python linenums="1" -{!> docs_src/rabbit/examples/fanout.py !} +{!> docs_src/rabbit/fanout.py !} ``` ### Объявление потребителей @@ -16,13 +16,13 @@ Для начала мы объявили наш **Fanout** exchange и несколько очередей, которые будут его слушать: ```python linenums="8" hl_lines="1" -{!> docs_src/rabbit/examples/fanout.py [ln:8-11]!} +{!> docs_src/rabbit/fanout.py [ln:8-11]!} ``` Затем мы подписали несколько потребителей с помощью объявленных очередей на созданный нами `exchange` ```python linenums="13" hl_lines="1 5 9" -{!> docs_src/rabbit/examples/fanout.py [ln:13-23]!} +{!> docs_src/rabbit/fanout.py [ln:13-23]!} ``` !!! note @@ -35,34 +35,34 @@ Теперь распределение сообщений между этими потребителями будет выглядеть следующим образом: ```python -{!> docs_src/rabbit/examples/fanout.py [ln:29]!} +{!> docs_src/rabbit/fanout.py [ln:27]!} ``` -Сообщений `1` будет отправлено в `handler1` и `handler3`, т.к. они слушает `exchange` с помощью разных очередей +Сообщение `1` будет отправлено в `handler1` и `handler3`, т.к. они слушает `exchange` с помощью разных очередей --- ```python -{!> docs_src/rabbit/examples/fanout.py [ln:30]!} +{!> docs_src/rabbit/fanout.py [ln:28]!} ``` -Сообщений `2` будет отправлено в `handler2` и `handler3`, т.к. `handler2` слушает `exchange` с помощью той же очереди, что и `handler1` +Сообщение `2` будет отправлено в `handler2` и `handler3`, т.к. `handler2` слушает `exchange` с помощью той же очереди, что и `handler1` --- ```python -{!> docs_src/rabbit/examples/fanout.py [ln:31]!} +{!> docs_src/rabbit/fanout.py [ln:29]!} ``` -Сообщений `3` будет отправлено в `handler1` и `handler3` +Сообщение `3` будет отправлено в `handler1` и `handler3` --- ```python -{!> docs_src/rabbit/examples/fanout.py [ln:32]!} +{!> docs_src/rabbit/fanout.py [ln:30]!} ``` -Сообщений `4` будет отправлено в `handler3` и `handler3` +Сообщение `4` будет отправлено в `handler3` и `handler3` --- diff --git a/docs/docs/ru/3_rabbit/5_examples/3_topic.md b/docs/docs/ru/rabbit/5_examples/3_topic.md similarity index 81% rename from docs/docs/ru/3_rabbit/5_examples/3_topic.md rename to docs/docs/ru/rabbit/5_examples/3_topic.md index 95f079e2..abc95473 100644 --- a/docs/docs/ru/3_rabbit/5_examples/3_topic.md +++ b/docs/docs/ru/rabbit/5_examples/3_topic.md @@ -8,7 +8,7 @@ ## Пример ```python linenums="1" -{!> docs_src/rabbit/examples/topic.py !} +{!> docs_src/rabbit/topic.py !} ``` ### Объявление потребителей @@ -16,7 +16,7 @@ Для начала мы объявили наш **Topic** exchange и несколько очередей, которые будут его слушать: ```python linenums="8" hl_lines="1 3-4" -{!> docs_src/rabbit/examples/topic.py [ln:8-11]!} +{!> docs_src/rabbit/topic.py [ln:8-11]!} ``` При этом в `routing_key` наших очередей мы указываем *паттерн* ключей маршрутизации, которые будут обрабатываться этой очередью. @@ -24,7 +24,7 @@ Затем мы подписали несколько потребителей с помощью объявленных очередей на созданный нами `exchange` ```python linenums="13" hl_lines="1 5 9" -{!> docs_src/rabbit/examples/topic.py [ln:13-23]!} +{!> docs_src/rabbit/topic.py [ln:13-23]!} ``` !!! note @@ -37,31 +37,31 @@ Теперь распределение сообщений между этими потребителями будет выглядеть следующим образом: ```python -{!> docs_src/rabbit/examples/topic.py [ln:29]!} +{!> docs_src/rabbit/topic.py [ln:27]!} ``` -Сообщений `1` будет отправлено в `handler1`, т.к. он слушает `exchange` с помощью очереди с ключом маршрутизации `*.info` +Сообщение `1` будет отправлено в `handler1`, т.к. он слушает `exchange` с помощью очереди с ключом маршрутизации `*.info` --- ```python -{!> docs_src/rabbit/examples/topic.py [ln:30]!} +{!> docs_src/rabbit/topic.py [ln:28]!} ``` -Сообщений `2` будет отправлено в `handler2`, т.к. он слушает `exchange` с помощью той же очереди, но `handler1` занят +Сообщение `2` будет отправлено в `handler2`, т.к. он слушает `exchange` с помощью той же очереди, но `handler1` занят --- ```python -{!> docs_src/rabbit/examples/topic.py [ln:31]!} +{!> docs_src/rabbit/topic.py [ln:29]!} ``` -Сообщений `3` снова будет отправлено в `handler1`, т.к. он освободился на данный момент +Сообщение `3` снова будет отправлено в `handler1`, т.к. он освободился на данный момент --- ```python -{!> docs_src/rabbit/examples/topic.py [ln:32]!} +{!> docs_src/rabbit/topic.py [ln:30]!} ``` -Сообщений `4` будет отправлено в `handler3`, т.к. он единственный слушает `exchange` с помощью очереди с ключом маршрутизации `*.debug` +Сообщение `4` будет отправлено в `handler3`, т.к. он единственный слушает `exchange` с помощью очереди с ключом маршрутизации `*.debug` diff --git a/docs/docs/ru/3_rabbit/5_examples/4_header.md b/docs/docs/ru/rabbit/5_examples/4_header.md similarity index 82% rename from docs/docs/ru/3_rabbit/5_examples/4_header.md rename to docs/docs/ru/rabbit/5_examples/4_header.md index 607714fa..9d1356a0 100644 --- a/docs/docs/ru/3_rabbit/5_examples/4_header.md +++ b/docs/docs/ru/rabbit/5_examples/4_header.md @@ -8,7 +8,7 @@ ## Пример ```python linenums="1" -{!> docs_src/rabbit/examples/header.py !} +{!> docs_src/rabbit/header.py !} ``` ### Объявление потребителей @@ -16,7 +16,7 @@ Для начала мы объявили наш **Fanout** exchange и несколько очередей, которые будут его слушать: ```python linenums="8" hl_lines="1 5 9 13" -{!> docs_src/rabbit/examples/header.py [ln:8-21]!} +{!> docs_src/rabbit/header.py [ln:8-21]!} ``` Аргумент `x-match` говорит о том, должны сопадать аргументы с заголовками сообщений полностью или частично. @@ -24,7 +24,7 @@ Затем мы подписали несколько потребителей с помощью объявленных очередей на созданный нами `exchange` ```python linenums="23" hl_lines="1 5 9 13" -{!> docs_src/rabbit/examples/header.py [ln:23-37]!} +{!> docs_src/rabbit/header.py [ln:23-37]!} ``` !!! note @@ -37,50 +37,50 @@ Теперь распределение сообщений между этими потребителями будет выглядеть следующим образом: ```python -{!> docs_src/rabbit/examples/header.py [ln:43]!} +{!> docs_src/rabbit/header.py [ln:41]!} ``` -Сообщений `1` будет отправлено в `handler1`, т.к. он слушает очередь, заголовок `key` которой, совпал с заголовком `key` сообщения +Сообщение `1` будет отправлено в `handler1`, т.к. он слушает очередь, заголовок `key` которой, совпал с заголовком `key` сообщения --- ```python -{!> docs_src/rabbit/examples/header.py [ln:44]!} +{!> docs_src/rabbit/header.py [ln:42]!} ``` -Сообщений `2` будет отправлено в `handler2`, т.к. он слушает `exchange` с помощью той же очереди, но `handler1` занят +Сообщение `2` будет отправлено в `handler2`, т.к. он слушает `exchange` с помощью той же очереди, но `handler1` занят --- ```python -{!> docs_src/rabbit/examples/header.py [ln:45]!} +{!> docs_src/rabbit/header.py [ln:43]!} ``` -Сообщений `3` снова будет отправлено в `handler1`, т.к. он освободился на данный момент +Сообщение `3` снова будет отправлено в `handler1`, т.к. он освободился на данный момент --- ```python -{!> docs_src/rabbit/examples/header.py [ln:46]!} +{!> docs_src/rabbit/header.py [ln:44]!} ``` -Сообщений `4` будет отправлено в `handler3`, т.к. он слушает очередь, заголовок `key` которой, совпал с заголовком `key` сообщения +Сообщение `4` будет отправлено в `handler3`, т.к. он слушает очередь, заголовок `key` которой, совпал с заголовком `key` сообщения --- ```python -{!> docs_src/rabbit/examples/header.py [ln:47]!} +{!> docs_src/rabbit/header.py [ln:45]!} ``` -Сообщений `5` будет отправлено в `handler3`, т.к. он слушает очередь, заголовок `key2` которой, совпал с заголовком `key2` сообщения +Сообщение `5` будет отправлено в `handler3`, т.к. он слушает очередь, заголовок `key2` которой, совпал с заголовком `key2` сообщения --- ```python -{!> docs_src/rabbit/examples/header.py [ln:48-49]!} +{!> docs_src/rabbit/header.py [ln:46-47]!} ``` -Сообщений `6` будет отправлено в `handler3` и `handler4`, т.к. заголовки сообщений полностью совпали с ключами очередей +Сообщение `6` будет отправлено в `handler3` и `handler4`, т.к. заголовки сообщений полностью совпали с ключами очередей --- diff --git a/docs/docs/ru/redis/1_redis-index.md b/docs/docs/ru/redis/1_redis-index.md new file mode 100644 index 00000000..4c072037 --- /dev/null +++ b/docs/docs/ru/redis/1_redis-index.md @@ -0,0 +1,50 @@ +# Redis Pub/Sub + +## Преимущества и недостатки + +Скорее всего в вашем проекте уже используется *Redis*. Если вы хотите использовать асинхронную отправку сообщений, но не хотите +внедрять новую тяжеловесную зависимость (*Kafka*, *RabbitMQ*, *Nats* и тд) в свою инфраструктуру, вам стоит воспользоваться им в качестве брокера сообщений. + +*Redis* работает быстро, не деградирует при большом количестве сообщений и самое главное - он уже у вас есть! + +!!! note + Подробнее с документацией *Redis Pub/Sub* вы можете ознакомиться на [официальном сайте](https://redis.io/docs/manual/pubsub/#messages-matching-both-a-pattern-and-a-channel-subscription){.external-link target="_blank"} + +Однако, *Redis* в качестве брокера сообщений обладает некоторыми существенными недостатками: + +* Сообщения не персистентны. Если сообщение будет опубликовано пока ваш потребитель отключен - оно будет потеряно. +* Отсутсвует возможность горизонтального масштабирования потребителей. +* Отстутствуют механизмы сложной маршрутизации. +* Отстутствуют механизмы подтверждения получения и обработки сообщений со стороны потребителя. +* Сообщения представлены сырыми байтами без метаинформации. + +Далеко не все эти особенности необходимы в вашем проекте, но вы должны иметь их в виду, когда выбираете *Redis* в качестве брокера сообщений. + +В любом случае, поскольку код приложения **Propan** слабо зависит от используемого брокера сообщений, вы можете построить прототип своей системы на базе *Redis*, а при необходимости - быстро адаптировать ее под использование другого брокера. + +Также, *Redis 5.0+* содержит механизм *Streams*, который также может служить в роли брокера сообщений и закрывает основные недостатки *Redis Pub/Sub*: персистентность сообщений и масштабирование потребителей. + +## Правила маршрутизации + +*Redis* не обладает возможностью настраивать сложные правила маршрутизации. Единственной сущностью в *Redis Pub/Sub* является `channel`, на который можно подписаться либо напрямую по имени, либо по паттерну регулярного выражения. + +Оба примера рассмотрены [чуть далее](../3_examples/1_direct). + +## Особенности **Propan** + +Так как *Redis* в качестве сообщения использует просто набор байт без заголовков и прочей метаинформации, **Propan** в качестве сообщения использует закодированный *json* со следующей структурой: + +```json +{ + "data": "", + "headers": {}, + "reply_to": "" +} +``` + +Это необходимо для корреткного распознавания *content-type* входящего сообщения (необходимо для правильного декодирования) и поддержки **RPC** запросов. + +Если **Propan** получает сообщение, отправленное с помощью другой библиотеки или фреймворка (или просто сообщение другого формата), +все тело этого сообщение будет воспринято как поле `data` принимаемого сообщения, а `content-type` будет распознан автоматически. + +При этом **RPC** запросы не будут работать, так как во входящем сообщении нет поля `reply_to`. diff --git a/docs/docs/ru/redis/2_publishing.md b/docs/docs/ru/redis/2_publishing.md new file mode 100644 index 00000000..4417cc19 --- /dev/null +++ b/docs/docs/ru/redis/2_publishing.md @@ -0,0 +1,55 @@ +# Redis Publishing + +Для отправки сообщений `RedisBroker` также использует унифицированный метод `publish`. + +```python +import asyncio +from propan import RedisBroker + +async def pub(): + async with RedisBroker() as broker: + await broker.publish("Hi!", channel="test") + +asyncio.run(pub()) +``` + +## Базовые аргументы + +Метод `publish` принимает следующие аргументы: + +* `message`: bytes | str | dict | Sequence[Any] | pydatic.BaseModel - сообщение для отправки +* `channel`: str = "" - *channel*, куда будет отправлено сообщение. + +## Параметры сообщения + +*Redis* по умолчанию отправляет сообщение в виде сырых `bytes`. **Propan** же использует собственный формат передачи сообщения: +при вызове метода `publish` в *Redis* отправляется *json* со следующими полями: + +```json +{ + "data": "", + "headers": {}, + "reply_to": "" +} +``` + +Самостоятельно вы можете выставить и использовать в рамках своего приложения заголовки отправляемого сообщения (там же автоматически выставляется `content-type`, по которому **Propan** определяет, как декодировать полученное сообщение) + +* `headers`: dict[str, Any] | None = None - заголовки отправляемого сообщения (используются потребителями) + +!!! note "" + Если **Propan** получает сообщение, отправленное с помощью другой библиотеки или фреймворка (или просто сообщение другого формата), + все тело этого сообщение будет воспринято как поле `data` принимаемого сообщения, а `content-type` будет распознан автоматически. + + При этом **RPC** запросы не будут работать, так как во входящем сообщении нет поля `reply_to`. + +## RPC аргументы + +Также `publish` поддерживает общие аргументы для создания [*RPC* запросов](../../getting_started/4_broker/5_rpc/#_3): + +* `reply_to`: str = "" - в какой *channel* отправить ответ (используется для асинхронных RPC запросов) +* `callback`: bool = False - ожидать ли ответ на сообщение +* `callback_timeout`: float | None = 30.0 - таймаут ожидания ответа. В случае `None` - ждет бесконечно +* `raise_timeout`: bool = False + * `False` - возвращать None в случае таймаута + * `True` - ошибка `asyncio.TimeoutError` в случае таймаута diff --git a/docs/docs/ru/redis/3_examples/1_direct.md b/docs/docs/ru/redis/3_examples/1_direct.md new file mode 100644 index 00000000..0c7cbfc6 --- /dev/null +++ b/docs/docs/ru/redis/3_examples/1_direct.md @@ -0,0 +1,57 @@ +# Direct + +**Direct** Channel - базовый способ маршрутизации сообщений в *Redis*. Его суть очень проста: +`channel` отправляет сообщения всем потребителям, подписанным на него. + +## Масштабирование + +Если один канал слушает несколько потребителей, сообщение будет получено **всеми** потребителями этого канала. +Таким образом, горизонтальное масштабирование путем увеличения количества сервисов-потребителей невозможно только средствами *Redis Pub/Sub*. + +Если вам нужен подобный функционал, посмотрите в сторону *Redis Streams* или других брокеров (например, *Nats* или *RabbitMQ*). + +## Пример + +**Direct** Channel - тип, используемый в **Propan** по умолчанию: вы можете просто объявить его следующим образом + +```python +@broker.handler("test_channel") +async def handler(): + ... +``` + +Полный пример: + +```python linenums="1" +{!> docs_src/redis/direct.py !} +``` + +### Объявление потребителей + +Для начала мы объявили несколько потребителей для двух каналов `test` и `test2`: + +```python linenums="8" hl_lines="1 6 11" +{!> docs_src/redis/direct.py [ln:8-20]!} +``` + +!!! note + Обратите внимание, что `handler1` и `handler2` подписаны на один `channel`: + cообщения будут приходить оба этих обработчика. + +### Распределение сообщений + +Теперь распределение сообщений между этими потребителями будет выглядеть следующим образом: + +```python +{!> docs_src/redis/direct.py [ln:25]!} +``` + +Сообщение `1` будет отправлено в `handler1` и `handler2`, т.к. они слушают `channel` с названием `test` + +--- + +```python +{!> docs_src/redis/direct.py [ln:26]!} +``` + +Сообщение `2` будет отправлено в `handler3`, т.к. он слушает `channel` с названием `test2` diff --git a/docs/docs/ru/redis/3_examples/2_pattern.md b/docs/docs/ru/redis/3_examples/2_pattern.md new file mode 100644 index 00000000..334c4130 --- /dev/null +++ b/docs/docs/ru/redis/3_examples/2_pattern.md @@ -0,0 +1,47 @@ +# Pattern + +**Pattern** Channel - мощный механизм маршрутизации *Redis*. Данный тип `channel` отправляет сообщения потребителям в соответсвии с *паттерном*, +указанном при их подключении к `channel` и ключом самого сообщения. + +## Масштабирование + +Если ключ сообщение совпадает с паттерном нескольких потребителей, оно будет отправлено **всем** им. +Таким образом, горизонтальное масштабирование путем увеличения количества сервисов-потребителей невозможно только средствами *Redis Pub/Sub*. + +Если вам нужен подобный функционал, посмотрите в сторону *Redis Streams* или других брокеров (например, *Nats* или *RabbitMQ*). + +## Пример + +```python linenums="1" +{!> docs_src/redis/pattern.py !} +``` + +### Объявление потребителей + +Для начала мы объявили несколько потребителей для двух каналов `*.info*` и `*.error`: + +```python linenums="8" hl_lines="1 6 11" +{!> docs_src/redis/pattern.py [ln:8-20]!} +``` + +!!! note + Обратите внимание, что `handler1` и `handler2` подписаны на один `channel`: + cообщения будут приходить оба этих обработчика. + +### Распределение сообщений + +Теперь распределение сообщений между этими потребителями будет выглядеть следующим образом: + +```python +{!> docs_src/redis/pattern.py [ln:25]!} +``` + +Сообщение `1` будет отправлено в `handler1` и `handler2`, т.к. они слушают `channel` с паттерном `*.info*` + +--- + +```python +{!> docs_src/redis/pattern.py [ln:26]!} +``` + +Сообщение `2` будет отправлено в `handler3`, т.к. он слушает `channel` с паттерном `*.error` diff --git a/docs/docs_src/index/01_redis_base.py b/docs/docs_src/index/01_redis_base.py new file mode 100644 index 00000000..0241b742 --- /dev/null +++ b/docs/docs_src/index/01_redis_base.py @@ -0,0 +1,9 @@ +from propan import PropanApp, RedisBroker + +broker = RedisBroker("redis://localhost:6379") + +app = PropanApp(broker) + +@broker.handle("test") +async def base_handler(body): + print(body) \ No newline at end of file diff --git a/docs/docs_src/index/02_redis_type_casting.py b/docs/docs_src/index/02_redis_type_casting.py new file mode 100644 index 00000000..5579ff4e --- /dev/null +++ b/docs/docs_src/index/02_redis_type_casting.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel +from propan import PropanApp, RedisBroker + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(broker) + +class SimpleMessage(BaseModel): + key: int + +@broker.handle("test2") +async def second_handler(body: SimpleMessage): + assert isinstance(body.key, int) \ No newline at end of file diff --git a/docs/docs_src/index/03_redis_dependencies.py b/docs/docs_src/index/03_redis_dependencies.py new file mode 100644 index 00000000..a164a2b9 --- /dev/null +++ b/docs/docs_src/index/03_redis_dependencies.py @@ -0,0 +1,14 @@ +from propan import PropanApp, RedisBroker, Context, Depends + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(gl_broker) + +async def base_dep(body: dict) -> bool: + return True + +@gl_broker.handle("test") +async def base_handler(body: dict, + dep: bool = Depends(base_dep), + broker: RedisBroker = Context()): + assert dep is True + assert broker is gl_broker diff --git a/docs/docs_src/index/04_redis_context.py b/docs/docs_src/index/04_redis_context.py new file mode 100644 index 00000000..c67f24f1 --- /dev/null +++ b/docs/docs_src/index/04_redis_context.py @@ -0,0 +1,15 @@ +from propan import PropanApp, RedisBroker +from propan.annotations import ContextRepo +from pydantic import BaseSettings + +broker = RedisBroker("redis://localhost:6379") + +app = PropanApp(broker) + +class Settings(BaseSettings): + ... + +@app.on_startup +async def setup(env: str, context: ContextRepo): + settings = Settings(_env_file=env) + context.set_global("settings", settings) \ No newline at end of file diff --git a/docs/docs_src/index/05_redis_http_example.py b/docs/docs_src/index/05_redis_http_example.py new file mode 100644 index 00000000..8675b0ac --- /dev/null +++ b/docs/docs_src/index/05_redis_http_example.py @@ -0,0 +1,17 @@ +from propan import RedisBroker +from sanic import Sanic + +app = Sanic("MyHelloWorldApp") +broker = RedisBroker("redis://localhost:6379") + +@broker.handle("test") +async def base_handler(body): + print(body) + +@app.after_server_start +async def start_broker(app, loop): + await broker.start() + +@app.after_server_stop +async def stop_broker(app, loop): + await broker.close() \ No newline at end of file diff --git a/docs/docs_src/index/06_redis_native_fastapi.py b/docs/docs_src/index/06_redis_native_fastapi.py new file mode 100644 index 00000000..c206bd71 --- /dev/null +++ b/docs/docs_src/index/06_redis_native_fastapi.py @@ -0,0 +1,19 @@ +from fastapi import Depends, FastAPI +from pydantic import BaseModel +from propan.fastapi import RedisRouter + +app = FastAPI() + +router = RedisRouter("redis://localhost:6379") + +class Incoming(BaseModel): + m: dict + +def call(): + return True + +@router.event("test") +async def hello(m: Incoming, d = Depends(call)) -> dict: + return { "response": "Hello, world!"} + +app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/integrations/fastapi_plugin_rabbit.py b/docs/docs_src/integrations/fastapi_plugin_rabbit.py index 781c0f97..c6b5ef52 100644 --- a/docs/docs_src/integrations/fastapi_plugin_rabbit.py +++ b/docs/docs_src/integrations/fastapi_plugin_rabbit.py @@ -14,10 +14,10 @@ def call(): @router.event("test") async def hello(m: Incoming, d = Depends(call)) -> dict: - return { "response": "Hello, world!" } + return { "response": "Hello, Rabbit!" } @router.get("/") async def hello_http(): - return "Hello, http!" + return "Hello, HTTP!" app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/integrations/fastapi_plugin_rabbit_depends.py b/docs/docs_src/integrations/fastapi_plugin_rabbit_depends.py index 3d956927..6c9bb701 100644 --- a/docs/docs_src/integrations/fastapi_plugin_rabbit_depends.py +++ b/docs/docs_src/integrations/fastapi_plugin_rabbit_depends.py @@ -13,6 +13,6 @@ def broker(): @router.get("/") async def hello_http(broker: Annotated[RabbitBroker, Depends(broker)]): await broker.publish("Hello, Rabbit!", routing_key="test") - return "Hello, http!" + return "Hello, HTTP!" app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/integrations/fastapi_plugin_rabbit_send.py b/docs/docs_src/integrations/fastapi_plugin_rabbit_send.py index fa03da0f..4406a308 100644 --- a/docs/docs_src/integrations/fastapi_plugin_rabbit_send.py +++ b/docs/docs_src/integrations/fastapi_plugin_rabbit_send.py @@ -8,6 +8,6 @@ @router.get("/") async def hello_http(): await router.broker.publish("Hello, Rabbit!", routing_key="test") - return "Hello, http!" + return "Hello, HTTP!" app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/integrations/fastapi_plugin_redis.py b/docs/docs_src/integrations/fastapi_plugin_redis.py new file mode 100644 index 00000000..595ed457 --- /dev/null +++ b/docs/docs_src/integrations/fastapi_plugin_redis.py @@ -0,0 +1,23 @@ +from fastapi import Depends, FastAPI +from pydantic import BaseModel +from propan.fastapi import RedisRouter + +app = FastAPI() + +router = RedisRouter("redis://localhost:6379") + +class Incoming(BaseModel): + m: dict + +def call(): + return True + +@router.event("test") +async def hello(m: Incoming, d = Depends(call)) -> dict: + return { "response": "Hello, Redis!" } + +@router.get("/") +async def hello_http(): + return "Hello, HTTP!" + +app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/integrations/fastapi_plugin_redis_depends.py b/docs/docs_src/integrations/fastapi_plugin_redis_depends.py new file mode 100644 index 00000000..3d52a5cf --- /dev/null +++ b/docs/docs_src/integrations/fastapi_plugin_redis_depends.py @@ -0,0 +1,18 @@ +from fastapi import FastAPI, Depends +from propan import RedisBroker +from propan.fastapi import RedisRouter +from typing_extensions import Annotated + +app = FastAPI() + +router = RedisRouter("redis://localhost:6379") + +def broker(): + return router.broker + +@router.get("/") +async def hello_http(broker: Annotated[RedisBroker, Depends(broker)]): + await broker.publish("Hello, Redis!", routing_key="test") + return "Hello, HTTP!" + +app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/integrations/fastapi_plugin_redis_send.py b/docs/docs_src/integrations/fastapi_plugin_redis_send.py new file mode 100644 index 00000000..e03b8ef4 --- /dev/null +++ b/docs/docs_src/integrations/fastapi_plugin_redis_send.py @@ -0,0 +1,13 @@ +from fastapi import FastAPI +from propan.fastapi import RedisRouter + +app = FastAPI() + +router = RedisRouter("redis://localhost:6379") + +@router.get("/") +async def hello_http(): + await router.broker.publish("Hello, Redis!", routing_key="test") + return "Hello, HTTP!" + +app.include_router(router) \ No newline at end of file diff --git a/docs/docs_src/quickstart/app/1_broker_redis.py b/docs/docs_src/quickstart/app/1_broker_redis.py new file mode 100644 index 00000000..3a522e24 --- /dev/null +++ b/docs/docs_src/quickstart/app/1_broker_redis.py @@ -0,0 +1,4 @@ +from propan import PropanApp, RedisBroker + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(broker) \ No newline at end of file diff --git a/docs/docs_src/quickstart/app/2_set_broker_redis.py b/docs/docs_src/quickstart/app/2_set_broker_redis.py new file mode 100644 index 00000000..0d0395f2 --- /dev/null +++ b/docs/docs_src/quickstart/app/2_set_broker_redis.py @@ -0,0 +1,8 @@ +from propan import PropanApp, RedisBroker + +app = PropanApp() + +@app.on_startup +def init_broker(): + broker = RedisBroker("redis://localhost:6379") + app.set_broker(broker) \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/publishing/1_nats_inside_propan.py b/docs/docs_src/quickstart/broker/publishing/1_nats_inside_propan.py index 6757e843..96067ae7 100644 --- a/docs/docs_src/quickstart/broker/publishing/1_nats_inside_propan.py +++ b/docs/docs_src/quickstart/broker/publishing/1_nats_inside_propan.py @@ -5,4 +5,4 @@ @broker.handle("test") async def handle(m: str): - await broker.publish(m, "another-queue") \ No newline at end of file + await broker.publish(m, "another-subject") \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/publishing/1_redis_inside_propan.py b/docs/docs_src/quickstart/broker/publishing/1_redis_inside_propan.py new file mode 100644 index 00000000..1b43d374 --- /dev/null +++ b/docs/docs_src/quickstart/broker/publishing/1_redis_inside_propan.py @@ -0,0 +1,8 @@ +from propan import PropanApp, RedisBroker + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(broker) + +@broker.handle("test") +async def handle(m: str): + await broker.publish(m, "another-channel") \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/publishing/2_nats_context.py b/docs/docs_src/quickstart/broker/publishing/2_nats_context.py index 62a04b7d..787fe579 100644 --- a/docs/docs_src/quickstart/broker/publishing/2_nats_context.py +++ b/docs/docs_src/quickstart/broker/publishing/2_nats_context.py @@ -1,2 +1,2 @@ async with NatsBroker("nats://localhost:4222") as broker: - await broker.publish(m, "another-queue") \ No newline at end of file + await broker.publish(m, "another-subject") \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/publishing/2_redis_context.py b/docs/docs_src/quickstart/broker/publishing/2_redis_context.py new file mode 100644 index 00000000..bff97c60 --- /dev/null +++ b/docs/docs_src/quickstart/broker/publishing/2_redis_context.py @@ -0,0 +1,2 @@ +async with RedisBroker("redis://localhost:6379") as broker: + await broker.publish(m, "another-channel") \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/rpc/1_redis_handler.py b/docs/docs_src/quickstart/broker/rpc/1_redis_handler.py new file mode 100644 index 00000000..00bd7bd4 --- /dev/null +++ b/docs/docs_src/quickstart/broker/rpc/1_redis_handler.py @@ -0,0 +1,7 @@ +from propan import RedisBroker + +broker = RedisBroker("redis://localhost:6379") + +@broker.handle("ping") +async def ping(m: str): + return "pong!" # <-- send RPC response diff --git a/docs/docs_src/quickstart/broker/rpc/2_redis_blocking_client.py b/docs/docs_src/quickstart/broker/rpc/2_redis_blocking_client.py new file mode 100644 index 00000000..8adb5a9e --- /dev/null +++ b/docs/docs_src/quickstart/broker/rpc/2_redis_blocking_client.py @@ -0,0 +1,11 @@ +from propan import RedisBroker + +async def main(): + async with RedisBroker("redis://localhost:6379") as broker: + r = await broker.publish( + "hi!", + channel="ping", + callback=True + ) + + assert r == "pong" # <-- take the RPC response \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/rpc/3_redis_blocking_client_timeout.py b/docs/docs_src/quickstart/broker/rpc/3_redis_blocking_client_timeout.py new file mode 100644 index 00000000..93a7d638 --- /dev/null +++ b/docs/docs_src/quickstart/broker/rpc/3_redis_blocking_client_timeout.py @@ -0,0 +1,6 @@ +await broker.publish( + "hi!", + channel="ping", + callback=True, + callback_timeout=3.0 # (1) +) \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/rpc/4_redis_blocking_client_timeout_none.py b/docs/docs_src/quickstart/broker/rpc/4_redis_blocking_client_timeout_none.py new file mode 100644 index 00000000..fda2daab --- /dev/null +++ b/docs/docs_src/quickstart/broker/rpc/4_redis_blocking_client_timeout_none.py @@ -0,0 +1,6 @@ +await broker.publish( + "hi!", + channel="ping", + callback=True, + callback_timeout=None +) \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/rpc/5_redis_blocking_client_timeout_error.py b/docs/docs_src/quickstart/broker/rpc/5_redis_blocking_client_timeout_error.py new file mode 100644 index 00000000..85d777ab --- /dev/null +++ b/docs/docs_src/quickstart/broker/rpc/5_redis_blocking_client_timeout_error.py @@ -0,0 +1,6 @@ +await broker.publish( + "hi!", + channel="ping", + callback=True, + raise_timeout=True +) \ No newline at end of file diff --git a/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_rabbit.py b/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_rabbit.py index 714f8e43..92f9bd15 100644 --- a/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_rabbit.py +++ b/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_rabbit.py @@ -1,5 +1,5 @@ import asyncio -from propan.brokers.rabbit import RabbitBroker +from propan import RabbitBroker broker = RabbitBroker("amqp://guest:guest@127.0.0.1/") diff --git a/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_redis.py b/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_redis.py new file mode 100644 index 00000000..e29849ee --- /dev/null +++ b/docs/docs_src/quickstart/broker/rpc/6_noblocking_client_redis.py @@ -0,0 +1,24 @@ +import asyncio +from propan import RedisBroker + +broker = RedisBroker("redis://localhost:6379") + +@broker.handle("reply") +async def get_message(m: str): + assert m == "pong!" # <-- take the RPC response + +async def main(): + await broker.start() + + await broker.publish( + "hello", + channel="test", + reply_to="reply" + ) + + try: + await asyncio.Future() + finally: + await broker.close() + +asyncio.run(main()) \ No newline at end of file diff --git a/docs/docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py b/docs/docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py new file mode 100644 index 00000000..daac0bac --- /dev/null +++ b/docs/docs_src/quickstart/dependencies/basic/1_propan_redis_depends.py @@ -0,0 +1,11 @@ +from propan import PropanApp, RedisBroker, Depends + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(broker) + +def simple_dependency(): + return 1 + +@broker.handle("test") +async def handler(body: dict, d: int = Depends(simple_dependency)): + assert d == 1 \ No newline at end of file diff --git a/docs/docs_src/quickstart/dependencies/basic/2_propan_redis_depends.py b/docs/docs_src/quickstart/dependencies/basic/2_propan_redis_depends.py new file mode 100644 index 00000000..5b7999cc --- /dev/null +++ b/docs/docs_src/quickstart/dependencies/basic/2_propan_redis_depends.py @@ -0,0 +1,17 @@ +from propan import PropanApp, RedisBroker, Depends + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(broker) + +def another_dependency(): + return 1 + +def simple_dependency(b: int = Depends(another_dependency)): # (1) + return b * 2 + +@broker.handle("test") +async def handler( + body: dict, + a: int = Depends(another_dependency), + b: int = Depends(simple_dependency)): + assert (a + b) == 3 \ No newline at end of file diff --git a/docs/docs_src/quickstart/lifespan/1_redis.py b/docs/docs_src/quickstart/lifespan/1_redis.py new file mode 100644 index 00000000..89883f4f --- /dev/null +++ b/docs/docs_src/quickstart/lifespan/1_redis.py @@ -0,0 +1,15 @@ +from propan import PropanApp, RedisBroker +from propan.annotations import ContextRepo +from pydantic import BaseSettings + +broker = RedisBroker() +app = PropanApp(broker) + +class Settings(BaseSettings): + redis_url: str + +@app.on_startup +async def setup(context: ContextRepo, env: str = ".env"): + settings = Settings(_env_file=env) + context.set_global("settings", settings) + await broker.connect(settings.redis_url) diff --git a/docs/docs_src/quickstart/lifespan/2_ml_nats.py b/docs/docs_src/quickstart/lifespan/2_ml_nats.py index d6fe6c00..ac58272a 100644 --- a/docs/docs_src/quickstart/lifespan/2_ml_nats.py +++ b/docs/docs_src/quickstart/lifespan/2_ml_nats.py @@ -20,7 +20,7 @@ async def shutdown_model(model: dict = Context()): # Clean up the ML models and release the resources model.clear() -@app.get("/test") +@broker.handle("test") async def predict(x: float, model = Context()): result = model["answer_to_everything"](x) return {"result": result} \ No newline at end of file diff --git a/docs/docs_src/quickstart/lifespan/2_ml_rabbit.py b/docs/docs_src/quickstart/lifespan/2_ml_rabbit.py index 60074a93..0bf2fb07 100644 --- a/docs/docs_src/quickstart/lifespan/2_ml_rabbit.py +++ b/docs/docs_src/quickstart/lifespan/2_ml_rabbit.py @@ -20,7 +20,7 @@ async def shutdown_model(model: dict = Context()): # Clean up the ML models and release the resources model.clear() -@broker.handle("/test") +@broker.handle("test") async def predict(x: float, model = Context()): result = model["answer_to_everything"](x) return {"result": result} \ No newline at end of file diff --git a/docs/docs_src/quickstart/lifespan/2_ml_redis.py b/docs/docs_src/quickstart/lifespan/2_ml_redis.py new file mode 100644 index 00000000..800373cd --- /dev/null +++ b/docs/docs_src/quickstart/lifespan/2_ml_redis.py @@ -0,0 +1,26 @@ +from propan import PropanApp, Context, RedisBroker +from propan.annotations import ContextRepo + +broker = RedisBroker("redis://localhost:6379") +app = PropanApp(broker) + +ml_models = {} # fake ML model + +def fake_answer_to_everything_ml_model(x: float): + return x * 42 + +@app.on_startup +async def setup_model(context: ContextRepo): + # Load the ML model + ml_models["answer_to_everything"] = fake_answer_to_everything_ml_model + context.set_global("model", ml_models) + +@app.on_shutdown +async def shutdown_model(model: dict = Context()): + # Clean up the ML models and release the resources + model.clear() + +@broker.handle("test") +async def predict(x: float, model = Context()): + result = model["answer_to_everything"](x) + return {"result": result} \ No newline at end of file diff --git a/docs/docs_src/rabbit/testing/1_main.py b/docs/docs_src/quickstart/testing/1_main_rabbit.py similarity index 100% rename from docs/docs_src/rabbit/testing/1_main.py rename to docs/docs_src/quickstart/testing/1_main_rabbit.py diff --git a/docs/docs_src/quickstart/testing/1_main_redis.py b/docs/docs_src/quickstart/testing/1_main_redis.py new file mode 100644 index 00000000..1ae5c98d --- /dev/null +++ b/docs/docs_src/quickstart/testing/1_main_redis.py @@ -0,0 +1,12 @@ +from propan import PropanApp, RedisBroker + +broker = RedisBroker() + +@broker.handler("ping") +async def healthcheck(msg: str) -> str: + if msg == "ping": + return "pong" + else: + return "wrong" + +app = PropanApp(broker) \ No newline at end of file diff --git a/docs/docs_src/rabbit/testing/2_test.py b/docs/docs_src/quickstart/testing/2_test_rabbit.py similarity index 100% rename from docs/docs_src/rabbit/testing/2_test.py rename to docs/docs_src/quickstart/testing/2_test_rabbit.py diff --git a/docs/docs_src/quickstart/testing/2_test_redis.py b/docs/docs_src/quickstart/testing/2_test_redis.py new file mode 100644 index 00000000..a9353fd7 --- /dev/null +++ b/docs/docs_src/quickstart/testing/2_test_redis.py @@ -0,0 +1,9 @@ +from propan.test import TestRedisBroker + +from main import broker + +def test_publish(): + async with TestRedisBroker(broker) as test_broker: + await test_broker.start() + r = await test_broker.publish("ping", channel="ping", callback=True) + assert r == "pong" \ No newline at end of file diff --git a/docs/docs_src/rabbit/testing/3_conftest.py b/docs/docs_src/quickstart/testing/3_conftest_rabbit.py similarity index 100% rename from docs/docs_src/rabbit/testing/3_conftest.py rename to docs/docs_src/quickstart/testing/3_conftest_rabbit.py diff --git a/docs/docs_src/quickstart/testing/3_conftest_redis.py b/docs/docs_src/quickstart/testing/3_conftest_redis.py new file mode 100644 index 00000000..b309201b --- /dev/null +++ b/docs/docs_src/quickstart/testing/3_conftest_redis.py @@ -0,0 +1,14 @@ +import pytest +from propan.test import TestRedisBroker + +from main import broker + +@pytest.fixture() +def test_broker(): + async with TestRedisBroker(broker) as b: + await b.start() + yield b + +def test_publish(test_broker): + r = await test_broker.publish("ping", queue="ping", callback=True) + assert r == "pong" \ No newline at end of file diff --git a/docs/docs_src/rabbit/testing/4_suppressed_exc.py b/docs/docs_src/quickstart/testing/4_suppressed_exc_rabbit.py similarity index 100% rename from docs/docs_src/rabbit/testing/4_suppressed_exc.py rename to docs/docs_src/quickstart/testing/4_suppressed_exc_rabbit.py diff --git a/docs/docs_src/quickstart/testing/4_suppressed_exc_redis.py b/docs/docs_src/quickstart/testing/4_suppressed_exc_redis.py new file mode 100644 index 00000000..68a43e06 --- /dev/null +++ b/docs/docs_src/quickstart/testing/4_suppressed_exc_redis.py @@ -0,0 +1,6 @@ +def test_publish(test_broker): + r = await test_broker.publish( + {"msg": "ping"}, channel="ping", + callback=True, callback_timeout=1 + ) + assert r == None \ No newline at end of file diff --git a/docs/docs_src/rabbit/testing/5_build_message.py b/docs/docs_src/quickstart/testing/5_build_message_rabbit.py similarity index 100% rename from docs/docs_src/rabbit/testing/5_build_message.py rename to docs/docs_src/quickstart/testing/5_build_message_rabbit.py diff --git a/docs/docs_src/quickstart/testing/5_build_message_redis.py b/docs/docs_src/quickstart/testing/5_build_message_redis.py new file mode 100644 index 00000000..a45b0b83 --- /dev/null +++ b/docs/docs_src/quickstart/testing/5_build_message_redis.py @@ -0,0 +1,7 @@ +from propan.test.redis import build_message + +from main import healthcheck + +def test_publish(test_broker): + msg = build_message("ping", queue="ping") + assert (await healthcheck(msg)) == "pong" \ No newline at end of file diff --git a/docs/docs_src/rabbit/testing/6_reraise.py b/docs/docs_src/quickstart/testing/6_reraise_rabbit.py similarity index 100% rename from docs/docs_src/rabbit/testing/6_reraise.py rename to docs/docs_src/quickstart/testing/6_reraise_rabbit.py diff --git a/docs/docs_src/quickstart/testing/6_reraise_redis.py b/docs/docs_src/quickstart/testing/6_reraise_redis.py new file mode 100644 index 00000000..d801522e --- /dev/null +++ b/docs/docs_src/quickstart/testing/6_reraise_redis.py @@ -0,0 +1,11 @@ + +import pytest +import pydantic +from propan.test.rabbit import build_message + +from main import healthcheck + +def test_publish(test_broker): + msg = build_message({ "msg": "ping" }, channel="ping") + with pytest.raises(pydantic.ValidationError): + await healthcheck(msg, reraise_exc=True) \ No newline at end of file diff --git a/docs/docs_src/rabbit/examples/direct.py b/docs/docs_src/rabbit/direct.py similarity index 90% rename from docs/docs_src/rabbit/examples/direct.py rename to docs/docs_src/rabbit/direct.py index c386d2ed..cc71d658 100644 --- a/docs/docs_src/rabbit/examples/direct.py +++ b/docs/docs_src/rabbit/direct.py @@ -5,7 +5,7 @@ broker = RabbitBroker() app = PropanApp(broker) -exch = RabbitExchange("exch", auto_delete=True) +exch = RabbitExchange("exchange", auto_delete=True) queue_1 = RabbitQueue("test-q-1", auto_delete=True) queue_2 = RabbitQueue("test-q-2", auto_delete=True) @@ -22,10 +22,8 @@ async def base_handler2(logger: Logger): async def base_handler3(logger: Logger): logger.info("base_handler3") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(queue="test-q-1", exchange=exch) # handlers: 1 await broker.publish(queue="test-q-1", exchange=exch) # handlers: 2 await broker.publish(queue="test-q-1", exchange=exch) # handlers: 1 diff --git a/docs/docs_src/rabbit/examples/fanout.py b/docs/docs_src/rabbit/fanout.py similarity index 82% rename from docs/docs_src/rabbit/examples/fanout.py rename to docs/docs_src/rabbit/fanout.py index dc3c61e9..7b8516fe 100644 --- a/docs/docs_src/rabbit/examples/fanout.py +++ b/docs/docs_src/rabbit/fanout.py @@ -5,7 +5,7 @@ broker = RabbitBroker() app = PropanApp(broker) -exch = RabbitExchange("exch", auto_delete=True, type=ExchangeType.FANOUT) +exch = RabbitExchange("exchange", auto_delete=True, type=ExchangeType.FANOUT) queue_1 = RabbitQueue("test-q-1", auto_delete=True) queue_2 = RabbitQueue("test-q-2", auto_delete=True) @@ -22,10 +22,8 @@ async def base_handler2(logger: Logger): async def base_handler3(logger: Logger): logger.info("base_handler3") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(exchange=exch) # handlers: 1, 3 await broker.publish(exchange=exch) # handlers: 2, 3 await broker.publish(exchange=exch) # handlers: 1, 3 diff --git a/docs/docs_src/rabbit/examples/header.py b/docs/docs_src/rabbit/header.py similarity index 82% rename from docs/docs_src/rabbit/examples/header.py rename to docs/docs_src/rabbit/header.py index 0df49ff7..beb7c4fd 100644 --- a/docs/docs_src/rabbit/examples/header.py +++ b/docs/docs_src/rabbit/header.py @@ -5,18 +5,18 @@ broker = RabbitBroker() app = PropanApp(broker) -exch = RabbitExchange("exch", auto_delete=True, type=ExchangeType.HEADERS) +exch = RabbitExchange("exchange", auto_delete=True, type=ExchangeType.HEADERS) queue_1 = RabbitQueue( - "test-q-1", auto_delete=True, + "test-queue-1", auto_delete=True, bind_arguments={ "key": 1 } ) queue_2 = RabbitQueue( - "test-q-2", auto_delete=True, + "test-queue-2", auto_delete=True, bind_arguments={ "key": 2, "key2": 2, "x-match": "any" } ) queue_3 = RabbitQueue( - "test-q-3", auto_delete=True, + "test-queue-3", auto_delete=True, bind_arguments={ "key": 2, "key2": 2, "x-match": "all" } ) @@ -36,10 +36,8 @@ async def base_handler3(logger: Logger): async def base_handler4(logger: Logger): logger.info("base_handler4") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(exchange=exch, headers={ "key": 1 }) # handlers: 1 await broker.publish(exchange=exch, headers={ "key": 1 }) # handlers: 2 await broker.publish(exchange=exch, headers={ "key": 1 }) # handlers: 1 diff --git a/docs/docs_src/rabbit/examples/topic.py b/docs/docs_src/rabbit/topic.py similarity index 75% rename from docs/docs_src/rabbit/examples/topic.py rename to docs/docs_src/rabbit/topic.py index 9b29f975..7a205c5e 100644 --- a/docs/docs_src/rabbit/examples/topic.py +++ b/docs/docs_src/rabbit/topic.py @@ -5,10 +5,10 @@ broker = RabbitBroker() app = PropanApp(broker) -exch = RabbitExchange("exch", auto_delete=True, type=ExchangeType.TOPIC) +exch = RabbitExchange("exchange", auto_delete=True, type=ExchangeType.TOPIC) -queue_1 = RabbitQueue("test-q-1", auto_delete=True, routing_key="*.info") -queue_2 = RabbitQueue("test-q-2", auto_delete=True, routing_key="*.debug") +queue_1 = RabbitQueue("test-queue-1", auto_delete=True, routing_key="*.info") +queue_2 = RabbitQueue("test-queue-2", auto_delete=True, routing_key="*.debug") @broker.handle(queue_1, exch) async def base_handler1(logger: Logger): @@ -22,10 +22,8 @@ async def base_handler2(logger: Logger): async def base_handler3(logger: Logger): logger.info("base_handler3") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(routing_key="logs.info", exchange=exch) # handlers: 1 await broker.publish(routing_key="logs.info", exchange=exch) # handlers: 2 await broker.publish(routing_key="logs.info", exchange=exch) # handlers: 1 diff --git a/docs/docs_src/redis/direct.py b/docs/docs_src/redis/direct.py new file mode 100644 index 00000000..015ead38 --- /dev/null +++ b/docs/docs_src/redis/direct.py @@ -0,0 +1,26 @@ +from propan import PropanApp, RedisBroker +from propan.annotations import Logger + +broker = RedisBroker() +app = PropanApp(broker) + + +@broker.handle("test") +async def handler1(logger: Logger): + logger.info("handler1") + + +@broker.handle("test") +async def handler2(logger: Logger): + logger.info("handler2") + + +@broker.handle("test2") +async def handler3(logger: Logger): + logger.info("handler3") + + +@app.after_startup +async def publish_smth(): + await broker.publish("", "test") # handlers: 1, 2 + await broker.publish("", "test2") # handlers: 3 diff --git a/docs/docs_src/redis/pattern.py b/docs/docs_src/redis/pattern.py new file mode 100644 index 00000000..66e21e79 --- /dev/null +++ b/docs/docs_src/redis/pattern.py @@ -0,0 +1,26 @@ +from propan import PropanApp, RedisBroker +from propan.annotations import Logger + +broker = RedisBroker() +app = PropanApp(broker) + + +@broker.handle("*.info", pattern=True) +async def handler1(b: str, logger: Logger): + logger.info("handler1") + + +@broker.handle("*.info", pattern=True) +async def handler2(b: str, logger: Logger): + logger.info("handler2") + + +@broker.handle("*.error", pattern=True) +async def handler3(logger: Logger): + logger.info("handler3") + + +@app.after_startup +async def publish_smth(): + await broker.publish("", "logs.info") # handlers: 1, 2 + await broker.publish("", "logs.error") # handlers: 3 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index bf1f9edc..82fe268f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -103,42 +103,48 @@ extra: nav: - Welcome: index.md - Getting Started: - - Hello, Propan!: 2_getting_started/1_quick-start.md - - CLI Tool: 2_getting_started/2_cli.md - - Application: 2_getting_started/3_app.md + - Hello, Propan!: getting_started/1_quick-start.md + - CLI Tool: getting_started/2_cli.md + - Application: getting_started/3_app.md - Brokers: - - Basics: 2_getting_started/4_broker/1_index.md - - Routing and Errors: 2_getting_started/4_broker/2_routing.md - - Incoming Messages: 2_getting_started/4_broker/3_type-casting.md - - Publish Messages: 2_getting_started/4_broker/4_publishing.md - - RPC over MQ: 2_getting_started/4_broker/5_rpc.md + - Basics: getting_started/4_broker/1_index.md + - Routing and Errors: getting_started/4_broker/2_routing.md + - Incoming Messages: getting_started/4_broker/3_type-casting.md + - Publish Messages: getting_started/4_broker/4_publishing.md + - RPC over MQ: getting_started/4_broker/5_rpc.md - Dependency Injection: - - Depends: 2_getting_started/5_dependency/1_di-index.md - - Context: 2_getting_started/5_dependency/2_context.md - - Lifespan: 2_getting_started/6_lifespans.md - - Testing: 2_getting_started/7_testing.md - - Logging: 2_getting_started/8_logging.md + - Depends: getting_started/5_dependency/1_di-index.md + - Context: getting_started/5_dependency/2_context.md + - Lifespan: getting_started/6_lifespans.md + - Testing: getting_started/7_testing.md + - Logging: getting_started/8_logging.md - RabbitMQ: - - Routing: 3_rabbit/1_routing.md - - Exchanges: 3_rabbit/2_exchanges.md - - Queues and Bindings: 3_rabbit/3_queues.md - - Rabbit Pubslishing: 3_rabbit/4_publishing.md + - Routing: rabbit/1_routing.md + - Exchanges: rabbit/2_exchanges.md + - Queues and Bindings: rabbit/3_queues.md + - Rabbit Pubslishing: rabbit/4_publishing.md - Examples: - - Direct Exchange: 3_rabbit/5_examples/1_direct.md - - Fanout Exchange: 3_rabbit/5_examples/2_fanout.md - - Topic Exchange: 3_rabbit/5_examples/3_topic.md - - Header Exchange: 3_rabbit/5_examples/4_header.md - - NATS: - - NATS Basics: 4_nats/1_nats-index.md - - Routing: 4_nats/2_routing.md - - Nats Pubslishing: 4_nats/3_publishing.md - - Integrations: 5_integrations/1_integrations-index.md - - FastAPI Plugin: 5_integrations/2_fastapi-plugin.md + - Direct Exchange: rabbit/5_examples/1_direct.md + - Fanout Exchange: rabbit/5_examples/2_fanout.md + - Topic Exchange: rabbit/5_examples/3_topic.md + - Header Exchange: rabbit/5_examples/4_header.md + - Redis Pub/Sub: + - Redis Basics: redis/1_redis-index.md + - Redis Pubslishing: redis/2_publishing.md + - Examples: + - Direct: redis/3_examples/1_direct.md + - Pattern: redis/3_examples/2_pattern.md + - Nats: + - Nats Basics: nats/1_nats-index.md + - Routing: nats/2_routing.md + - Nats Pubslishing: nats/3_publishing.md + - Integrations: integrations/1_integrations-index.md + - FastAPI Plugin: integrations/2_fastapi-plugin.md - Contributing: - - TODOS: 6_contributing/1_todo.md - - Environment: 6_contributing/2_contributing-index.md - - Documentation: 6_contributing/3_docs.md - - Adapters: 6_contributing/4_adapters.md - - Alternatives: 7_alternatives.md - - Help Propan: 8_help.md - - Release Notes: 9_CHANGELOG.md + - TODOS: contributing/1_todo.md + - Environment: contributing/2_contributing-index.md + - Documentation: contributing/3_docs.md + - Adapters: contributing/4_adapters.md + - Alternatives: alternatives.md + - Help Propan: help.md + - Release Notes: CHANGELOG.md diff --git a/examples/rabbit/direct.py b/examples/rabbit/direct.py index 506a75e6..89e44a51 100644 --- a/examples/rabbit/direct.py +++ b/examples/rabbit/direct.py @@ -25,10 +25,8 @@ async def base_handler3(logger: Logger): logger.info("base_handler3") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(queue=queue_1, exchange=direct_exchange) await broker.publish(queue=queue_1, exchange=direct_exchange) await broker.publish(queue=queue_1, exchange=direct_exchange) diff --git a/examples/rabbit/fanout.py b/examples/rabbit/fanout.py index f42463e3..bee51771 100644 --- a/examples/rabbit/fanout.py +++ b/examples/rabbit/fanout.py @@ -25,10 +25,8 @@ async def base_handler3(logger: Logger): logger.info("base_handler3") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(exchange=fanout_exchange) await broker.publish(exchange=fanout_exchange) await broker.publish(exchange=fanout_exchange) diff --git a/examples/rabbit/header.py b/examples/rabbit/header.py index 04f517b6..dec1d8b2 100644 --- a/examples/rabbit/header.py +++ b/examples/rabbit/header.py @@ -42,10 +42,8 @@ async def base_handler4(logger: Logger): logger.info("base_handler4") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(exchange=header_exchange, headers={"key": 1}) await broker.publish(exchange=header_exchange, headers={"key": 1}) await broker.publish(exchange=header_exchange, headers={"key": 1}) diff --git a/examples/rabbit/topic.py b/examples/rabbit/topic.py index 683fa5f0..868ddc6f 100644 --- a/examples/rabbit/topic.py +++ b/examples/rabbit/topic.py @@ -25,10 +25,8 @@ async def base_handler3(logger: Logger): logger.info("base_handler3") -@app.on_startup +@app.after_startup async def send_messages(): - await broker.start() - await broker.publish(routing_key="logs.info", exchange=topic_exchange) await broker.publish(routing_key="logs.info", exchange=topic_exchange) await broker.publish(routing_key="logs.info", exchange=topic_exchange) diff --git a/examples/redis/direct.py b/examples/redis/direct.py new file mode 100644 index 00000000..41e2fba9 --- /dev/null +++ b/examples/redis/direct.py @@ -0,0 +1,26 @@ +from propan import PropanApp, RedisBroker +from propan.annotations import Logger + +broker = RedisBroker() +app = PropanApp(broker) + + +@broker.handle("test") +async def handler1(logger: Logger): + logger.info("handler1") + + +@broker.handle("test") +async def handler2(logger: Logger): + logger.info("handler2") + + +@broker.handle("test2") +async def handler3(logger: Logger): + logger.info("handler3") + + +@app.after_startup +async def publish_smth(): + await broker.publish("", "test") + await broker.publish("", "test2") diff --git a/examples/redis/pattern.py b/examples/redis/pattern.py new file mode 100644 index 00000000..45e9f3bd --- /dev/null +++ b/examples/redis/pattern.py @@ -0,0 +1,26 @@ +from propan import PropanApp, RedisBroker +from propan.annotations import Logger + +broker = RedisBroker() +app = PropanApp(broker) + + +@broker.handle("*.info", pattern=True) +async def handler1(b: str, logger: Logger): + logger.info("handler1") + + +@broker.handle("*.info", pattern=True) +async def handler2(b: str, logger: Logger): + logger.info("handler2") + + +@broker.handle("*.error", pattern=True) +async def handler3(logger: Logger): + logger.info("handler3") + + +@app.after_startup +async def publish_smth(): + await broker.publish("", "logs.info") + await broker.publish("", "logs.error") diff --git a/propan/__about__.py b/propan/__about__.py index 521921d1..7ee2ba97 100644 --- a/propan/__about__.py +++ b/propan/__about__.py @@ -1,3 +1,3 @@ """Simple and fast framework to create message brokers based microservices""" -__version__ = "0.1.0.0" +__version__ = "0.1.1.0" diff --git a/propan/__init__.py b/propan/__init__.py index 4ee431c8..3256c45f 100644 --- a/propan/__init__.py +++ b/propan/__init__.py @@ -13,11 +13,18 @@ except Exception: NatsBroker = None # type: ignore -assert any((RabbitBroker, NatsBroker)), ( +try: + from propan.brokers.redis import RedisBroker +except Exception as e: + print(e) + RedisBroker = None # type: ignore + +assert any((RabbitBroker, NatsBroker, RedisBroker)), ( "You should specify using broker!\n" "Install it using one of the following commands:\n" 'pip install "propan[async-rabbit]"\n' 'pip install "propan[async-nats]"\n' + 'pip install "propan[async-redis]"\n' ) @@ -38,4 +45,5 @@ # brokers "NatsBroker", "RabbitBroker", + "RedisBroker", ) diff --git a/propan/annotations.py b/propan/annotations.py index 2d8dc51f..87845694 100644 --- a/propan/annotations.py +++ b/propan/annotations.py @@ -31,14 +31,23 @@ except Exception: NatsBroker = NatsMessage = None # type: ignore +try: + from propan.brokers.redis import RedisBroker as RedB + + RedisBroker = Annotated[RedB, ContextField("broker")] +except Exception: + RedisBroker = None # type: ignore + assert any( ( all((RabbitBroker, RabbitMessage)), all((NatsBroker, NatsMessage)), + RedisBroker, ) ), ( "You should specify using broker!\n" "Install it using one of the following commands:\n" 'pip install "propan[async-rabbit]"\n' 'pip install "propan[async-nats]"\n' + 'pip install "propan[async-redis]"\n' ) diff --git a/propan/brokers/model/broker_usecase.py b/propan/brokers/model/broker_usecase.py index 440b98f4..b20d31bb 100644 --- a/propan/brokers/model/broker_usecase.py +++ b/propan/brokers/model/broker_usecase.py @@ -20,9 +20,9 @@ from propan.log import access_logger from propan.types import ( AnyCallable, + AnyDict, DecodedMessage, DecoratedAsync, - DecoratedCallable, HandlerWrapper, SendableMessage, Wrapper, @@ -39,7 +39,7 @@ class BrokerUsecase(ABC): log_level: int handlers: List[Any] _connection: Any - _fmt: str + _fmt: Optional[str] def __init__( self, @@ -47,7 +47,7 @@ def __init__( apply_types: bool = True, logger: Optional[logging.Logger] = access_logger, log_level: int = logging.INFO, - log_fmt: str = "%(asctime)s %(levelname)s - %(message)s", + log_fmt: Optional[str] = "%(asctime)s %(levelname)s - %(message)s", **kwargs: Any, ) -> None: self.logger = logger @@ -134,11 +134,12 @@ def _encode_message(msg: SendableMessage) -> Tuple[bytes, Optional[ContentType]] @property def fmt(self) -> str: # pragma: no cover - return self._fmt + return self._fmt or "" async def start(self) -> None: if self.logger is not None: change_logger_handlers(self.logger, self.fmt) + await self.connect() async def __aenter__(self: Cls) -> Cls: @@ -159,11 +160,11 @@ def _wrap_handler( if self._is_apply_types is True: f = apply_types(f) + f = self._wrap_decode_message(f) + if self.logger is not None: f = self._log_execution(**broker_args)(f) - f = self._wrap_decode_message(f) - f = self._process_message(f, get_watcher(self.logger, retry)) f = self._wrap_parse_message(f) @@ -196,18 +197,18 @@ def _log_execution( self, **broker_args: Any, ) -> Wrapper: - def decor(func: AnyCallable) -> DecoratedCallable: + def decor( + func: Callable[[PropanMessage], Awaitable[T]] + ) -> Callable[[PropanMessage], Awaitable[T]]: @wraps(func) - async def wrapper(*args: Any, **kwargs: Any) -> Any: - message = context.get("message") - + async def wrapper(message: PropanMessage) -> T: log_context = self._get_log_context(message=message, **broker_args) with context.scope("log_context", log_context): self._log("Received") try: - r = await func(*args, **kwargs) + r = await func(message) except Exception as e: self._log(repr(e), logging.ERROR) raise e @@ -223,6 +224,9 @@ def _log( self, message: str, log_level: Optional[int] = None, + extra: Optional[AnyDict] = None, ) -> None: if self.logger is not None: - self.logger.log(level=(log_level or self.log_level), msg=message) + self.logger.log( + level=(log_level or self.log_level), msg=message, extra=extra + ) diff --git a/propan/brokers/model/schemas.py b/propan/brokers/model/schemas.py index 42a2122f..6f910493 100644 --- a/propan/brokers/model/schemas.py +++ b/propan/brokers/model/schemas.py @@ -1,17 +1,23 @@ import json +from dataclasses import dataclass from enum import Enum -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Sequence, Tuple, Union from uuid import uuid4 -from pydantic import BaseModel, Field -from pydantic.dataclasses import dataclass -from typing_extensions import TypeAlias +from pydantic import BaseModel, Field, Json +from pydantic.dataclasses import dataclass as pydantic_dataclass +from typing_extensions import TypeAlias, assert_never -from propan.types import AnyDict, DecodedMessage, SendableMessage +from propan.types import AnyDict, DecodedMessage, DecoratedCallable, SendableMessage ContentType: TypeAlias = str +@dataclass +class BaseHandler: + callback: DecoratedCallable + + class ContentTypes(str, Enum): text = "text/plain" json = "application/json" @@ -23,9 +29,6 @@ class NameRequired(BaseModel): def __init__(self, name: str, **kwargs: Any): super().__init__(name=name, **kwargs) - def __eq__(self, other: object) -> bool: - return isinstance(other, NameRequired) and self.name == other.name - class Queue(NameRequired): name: Optional[str] = Field(...) @@ -47,19 +50,23 @@ def to_send(cls, msg: SendableMessage) -> Tuple[bytes, Optional[ContentType]]: m = cls(message=msg).message # type: ignore - if isinstance(m, dict): - return json.dumps(m).encode(), ContentTypes.json.value - if isinstance(m, str): return m.encode(), ContentTypes.text.value - return m, None + if isinstance(m, (Dict, Sequence)): + return json.dumps(m).encode(), ContentTypes.json.value + assert_never() # pragma: no cover -@dataclass + +class RawDecoced(BaseModel): + message: Union[Json[Any], str] + + +@pydantic_dataclass class PropanMessage: body: bytes raw_message: Any - content_type: str + content_type: Optional[str] = None headers: AnyDict = Field(default_factory=dict) message_id: str = Field(default_factory=lambda: str(uuid4())) diff --git a/propan/brokers/nats/nats_broker.py b/propan/brokers/nats/nats_broker.py index e094b63b..e7cef131 100644 --- a/propan/brokers/nats/nats_broker.py +++ b/propan/brokers/nats/nats_broker.py @@ -29,7 +29,14 @@ def __init__(self, *args: Any, log_fmt: Optional[str] = None, **kwargs: AnyDict) self.__max_queue_len = 0 self.__max_subject_len = 4 - async def _connect(self, *args: Any, **kwargs: Any) -> Client: + async def _connect( + self, + *args: Any, + url: Optional[str] = None, + **kwargs: Any, + ) -> Client: + if url is not None: + kwargs["servers"] = kwargs.pop("servers", []) + [url] return await nats.connect(*args, **kwargs) def handle( @@ -71,9 +78,8 @@ async def start(self) -> None: for handler in self.handlers: func = handler.callback - if self.logger: - self._get_log_context(None, handler.subject, handler.queue) - self.logger.info(f"`{func.__name__}` waiting for messages") + c = self._get_log_context(None, handler.subject, handler.queue) + self._log(f"`{func.__name__}` waiting for messages", extra=c) sub = await self._connection.subscribe(handler.subject, cb=func) handler.subscription = sub @@ -87,7 +93,7 @@ async def publish( if self._connection is None: raise ValueError("NatsConnection not started yet") - msg, content_type = super()._encode_message(message) + msg, content_type = self._encode_message(message) return await self._connection.publish( subject, diff --git a/propan/brokers/nats/schemas.py b/propan/brokers/nats/schemas.py index 5699c482..d9d13c8c 100644 --- a/propan/brokers/nats/schemas.py +++ b/propan/brokers/nats/schemas.py @@ -5,12 +5,11 @@ from nats.js.api import DEFAULT_PREFIX from pydantic import BaseModel -from propan.types import DecoratedCallable +from propan.brokers.model.schemas import BaseHandler @dataclass -class Handler: - callback: DecoratedCallable +class Handler(BaseHandler): subject: str queue: str = "" diff --git a/propan/brokers/rabbit/rabbit_broker.py b/propan/brokers/rabbit/rabbit_broker.py index 770a624c..cbbe04f6 100644 --- a/propan/brokers/rabbit/rabbit_broker.py +++ b/propan/brokers/rabbit/rabbit_broker.py @@ -1,5 +1,4 @@ import asyncio -import logging from functools import wraps from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union from uuid import uuid4 @@ -13,7 +12,6 @@ from propan.brokers.push_back_watcher import BaseWatcher, WatcherContext from propan.brokers.rabbit.schemas import Handler, RabbitExchange, RabbitQueue from propan.types import AnyDict, DecoratedCallable, SendableMessage, Wrapper -from propan.utils.context import context as global_context TimeoutType = Optional[Union[int, float]] PikaSendableMessage = Union[aio_pika.message.Message, SendableMessage] @@ -67,7 +65,7 @@ async def _connect( self._channel = await connection.channel() if max_consumers: - self._log(f"Set max consumers to {max_consumers}", logging.INFO) + self._log(f"Set max consumers to {max_consumers}") await self._channel.set_qos(prefetch_count=int(self._max_consumers)) return connection @@ -104,10 +102,8 @@ async def start(self) -> None: func = handler.callback - if self.logger is not None: - context = self._get_log_context(None, handler.queue, handler.exchange) - global_context.set_local("log_context", context) - self.logger.info(f"`{func.__name__}` waiting for messages") + c = self._get_log_context(None, handler.queue, handler.exchange) + self._log(f"`{func.__name__}` waiting for messages", extra=c) await queue.consume(func) diff --git a/propan/brokers/rabbit/schemas.py b/propan/brokers/rabbit/schemas.py index d42ff17a..01973f98 100644 --- a/propan/brokers/rabbit/schemas.py +++ b/propan/brokers/rabbit/schemas.py @@ -1,10 +1,10 @@ +from dataclasses import dataclass from typing import Any, Dict, Optional from aio_pika.abc import ExchangeType, TimeoutType -from pydantic import BaseModel, Field +from pydantic import Field -from propan.brokers.model.schemas import NameRequired, Queue -from propan.types import DecoratedCallable +from propan.brokers.model.schemas import BaseHandler, NameRequired, Queue __all__ = ( "RabbitQueue", @@ -47,7 +47,7 @@ class RabbitExchange(NameRequired): routing_key: str = Field("", exclude=True) -class Handler(BaseModel): - callback: DecoratedCallable +@dataclass +class Handler(BaseHandler): queue: RabbitQueue exchange: Optional[RabbitExchange] = None diff --git a/propan/brokers/redis/__init__.py b/propan/brokers/redis/__init__.py new file mode 100644 index 00000000..519811df --- /dev/null +++ b/propan/brokers/redis/__init__.py @@ -0,0 +1,3 @@ +from propan.brokers.redis.redis_broker import RedisBroker + +__all__ = ("RedisBroker",) diff --git a/propan/brokers/redis/redis_broker.py b/propan/brokers/redis/redis_broker.py new file mode 100644 index 00000000..3b000ecb --- /dev/null +++ b/propan/brokers/redis/redis_broker.py @@ -0,0 +1,254 @@ +import asyncio +import logging +from functools import wraps +from typing import Any, Callable, Coroutine, Dict, List, NoReturn, Optional, TypeVar +from uuid import uuid4 + +from redis.asyncio.client import PubSub, Redis + +from propan.brokers.model import BrokerUsecase +from propan.brokers.model.schemas import PropanMessage, RawDecoced +from propan.brokers.push_back_watcher import BaseWatcher +from propan.brokers.redis.schemas import Handler, RedisMessage +from propan.types import ( + AnyCallable, + DecodedMessage, + DecoratedCallable, + SendableMessage, + Wrapper, +) + +T = TypeVar("T") + + +class RedisBroker(BrokerUsecase): + handlers: List[Handler] + _connection: Redis + __max_channel_len: int + _polling_interval: float + + def __init__( + self, + url: str = "redis://localhost:6379", + polling_interval: float = 1.0, + *, + log_fmt: Optional[str] = None, + **kwargs: Any, + ) -> None: + super().__init__(url=url, log_fmt=log_fmt, **kwargs) + self.__max_channel_len = 0 + self._polling_interval = polling_interval + + async def _connect( + self, + url: str, + **kwargs: Any, + ) -> Redis: + return Redis.from_url(url, **kwargs) + + async def connect( + self, + url: Optional[str] = None, + *args: Any, + **kwargs: Any, + ) -> Coroutine[Any, Any, Any]: + if url is not None: + kwargs["url"] = url + return await super().connect(*args, **kwargs) + + async def close(self) -> None: + for h in self.handlers: + if h.task is not None: # pragma: no branch + h.task.cancel() + + if h.subscription is not None: # pragma: no branch + await h.subscription.unsubscribe() + await h.subscription.reset() + + if self._connection is not None: # pragma: no branch + await self._connection.close() + + def _process_message( + self, + func: Callable[[PropanMessage], T], + watcher: Optional[BaseWatcher], + ) -> Callable[[PropanMessage], T]: + @wraps(func) + async def wrapper(message: PropanMessage) -> T: + r = await func(message) + + msg = message.raw_message + if isinstance(msg, RedisMessage) and msg.reply_to: + await self.publish(r or "", msg.reply_to) + + return r + + return wrapper + + def handle( + self, + channel: str = "", + *, + pattern: bool = False, + ) -> Wrapper: + self.__max_channel_len = max(self.__max_channel_len, len(channel)) + + def wrapper(func: AnyCallable) -> DecoratedCallable: + func = self._wrap_handler( + func, + channel=channel, + ) + handler = Handler(callback=func, channel=channel, pattern=pattern) + self.handlers.append(handler) + + return func + + return wrapper + + async def start(self) -> None: + await super().start() + + for handler in self.handlers: # pragma: no branch + c = self._get_log_context(None, handler.channel) + self._log(f"`{handler.callback.__name__}` waiting for messages", extra=c) + + psub = self._connection.pubsub() + if handler.pattern is True: + await psub.psubscribe(handler.channel) + else: + await psub.subscribe(handler.channel) + + handler.subscription = psub + handler.task = asyncio.create_task(self._consume(handler, psub)) + + async def publish( + self, + message: SendableMessage = "", + channel: str = "", + *, + reply_to: str = "", + headers: Optional[Dict[str, Any]] = None, + callback: bool = False, + callback_timeout: Optional[float] = 30.0, + raise_timeout: bool = False, + ) -> None: + if self._connection is None: + raise ValueError("Redis connection not established yet") + + msg, content_type = self._encode_message(message) + + if callback is True: + callback_channel = str(uuid4()) + psub = self._connection.pubsub() + response_queue = asyncio.Queue(maxsize=1) + await psub.subscribe(callback_channel) + task = asyncio.create_task(_consume_one(response_queue, psub)) + else: + callback_channel = reply_to + psub = None + response_queue = None + task = None + + await self._connection.publish( + channel, + RedisMessage( + data=msg, + headers={ + "content-type": content_type or "", + **(headers or {}), + }, + reply_to=callback_channel, + ).json(), + ) + + if psub and response_queue and task: + try: + response = await asyncio.wait_for( + response_queue.get(), callback_timeout + ) + except asyncio.TimeoutError as e: + if raise_timeout is True: + raise e + return None + else: + return await self._decode_message(await self._parse_message(response)) + finally: + await psub.unsubscribe(callback_channel) + await psub.reset() + task.cancel() + + @staticmethod + async def _parse_message(message: Any) -> PropanMessage: + data = message.get("data", b"") + + try: + obj = RedisMessage.parse_raw(data) + except Exception: + msg = PropanMessage( + body=data, + raw_message=message, + ) + else: + msg = PropanMessage( + body=obj.data, + content_type=obj.headers.get("content-type", ""), + headers=obj.headers, + raw_message=obj, + ) + + return msg + + async def _decode_message(self, message: PropanMessage) -> DecodedMessage: + if message.headers.get("content-type") is not None: + return await super()._decode_message(message) + else: + return RawDecoced(message=message.body).message + + def _get_log_context(self, message: PropanMessage, channel: str) -> Dict[str, Any]: + context = { + "channel": channel, + **super()._get_log_context(message), + } + return context + + @property + def fmt(self) -> str: + return self._fmt or ( + "%(asctime)s %(levelname)s - " + f"%(channel)-{self.__max_channel_len}s | " + "%(message_id)-10s " + "- %(message)s" + ) + + async def _consume(self, handler: Handler, psub: PubSub) -> NoReturn: + c = self._get_log_context(None, handler.channel) + + connected = True + while True: + try: + m = await psub.get_message( + ignore_subscribe_messages=True, + timeout=self._polling_interval, + ) + except Exception: + if connected is True: + self._log("Connection broken", logging.WARNING, c) + connected = False + await asyncio.sleep(5) + else: + if connected is False: + self._log("Connection established", logging.INFO, c) + connected = True + + if m: # pragma: no branch + await handler.callback(m) + finally: + await asyncio.sleep(0.01) + + +async def _consume_one(queue: asyncio.Queue, psub: PubSub) -> NoReturn: + async for m in psub.listen(): + t = m.get("type") + if t and "message" in t: # pragma: no branch + await queue.put(m) + break diff --git a/propan/brokers/redis/redis_broker.pyi b/propan/brokers/redis/redis_broker.pyi new file mode 100644 index 00000000..cb6b0363 --- /dev/null +++ b/propan/brokers/redis/redis_broker.pyi @@ -0,0 +1,108 @@ +import logging +from typing import Any, Callable, Dict, List, Mapping, Optional, Type, TypeVar, Union + +from redis.asyncio.client import Redis +from redis.asyncio.connection import BaseParser, Connection, DefaultParser, Encoder + +from propan.brokers.model import BrokerUsecase +from propan.brokers.model.schemas import PropanMessage +from propan.brokers.push_back_watcher import BaseWatcher +from propan.brokers.redis.schemas import Handler +from propan.log import access_logger +from propan.types import DecodedMessage, SendableMessage, Wrapper + +T = TypeVar("T") + +class RedisBroker(BrokerUsecase): + handlers: List[Handler] + _connection: Redis[bytes] + __max_channel_len: int + + def __init__( + self, + url: str = "redis://localhost:6379", + polling_interval: float = 1.0, + *, + connection_class: Type[Connection] = Connection, + max_connections: Optional[int] = None, + host: str = "localhost", + port: Union[str, int] = 6379, + db: Union[str, int] = 0, + password: Optional[str] = None, + socket_timeout: Optional[float] = None, + socket_connect_timeout: Optional[float] = None, + socket_keepalive: bool = False, + socket_keepalive_options: Optional[Mapping[int, Union[int, bytes]]] = None, + socket_type: int = 0, + retry_on_timeout: bool = False, + encoding: str = "utf-8", + encoding_errors: str = "strict", + decode_responses: bool = False, + parser_class: Type[BaseParser] = DefaultParser, + socket_read_size: int = 65536, + health_check_interval: float = 0, + client_name: Optional[str] = None, + username: Optional[str] = None, + encoder_class: Type[Encoder] = Encoder, + # broker kwargs + logger: Optional[logging.Logger] = access_logger, + log_level: int = logging.INFO, + log_fmt: Optional[str] = None, + apply_types: bool = True, + ) -> None: ... + async def _connect( + self, + url: str = "redis://localhost:6379", + host: str = "localhost", + port: Union[str, int] = 6379, + db: Union[str, int] = 0, + password: Optional[str] = None, + socket_timeout: Optional[float] = None, + socket_connect_timeout: Optional[float] = None, + socket_keepalive: bool = False, + socket_keepalive_options: Optional[Mapping[int, Union[int, bytes]]] = None, + socket_type: int = 0, + retry_on_timeout: bool = False, + encoding: str = "utf-8", + encoding_errors: str = "strict", + decode_responses: bool = False, + parser_class: Type[BaseParser] = DefaultParser, + socket_read_size: int = 65536, + health_check_interval: float = 0, + client_name: Optional[str] = None, + username: Optional[str] = None, + encoder_class: Type[Encoder] = Encoder, + ) -> Redis[bytes]: ... + async def close(self) -> None: ... + @staticmethod + async def _parse_message(message: Any) -> PropanMessage: ... + def _process_message( + self, + func: Callable[[PropanMessage], T], + watcher: Optional[BaseWatcher], + ) -> Callable[[PropanMessage], T]: ... + def handle( # type: ignore[override] + self, + channel: str, + *, + pattern: bool = False, + ) -> Wrapper: ... + def _get_log_context( # type: ignore[override] + self, message: PropanMessage, channel: str + ) -> Dict[str, Any]: ... + @staticmethod + async def _decode_message(message: PropanMessage) -> DecodedMessage: ... + @property + def fmt(self) -> str: ... + async def start(self) -> None: ... + async def publish( # type: ignore[override] + self, + message: SendableMessage = "", + channel: str = "", + *, + reply_to: str = "", + headers: Optional[Dict[str, Any]] = None, + callback: bool = False, + callback_timeout: Optional[float] = 30.0, + raise_timeout: bool = False, + ) -> None: ... diff --git a/propan/brokers/redis/schemas.py b/propan/brokers/redis/schemas.py new file mode 100644 index 00000000..b1b78e36 --- /dev/null +++ b/propan/brokers/redis/schemas.py @@ -0,0 +1,25 @@ +import asyncio +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field +from redis.asyncio.client import PubSub + +from propan.brokers.model.schemas import BaseHandler +from propan.types import AnyCallable + + +@dataclass +class Handler(BaseHandler): + callback: AnyCallable + channel: str + pattern: bool = False + + task: Optional["asyncio.Task[Any]"] = None + subscription: Optional[PubSub] = None + + +class RedisMessage(BaseModel): + data: bytes + headers: Dict[str, str] = Field(default_factory=dict) + reply_to: str = "" diff --git a/propan/cli/app.py b/propan/cli/app.py index 3f11db69..0059d8e4 100644 --- a/propan/cli/app.py +++ b/propan/cli/app.py @@ -9,7 +9,7 @@ from propan.cli.supervisors.utils import set_exit from propan.cli.utils.parser import SettingField from propan.log import logger -from propan.types import AnyCallable, AsyncFunc, DecoratedCallableNone +from propan.types import AnyCallable, AsyncFunc from propan.utils import apply_types, context from propan.utils.functions import to_async @@ -24,7 +24,9 @@ async def close(self) -> None: class PropanApp: _on_startup_calling: List[AsyncFunc] + _after_startup_calling: List[AsyncFunc] _on_shutdown_calling: List[AsyncFunc] + _after_shutdown_calling: List[AsyncFunc] _stop_stream: Optional[MemoryObjectSendStream[bool]] _receive_stream: Optional[MemoryObjectReceiveStream[bool]] @@ -40,7 +42,9 @@ def __init__( context.set_global("app", self) self._on_startup_calling = [] + self._after_startup_calling = [] self._on_shutdown_calling = [] + self._after_shutdown_calling = [] self._stop_stream = None self._receive_stream = None self._command_line_options: Dict[str, SettingField] = {} @@ -48,15 +52,17 @@ def __init__( def set_broker(self, broker: Runnable) -> None: self.broker = broker - def on_startup(self, func: AnyCallable) -> DecoratedCallableNone: - f: AsyncFunc = apply_types(to_async(func)) - self._on_startup_calling.append(f) - return func + def on_startup(self, func: AnyCallable) -> AnyCallable: + return _set_async_hook(self._on_startup_calling, func) - def on_shutdown(self, func: AnyCallable) -> DecoratedCallableNone: - f: AsyncFunc = apply_types(to_async(func)) - self._on_shutdown_calling.append(f) - return func + def on_shutdown(self, func: AnyCallable) -> AnyCallable: + return _set_async_hook(self._on_shutdown_calling, func) + + def after_startup(self, func: AnyCallable) -> AnyCallable: + return _set_async_hook(self._after_startup_calling, func) + + def after_shutdown(self, func: AnyCallable) -> AnyCallable: + return _set_async_hook(self._after_shutdown_calling, func) async def run(self, log_level: int = logging.INFO) -> None: self._init_async_cycle() @@ -93,16 +99,28 @@ async def _startup(self) -> None: if self.broker is not None: await self.broker.start() + for func in self._after_startup_calling: + await func() + async def _shutdown(self) -> None: + for func in self._on_shutdown_calling: + await func() + if ( self.broker is not None and getattr(self.broker, "_connection", False) is not False ): await self.broker.close() - for func in self._on_shutdown_calling: + for func in self._after_shutdown_calling: await func() async def __exit(self, flag: bool) -> None: if self._stop_stream is not None: # pragma: no branch await self._stop_stream.send(flag) + + +def _set_async_hook(hooks: List[AsyncFunc], func: AnyCallable) -> AnyCallable: + f: AsyncFunc = apply_types(to_async(func)) + hooks.append(f) + return func diff --git a/propan/cli/main.py b/propan/cli/main.py index a44356c4..c1b0b88b 100644 --- a/propan/cli/main.py +++ b/propan/cli/main.py @@ -8,12 +8,16 @@ from propan.__about__ import __version__ from propan.cli.app import PropanApp +from propan.cli.startproject import create_app from propan.cli.utils.imports import get_app_path, import_object from propan.cli.utils.logs import LogLevels, get_log_level, set_log_level from propan.cli.utils.parser import SettingField, parse_cli_args from propan.log import logger -cli = typer.Typer() +cli = typer.Typer(pretty_exceptions_short=True) +cli.add_typer( + create_app, name="create", help="Create a new Propan project at [APPNAME] directory" +) def version_callback(version: bool) -> None: @@ -37,6 +41,7 @@ def version_callback(version: bool) -> None: def main( version: Optional[bool] = typer.Option( False, + "-v", "--version", callback=version_callback, is_eager=True, @@ -48,15 +53,6 @@ def main( """ -@cli.command() -def create(appname: str) -> None: - """Create a new Propan project at [APPNAME] directory""" - from propan.cli.startproject import create - - project = create(Path.cwd() / appname, __version__) - typer.echo(f"Create Propan project template at: {project}") - - @cli.command( context_settings={"allow_extra_args": True, "ignore_unknown_options": True} ) @@ -127,13 +123,10 @@ def _run( propan_app._command_line_options = extra_options if sys.platform not in ("win32", "cygwin", "cli"): - import uvloop - - if sys.version_info >= (3, 11): - with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner: - runner.run(propan_app.run(log_level=app_level)) - return - + try: + import uvloop + except Exception: + logger.warning("You have no installed `uvloop`") else: uvloop.install() diff --git a/propan/cli/startproject.py b/propan/cli/startproject.py deleted file mode 100644 index d45349c5..00000000 --- a/propan/cli/startproject.py +++ /dev/null @@ -1,225 +0,0 @@ -from pathlib import Path -from typing import Union, cast - - -def create(project_dir: Path, version: str) -> Path: - project_dir = _create_project_dir(project_dir, version) - app_dir = _create_app_dir(project_dir / "app") - _create_config_dir(app_dir / "config") - _create_core_dir(app_dir / "core") - _create_apps_dir(app_dir / "apps") - return project_dir - - -def _create_project_dir(dirname: Path, version: str) -> Path: - project_dir = _touch_dir(dirname) - - _write_file(project_dir / "README.md") - - _write_file( - project_dir / "Dockerfile", - "FROM python:3.11.3-slim", - "", - "RUN useradd -ms /bin/bash user", - "", - "USER user", - "WORKDIR /home/user", - "", - "COPY requirements.txt requirements.txt", - "", - "RUN pip install -r requirements.txt", - "", - "COPY ./app ./app", - "", - 'ENTRYPOINT [ "python", "-m", "propan", "run", "app.serve:app" ]', - ) - - _write_file( - project_dir / "docker-compose.yaml", - 'version: "3"', - "", - "services:", - " rabbit:", - " image: rabbitmq", - " environment:", - " RABBITMQ_DEFAULT_USER: guest", - " RABBITMQ_DEFAULT_PASS: guest", - " ports:", - " - 5672:5672", - "", - " app:", - " build: .", - " environment:", - " APP_RABBIT__URL: amqp://guest:guest@rabbit:5672/", - " volumes:", - " - ./app:/home/user/app:ro", - " depends_on:", - " - rabbit", - ) - - _write_file( - project_dir / "requirements.txt", - f"propan[async_rabbit]=={version}", - "pydantic[dotenv]", - ) - - _write_file( - project_dir / ".gitignore", - "*.env*", - "__pycache__", - "env/", - "venv/", - ) - - return project_dir - - -def _create_app_dir(app: Path) -> Path: - app_dir = _touch_dir(app) - - _write_file( - app_dir / "serve.py", - "import logging", - "from typing import Optional", - "", - "from propan import PropanApp", - "from propan.utils import Context", - "from propan.brokers.rabbit import RabbitBroker", - "", - "from core import broker", - "from config import init_settings", - "", - "from apps import * # import to register handlers # NOQA", - "", - "", - "app = PropanApp(broker)", - "", - "", - "@app.on_startup", - "async def init_app(broker: RabbitBroker, env: Optional[str], context: Context):", - " settings = init_settings(env)", - ' context.set_global("settings", settings)', - "", - " logger_level = logging.DEBUG if settings.debug else logging.INFO", - " app.logger.setLevel(logger_level)", - " broker.logger.setLevel(logger_level)", - "", - " await broker.connect(url=settings.rabbit.url)", - "", - "", - 'if __name__ == "__main__":', - " app.run()", - ) - - return app_dir - - -def _create_config_dir(config: Path) -> Path: - config_dir = _touch_dir(config) - - _write_file( - config_dir / ".env", - "APP_DEBUG=True", - "APP_RABBIT__URL=amqp://guest:guest@localhost:5672/", - ) - - _write_file( - config_dir / "settings.py", - "from pathlib import Path", - "from typing import Optional", - "", - "from pydantic import BaseSettings, BaseModel, Field", - "", - "", - "CONFIG_DIR = Path(__file__).resolve().parent", - "BASE_DIR = CONFIG_DIR.parent", - "", - "", - "class RabbitSettings(BaseModel):", - " url: str = Field(...)", - "", - "", - "class Settings(BaseSettings):", - " debug: bool = Field(True)", - " rabbit: RabbitSettings = Field(default_factory=RabbitSettings)", - " base_dir: Path = BASE_DIR", - "", - " class Config:", - " env_prefix = 'APP_'", - " env_file_encoding = 'utf-8'", - " env_nested_delimiter = '__'", - "", - "", - "def init_settings(env_file: Optional[str]) -> Settings:", - ' env_file = CONFIG_DIR / (env_file or ".env")', - " if env_file.exists():", - " settings = Settings(_env_file=env_file)", - " else:", - " settings = Settings()", - " return settings", - ) - - _write_file( - config_dir / "__init__.py", - "from .settings import Settings, init_settings", - ) - - return config_dir - - -def _create_core_dir(core: Path) -> Path: - core_dir = _touch_dir(core) - - _write_file( - core_dir / "__init__.py", - "from .dependencies import broker", - ) - - _write_file( - core_dir / "dependencies.py", - "from propan.brokers.rabbit import RabbitBroker", - "", - "broker = RabbitBroker()", - ) - - return core_dir - - -def _create_apps_dir(apps: Path) -> Path: - apps_dir = _touch_dir(apps) - - _write_file( - apps_dir / "__init__.py", - "from .handlers import base_handler", - ) - - _write_file( - apps_dir / "handlers.py", - "from core import broker", - "", - "from propan.brokers.rabbit import RabbitQueue, RabbitExchange", - "", - "", - '@broker.handle(queue=RabbitQueue("test"),', - ' exchange=RabbitExchange("test"))', - "async def base_handler(body: dict, logger):", - " logger.info(body)", - ) - - return apps_dir - - -def _touch_dir(dir: Union[Path, str]) -> Path: - if isinstance(dir, str) is True: - dir = Path(dir).resolve() - - dir = cast(Path, dir) - if dir.exists() is False: - dir.mkdir() - return dir - - -def _write_file(path: Path, *content: str) -> None: - path.touch() - if content: - path.write_text("\n".join(content)) diff --git a/propan/cli/startproject/__init__.py b/propan/cli/startproject/__init__.py new file mode 100644 index 00000000..971d8c0d --- /dev/null +++ b/propan/cli/startproject/__init__.py @@ -0,0 +1,3 @@ +from propan.cli.startproject.app import create_app + +__all__ = ("create_app",) diff --git a/propan/cli/startproject/app.py b/propan/cli/startproject/app.py new file mode 100644 index 00000000..5bf2ba7d --- /dev/null +++ b/propan/cli/startproject/app.py @@ -0,0 +1,8 @@ +import typer + +from propan.cli.startproject.async_app import async_app +from propan.cli.startproject.sync_app import sync_app + +create_app = typer.Typer(pretty_exceptions_short=True) +create_app.add_typer(async_app, name="async", help="Create an asynchronous app") +create_app.add_typer(sync_app, name="sync", help="Create a synchronous app") diff --git a/propan/cli/startproject/async_app/__init__.py b/propan/cli/startproject/async_app/__init__.py new file mode 100644 index 00000000..0e1bc0ba --- /dev/null +++ b/propan/cli/startproject/async_app/__init__.py @@ -0,0 +1,3 @@ +from propan.cli.startproject.async_app.app import async_app + +__all__ = ("async_app",) diff --git a/propan/cli/startproject/async_app/app.py b/propan/cli/startproject/async_app/app.py new file mode 100644 index 00000000..83bc89b9 --- /dev/null +++ b/propan/cli/startproject/async_app/app.py @@ -0,0 +1,30 @@ +from pathlib import Path + +import typer + +from propan.cli.startproject.async_app.nats import create_nats +from propan.cli.startproject.async_app.rabbit import create_rabbit +from propan.cli.startproject.async_app.redis import create_redis + +async_app = typer.Typer(pretty_exceptions_short=True) + + +@async_app.command() +def rabbit(appname: str) -> None: + """Create an asyncronous RabbiMQ Propan project at [APPNAME] directory""" + project = create_rabbit(Path.cwd() / appname) + typer.echo(f"Create an asyncronous RabbiMQ Propan project at: {project}") + + +@async_app.command() +def redis(appname: str) -> None: + """Create an asyncronous Redis Propan project at [APPNAME] directory""" + project = create_redis(Path.cwd() / appname) + typer.echo(f"Create an asyncronous Redis Propan project at: {project}") + + +@async_app.command() +def nats(appname: str) -> None: + """Create an asyncronous Nats Propan project at [APPNAME] directory""" + project = create_nats(Path.cwd() / appname) + typer.echo(f"Create an asyncronous Nats Propan project at: {project}") diff --git a/propan/cli/startproject/async_app/core.py b/propan/cli/startproject/async_app/core.py new file mode 100644 index 00000000..44c9f0a7 --- /dev/null +++ b/propan/cli/startproject/async_app/core.py @@ -0,0 +1,37 @@ +from pathlib import Path + +from propan.cli.startproject.utils import write_file + + +def create_app_file(app_dir: Path, broker_annotation: str) -> None: + write_file( + app_dir / "serve.py", + "import logging", + "from typing import Optional", + "", + "from propan import PropanApp", + f"from propan.annotations import {broker_annotation}, ContextRepo", + "from core import broker", + "from config import init_settings", + "", + "from apps import * # import to register handlers # NOQA", + "", + "", + "app = PropanApp(broker)", + "", + "", + "@app.on_startup", + f"async def init_app(broker: {broker_annotation}, context: ContextRepo, env: Optional[str] = None):", + " settings = init_settings(env)", + ' context.set_global("settings", settings)', + "", + " logger_level = logging.DEBUG if settings.debug else logging.INFO", + " app.logger.setLevel(logger_level)", + " broker.logger.setLevel(logger_level)", + "", + " await broker.connect(url=settings.broker.url)", + "", + "", + 'if __name__ == "__main__":', + " app.run()", + ) diff --git a/propan/cli/startproject/async_app/nats.py b/propan/cli/startproject/async_app/nats.py new file mode 100644 index 00000000..1c555130 --- /dev/null +++ b/propan/cli/startproject/async_app/nats.py @@ -0,0 +1,82 @@ +from pathlib import Path + +from propan.cli.startproject.async_app.core import create_app_file +from propan.cli.startproject.core import ( + create_apps_dir, + create_config_dir, + create_core_dir, + create_env, + create_project_dir, +) +from propan.cli.startproject.utils import touch_dir, write_file + + +def create_nats(dir: Path) -> Path: + project_dir = _create_project_dir(dir) + app_dir = _create_app_dir(project_dir / "app") + _create_config_dir(app_dir / "config") + _create_core_dir(app_dir / "core") + _create_apps_dir(app_dir / "apps") + return project_dir + + +def _create_project_dir(dirname: Path) -> Path: + project_dir = create_project_dir(dirname, "propan[async-nats]") + + write_file( + project_dir / "docker-compose.yaml", + 'version: "3"', + "", + "services:", + " nats:", + " image: nats", + " ports:", + " - 4222:4222", + " - 8222:8222 # management", + "", + " app:", + " build: .", + " environment:", + " APP_BROKER__URL: nats://nats:4222/", + " volumes:", + " - ./app:/home/user/app:ro", + " depends_on:", + " - nats", + ) + + return project_dir + + +def _create_app_dir(app: Path) -> Path: + app_dir = touch_dir(app) + create_app_file(app_dir, "NatsBroker") + return app_dir + + +def _create_config_dir(config: Path) -> Path: + config_dir = create_config_dir(config) + create_env(config_dir, "nats://localhost:4222/") + return config_dir + + +def _create_core_dir(core: Path) -> Path: + core_dir = create_core_dir(core, "NatsBroker") + return core_dir + + +def _create_apps_dir(apps: Path) -> Path: + apps_dir = create_apps_dir(apps) + + write_file( + apps_dir / "handlers.py", + "from propan.annotations import Logger", + "", + "from core import broker", + "", + "", + "@broker.handle('test')", + "async def base_handler(body: dict, logger: Logger):", + " logger.info(body)", + ) + + return apps_dir diff --git a/propan/cli/startproject/async_app/rabbit.py b/propan/cli/startproject/async_app/rabbit.py new file mode 100644 index 00000000..7b657204 --- /dev/null +++ b/propan/cli/startproject/async_app/rabbit.py @@ -0,0 +1,93 @@ +from pathlib import Path + +from propan.cli.startproject.async_app.core import create_app_file +from propan.cli.startproject.core import ( + create_apps_dir, + create_config_dir, + create_core_dir, + create_env, + create_project_dir, +) +from propan.cli.startproject.utils import touch_dir, write_file + + +def create_rabbit(dir: Path) -> Path: + project_dir = _create_project_dir(dir) + app_dir = _create_app_dir(project_dir / "app") + _create_config_dir(app_dir / "config") + _create_core_dir(app_dir / "core") + _create_apps_dir(app_dir / "apps") + return project_dir + + +def _create_project_dir(dirname: Path) -> Path: + project_dir = create_project_dir(dirname, "propan[async-rabbit]") + + write_file( + project_dir / "docker-compose.yaml", + 'version: "3"', + "", + "services:", + " rabbit:", + " image: rabbitmq", + " environment:", + " RABBITMQ_DEFAULT_USER: guest", + " RABBITMQ_DEFAULT_PASS: guest", + " healthcheck:", + " test: rabbitmq-diagnostics -q ping", + " interval: 3s", + " timeout: 30s", + " retries: 3", + " ports:", + " - 5672:5672", + "", + " app:", + " build: .", + " environment:", + " APP_BROKER__URL: amqp://guest:guest@rabbit:5672/", + " volumes:", + " - ./app:/home/user/app:ro", + " depends_on:", + " rabbit:", + " condition: service_healthy", + ) + + return project_dir + + +def _create_app_dir(app: Path) -> Path: + app_dir = touch_dir(app) + create_app_file(app_dir, "RabbitBroker") + return app_dir + + +def _create_config_dir(config: Path) -> Path: + config_dir = create_config_dir(config) + create_env(config_dir, "amqp://guest:guest@localhost:5672/") + return config_dir + + +def _create_core_dir(core: Path) -> Path: + core_dir = create_core_dir(core, "RabbitBroker") + return core_dir + + +def _create_apps_dir(apps: Path) -> Path: + apps_dir = create_apps_dir(apps) + + write_file( + apps_dir / "handlers.py", + "from propan.annotations import Logger", + "", + "from core import broker", + "", + "from propan.brokers.rabbit import RabbitQueue, RabbitExchange", + "", + "", + '@broker.handle(queue=RabbitQueue("test"),', + ' exchange=RabbitExchange("test"))', + "async def base_handler(body: dict, logger: Logger):", + " logger.info(body)", + ) + + return apps_dir diff --git a/propan/cli/startproject/async_app/redis.py b/propan/cli/startproject/async_app/redis.py new file mode 100644 index 00000000..6dac1af0 --- /dev/null +++ b/propan/cli/startproject/async_app/redis.py @@ -0,0 +1,87 @@ +from pathlib import Path + +from propan.cli.startproject.async_app.core import create_app_file +from propan.cli.startproject.core import ( + create_apps_dir, + create_config_dir, + create_core_dir, + create_env, + create_project_dir, +) +from propan.cli.startproject.utils import touch_dir, write_file + + +def create_redis(dir: Path) -> Path: + project_dir = _create_project_dir(dir) + app_dir = _create_app_dir(project_dir / "app") + _create_config_dir(app_dir / "config") + _create_core_dir(app_dir / "core") + _create_apps_dir(app_dir / "apps") + return project_dir + + +def _create_project_dir(dirname: Path) -> Path: + project_dir = create_project_dir(dirname, "propan[async-redis]") + + write_file( + project_dir / "docker-compose.yaml", + 'version: "3"', + "", + "services:", + " redis:", + " image: redis", + " healthcheck:", + " test: ['CMD', 'redis-cli', 'ping']", + " interval: 3s", + " timeout: 30s", + " retries: 3", + " ports:", + " - 6379:6379", + "", + " app:", + " build: .", + " environment:", + " APP_BROKER__URL: redis://redis:6379/", + " volumes:", + " - ./app:/home/user/app:ro", + " depends_on:", + " redis:", + " condition: service_healthy", + ) + + return project_dir + + +def _create_app_dir(app: Path) -> Path: + app_dir = touch_dir(app) + create_app_file(app_dir, "RedisBroker") + return app_dir + + +def _create_config_dir(config: Path) -> Path: + config_dir = create_config_dir(config) + create_env(config_dir, "redis://localhost:6379/") + return config_dir + + +def _create_core_dir(core: Path) -> Path: + core_dir = create_core_dir(core, "RedisBroker") + return core_dir + + +def _create_apps_dir(apps: Path) -> Path: + apps_dir = create_apps_dir(apps) + + write_file( + apps_dir / "handlers.py", + "from propan.annotations import Logger", + "", + "from core import broker", + "", + "", + "@broker.handle('test')", + "async def base_handler(body: dict, logger: Logger):", + " logger.info(body)", + ) + + return apps_dir diff --git a/propan/cli/startproject/core.py b/propan/cli/startproject/core.py new file mode 100644 index 00000000..85d0a19f --- /dev/null +++ b/propan/cli/startproject/core.py @@ -0,0 +1,143 @@ +from pathlib import Path + +from propan.__about__ import __version__ +from propan.cli.startproject.utils import touch_dir, write_file + + +def create_project_dir(dirname: Path, version: str) -> Path: + project_dir = touch_dir(dirname) + create_readme(project_dir) + create_dockerfile(project_dir) + create_requirements(project_dir, version) + create_gitignore(project_dir) + return project_dir + + +def create_readme(project_dir: Path) -> None: + write_file(project_dir / "README.md") + + +def create_dockerfile(project_dir: Path) -> None: + write_file( + project_dir / "Dockerfile", + "FROM python:3.11.3-slim", + "", + "ENV PYTHONUNBUFFERED=1 \\", + " PYTHONDONTWRITEBYTECODE=1 \\", + " PIP_DISABLE_PIP_VERSION_CHECK=on", + "", + "RUN useradd -ms /bin/bash user", + "", + "USER user", + "WORKDIR /home/user", + "", + "COPY requirements.txt requirements.txt", + "", + "RUN pip install --no-warn-script-location --no-cahche-dir -r requirements.txt", + "", + "COPY ./app ./app", + "", + 'ENTRYPOINT [ "python", "-m", "propan", "run", "app.serve:app" ]', + ) + + +def create_gitignore(project_dir: Path) -> None: + write_file( + project_dir / ".gitignore", + "*.env*", + "__pycache__", + "env/", + "venv/", + ) + + +def create_requirements(project_dir: Path, version: str) -> None: + write_file( + project_dir / "requirements.txt", + f"{version}=={__version__}", + "python-dotenv", + ) + + +def create_env(config_dir: Path, url: str) -> None: + write_file( + config_dir / ".env", + "APP_DEBUG=True", + f"APP_BROKER__URL={url}", + ) + + +def create_config_dir(config: Path) -> Path: + config_dir = touch_dir(config) + + write_file( + config_dir / "settings.py", + "from pathlib import Path", + "from typing import Optional", + "", + "from pydantic import BaseSettings, BaseModel, Field", + "", + "", + "CONFIG_DIR = Path(__file__).resolve().parent", + "BASE_DIR = CONFIG_DIR.parent", + "", + "", + "class BrokerSettings(BaseModel):", + " url: str = Field(...)", + "", + "", + "class Settings(BaseSettings):", + " debug: bool = Field(True)", + " broker: BrokerSettings = Field(default_factory=BrokerSettings)", + " base_dir: Path = BASE_DIR", + "", + " class Config:", + " env_prefix = 'APP_'", + " env_file_encoding = 'utf-8'", + " env_nested_delimiter = '__'", + "", + "", + "def init_settings(env_file: Optional[str] = None) -> Settings:", + ' env_file = CONFIG_DIR / (env_file or ".env")', + " if env_file.exists():", + " settings = Settings(_env_file=env_file)", + " else:", + " settings = Settings()", + " return settings", + ) + + write_file( + config_dir / "__init__.py", + "from .settings import Settings, init_settings", + ) + + return config_dir + + +def create_core_dir(core: Path, broker_class: str) -> Path: + core_dir = touch_dir(core) + + write_file( + core_dir / "__init__.py", + "from .dependencies import broker", + ) + + write_file( + core_dir / "dependencies.py", + f"from propan import {broker_class}", + "", + f"broker = {broker_class}()", + ) + + return core_dir + + +def create_apps_dir(apps: Path) -> Path: + apps_dir = touch_dir(apps) + + write_file( + apps_dir / "__init__.py", + "from .handlers import base_handler", + ) + + return apps_dir diff --git a/propan/cli/startproject/sync_app/__init__.py b/propan/cli/startproject/sync_app/__init__.py new file mode 100644 index 00000000..804e21c4 --- /dev/null +++ b/propan/cli/startproject/sync_app/__init__.py @@ -0,0 +1,3 @@ +from propan.cli.startproject.sync_app.app import sync_app + +__all__ = ("sync_app",) diff --git a/propan/cli/startproject/sync_app/app.py b/propan/cli/startproject/sync_app/app.py new file mode 100644 index 00000000..6cacceb4 --- /dev/null +++ b/propan/cli/startproject/sync_app/app.py @@ -0,0 +1,3 @@ +import typer + +sync_app = typer.Typer(pretty_exceptions_short=True) diff --git a/propan/cli/startproject/utils.py b/propan/cli/startproject/utils.py new file mode 100644 index 00000000..7d41c4f4 --- /dev/null +++ b/propan/cli/startproject/utils.py @@ -0,0 +1,18 @@ +from pathlib import Path +from typing import Union, cast + + +def touch_dir(dir: Union[Path, str]) -> Path: + if isinstance(dir, str) is True: + dir = Path(dir).resolve() + + dir = cast(Path, dir) + if dir.exists() is False: + dir.mkdir() + return dir + + +def write_file(path: Path, *content: str) -> None: + path.touch() + if content: + path.write_text("\n".join(content)) diff --git a/propan/fastapi/__init__.py b/propan/fastapi/__init__.py index dd7ca6e4..6ef56fef 100644 --- a/propan/fastapi/__init__.py +++ b/propan/fastapi/__init__.py @@ -1,3 +1,11 @@ -from propan.fastapi.rabbit import RabbitRouter +try: + from propan.fastapi.rabbit import RabbitRouter +except Exception: + RabbitRouter = None # type: ignore -__all__ = ("RabbitRouter",) +try: + from propan.fastapi.redis import RedisRouter +except Exception: + RedisRouter = None # type: ignore + +__all__ = ("RabbitRouter", "RedisRouter") diff --git a/propan/fastapi/core/route.py b/propan/fastapi/core/route.py index 679f68f9..66b8f546 100644 --- a/propan/fastapi/core/route.py +++ b/propan/fastapi/core/route.py @@ -100,7 +100,7 @@ async def app(request: PropanMessage) -> Any: values, errors, _, _2, _3 = solved_result if errors: - raise ValidationError(errors, create_model("MQRoute")) + raise ValidationError(errors, create_model("PropanRoute")) return await run_endpoint_function( dependant=dependant, diff --git a/propan/fastapi/rabbit/__init__.py b/propan/fastapi/rabbit/__init__.py index c1953309..c2e67bcb 100644 --- a/propan/fastapi/rabbit/__init__.py +++ b/propan/fastapi/rabbit/__init__.py @@ -1,3 +1,3 @@ -from propan.fastapi.rabbit.rabbit_router import RabbitRouter +from propan.fastapi.rabbit.router import RabbitRouter __all__ = ("RabbitRouter",) diff --git a/propan/fastapi/rabbit/rabbit_router.py b/propan/fastapi/rabbit/router.py similarity index 100% rename from propan/fastapi/rabbit/rabbit_router.py rename to propan/fastapi/rabbit/router.py diff --git a/propan/fastapi/rabbit/rabbit_router.pyi b/propan/fastapi/rabbit/router.pyi similarity index 100% rename from propan/fastapi/rabbit/rabbit_router.pyi rename to propan/fastapi/rabbit/router.pyi diff --git a/propan/fastapi/redis/__init__.py b/propan/fastapi/redis/__init__.py new file mode 100644 index 00000000..daf1f93b --- /dev/null +++ b/propan/fastapi/redis/__init__.py @@ -0,0 +1,3 @@ +from propan.fastapi.redis.router import RedisRouter + +__all__ = ("RedisRouter",) diff --git a/propan/fastapi/redis/router.py b/propan/fastapi/redis/router.py new file mode 100644 index 00000000..68a70d2d --- /dev/null +++ b/propan/fastapi/redis/router.py @@ -0,0 +1,7 @@ +from propan.brokers.redis import RedisBroker +from propan.fastapi.core.router import PropanRouter + + +class RedisRouter(PropanRouter): + broker_class = RedisBroker + broker_class: RedisBroker diff --git a/propan/fastapi/redis/router.pyi b/propan/fastapi/redis/router.pyi new file mode 100644 index 00000000..c3348b46 --- /dev/null +++ b/propan/fastapi/redis/router.pyi @@ -0,0 +1,87 @@ +import logging +from enum import Enum +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Type, Union + +from fastapi import params +from fastapi.datastructures import Default +from fastapi.routing import APIRoute +from fastapi.utils import generate_unique_id +from redis.asyncio.connection import BaseParser, DefaultParser, Encoder +from starlette import routing +from starlette.responses import JSONResponse, Response +from starlette.types import ASGIApp + +from propan.fastapi.core.router import PropanRouter +from propan.log import access_logger +from propan.types import AnyCallable + +class RedisRouter(PropanRouter): + def __init__( + self, + url: str = "redis://localhost:6379", + polling_interval: float = 1.0, + host: str = "localhost", + port: Union[str, int] = 6379, + db: Union[str, int] = 0, + password: Optional[str] = None, + socket_timeout: Optional[float] = None, + socket_connect_timeout: Optional[float] = None, + socket_keepalive: bool = False, + socket_keepalive_options: Optional[Mapping[int, Union[int, bytes]]] = None, + socket_type: int = 0, + retry_on_timeout: bool = False, + encoding: str = "utf-8", + encoding_errors: str = "strict", + decode_responses: bool = False, + parser_class: Type[BaseParser] = DefaultParser, + socket_read_size: int = 65536, + health_check_interval: float = 0, + client_name: Optional[str] = None, + username: Optional[str] = None, + encoder_class: Type[Encoder] = Encoder, + # FastAPI kwargs + prefix: str = "", + tags: Optional[List[Union[str, Enum]]] = None, + dependencies: Optional[Sequence[params.Depends]] = None, + default_response_class: Type[Response] = Default(JSONResponse), + responses: Optional[Dict[Union[int, str], Dict[str, Any]]] = None, + callbacks: Optional[List[routing.BaseRoute]] = None, + routes: Optional[List[routing.BaseRoute]] = None, + redirect_slashes: bool = True, + default: Optional[ASGIApp] = None, + dependency_overrides_provider: Optional[Any] = None, + route_class: Type[APIRoute] = APIRoute, + on_startup: Optional[Sequence[Callable[[], Any]]] = None, + on_shutdown: Optional[Sequence[Callable[[], Any]]] = None, + deprecated: Optional[bool] = None, + include_in_schema: bool = True, + generate_unique_id_function: Callable[[APIRoute], str] = Default( + generate_unique_id + ), + # Broker kwargs + logger: Optional[logging.Logger] = access_logger, + log_level: int = logging.INFO, + log_fmt: Optional[str] = None, + apply_types: bool = True, + consumers: Optional[int] = None, + ) -> None: + pass + def add_api_mq_route( # type: ignore[override] + self, + channel: str, + *, + endpoint: AnyCallable, + name: Optional[str] = None, + pattern: bool = False, + retry: Union[bool, int] = False, + ) -> None: + pass + def event( # type: ignore[override] + self, + channel: str, + *, + pattern: bool = False, + retry: Union[bool, int] = False, + name: Optional[str] = None, + ) -> None: + pass diff --git a/propan/test/__init__.py b/propan/test/__init__.py index fa01e003..33ba3aa7 100644 --- a/propan/test/__init__.py +++ b/propan/test/__init__.py @@ -3,4 +3,12 @@ except Exception: TestRabbitBroker = None # type: ignore -__all__ = ("TestRabbitBroker",) +try: + from propan.test.redis import TestRedisBroker +except Exception: + TestRedisBroker = None # type: ignore + +__all__ = ( + "TestRabbitBroker", + "TestRedisBroker", +) diff --git a/propan/test/rabbit.py b/propan/test/rabbit.py index a0aa6779..03161f6e 100644 --- a/propan/test/rabbit.py +++ b/propan/test/rabbit.py @@ -1,4 +1,3 @@ -import asyncio import sys from contextlib import asynccontextmanager from types import MethodType @@ -24,12 +23,12 @@ RabbitQueue, ) from propan.brokers.rabbit.rabbit_broker import ( # type: ignore - Handler, PikaSendableMessage, TimeoutType, _validate_exchange, _validate_queue, ) +from propan.test.utils import call_handler __all__ = ( "build_message", @@ -43,25 +42,6 @@ async def process(self): # type: ignore yield -async def call_handler( - handler: Handler, - message: IncomingMessage, - callback: bool = False, - callback_timeout: Optional[float] = 30.0, - raise_timeout: bool = False, -) -> Any: - r = handler.callback(message) - try: - result = await asyncio.wait_for(r, timeout=callback_timeout) - except asyncio.TimeoutError as e: - if raise_timeout is True: # pragma: no branch - raise e - result = None - - if callback is True: # pragma: no branch - return result - - def build_message( message: PikaSendableMessage = "", queue: Union[RabbitQueue, str] = "", diff --git a/propan/test/redis.py b/propan/test/redis.py new file mode 100644 index 00000000..18d71c14 --- /dev/null +++ b/propan/test/redis.py @@ -0,0 +1,86 @@ +import re +import sys +from types import MethodType +from typing import Any, Dict, Optional, Union + +from typing_extensions import TypeAlias + +if sys.version_info < (3, 8): + from asyncmock import AsyncMock +else: + from unittest.mock import AsyncMock + +from propan.brokers.redis.redis_broker import RedisBroker +from propan.brokers.redis.schemas import RedisMessage +from propan.test.utils import call_handler +from propan.types import SendableMessage + +__all__ = ( + "build_message", + "TestRedisBroker", +) + +Msg: TypeAlias = Dict[str, Union[bytes, str, None]] + + +def build_message( + message: SendableMessage, + channel: str, + *, + reply_to: str = "", + headers: Optional[Dict[str, Any]] = None, +) -> Msg: + msg, content_type = RedisBroker._encode_message(message) + return { + "type": "message", + "pattern": None, + "channel": channel.encode(), + "data": RedisMessage( + data=msg, + headers={ + "content-type": content_type or "", + **(headers or {}), + }, + reply_to=reply_to, + ) + .json() + .encode(), + } + + +async def publish( + self: RedisBroker, + message: SendableMessage, + channel: str, + *, + reply_to: str = "", + headers: Optional[Dict[str, Any]] = None, + callback: bool = False, + callback_timeout: Optional[float] = 30.0, + raise_timeout: bool = False, +) -> Any: + incoming = build_message( + message=message, + channel=channel, + reply_to=reply_to, + headers=headers, + ) + + for handler in self.handlers: # pragma: no branch + if ( + not handler.pattern and handler.channel == channel + ) or ( # pragma: no branch + handler.pattern and re.match(handler.channel, channel) + ): + r = await call_handler( + handler, incoming, callback, callback_timeout, raise_timeout + ) + if callback: # pragma: no branch + return r + + +def TestRedisBroker(broker: RedisBroker) -> RedisBroker: + broker.connect = AsyncMock() # type: ignore + broker.start = AsyncMock() # type: ignore + broker.publish = MethodType(publish, broker) # type: ignore + return broker diff --git a/propan/test/utils.py b/propan/test/utils.py new file mode 100644 index 00000000..6b8a40b9 --- /dev/null +++ b/propan/test/utils.py @@ -0,0 +1,23 @@ +import asyncio +from typing import Any, Optional + +from propan.brokers.model.schemas import BaseHandler + + +async def call_handler( + handler: BaseHandler, + message: Any, + callback: bool = False, + callback_timeout: Optional[float] = 30.0, + raise_timeout: bool = False, +) -> Any: + r = handler.callback(message) + try: + result = await asyncio.wait_for(r, timeout=callback_timeout) + except asyncio.TimeoutError as e: + if raise_timeout is True: # pragma: no branch + raise e + result = None + + if callback is True: # pragma: no branch + return result diff --git a/propan/types.py b/propan/types.py index 4930e28f..02a692e2 100644 --- a/propan/types.py +++ b/propan/types.py @@ -1,4 +1,4 @@ -from typing import Any, Awaitable, Callable, Coroutine, Dict, Union +from typing import Any, Awaitable, Callable, Coroutine, Dict, Sequence, Union from pydantic import BaseModel from typing_extensions import TypeAlias @@ -15,7 +15,7 @@ Wrapper: TypeAlias = Callable[[AnyCallable], DecoratedCallable] AsyncWrapper: TypeAlias = Callable[[AnyCallable], DecoratedAsync] -DecodedMessage: TypeAlias = Union[str, AnyDict, bytes] +DecodedMessage: TypeAlias = Union[AnyDict, Sequence[Any], str, bytes] SendableMessage: TypeAlias = Union[DecodedMessage, BaseModel, None] HandlerWrapper: TypeAlias = Callable[ diff --git a/propan/utils/context/main.py b/propan/utils/context/main.py index 7ca2dc3e..988a2fca 100644 --- a/propan/utils/context/main.py +++ b/propan/utils/context/main.py @@ -49,6 +49,7 @@ def __getattr__(self, __name: str) -> Any: @property def context(self) -> Dict[str, Any]: return { + "context": self, **{i: j.get() for i, j in self._scope_context.items()}, **self._global_context, } diff --git a/pyproject.toml b/pyproject.toml index eb590e40..f2a0cb14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,9 +71,14 @@ async-nats = [ "nats-py>=2" ] +async-redis = [ + "redis>=4.2.0rc1" +] + test = [ "propan[async-rabbit]", "propan[async-nats]", + "propan[async-redis]", "coverage[toml]>=7.2", "pytest>=7", @@ -98,6 +103,8 @@ dev = [ "propan[test]", "propan[doc]", + "types-redis", + "mypy==1.1.1", "black==23.3.0", "isort>=5", @@ -204,6 +211,7 @@ markers = [ "slow", "rabbit", "nats", + "redis", "all", ] diff --git a/tests/brokers/rabbit/conftest.py b/tests/brokers/rabbit/conftest.py index 4a30d9f0..85e32be3 100644 --- a/tests/brokers/rabbit/conftest.py +++ b/tests/brokers/rabbit/conftest.py @@ -53,7 +53,7 @@ async def full_broker(settings): @pytest_asyncio.fixture -async def test_broker(broker): +async def test_broker(): broker = RabbitBroker() yield TestRabbitBroker(broker) await broker.close() diff --git a/tests/brokers/rabbit/test_publish.py b/tests/brokers/rabbit/test_publish.py index c298b7f9..79c5e316 100644 --- a/tests/brokers/rabbit/test_publish.py +++ b/tests/brokers/rabbit/test_publish.py @@ -15,6 +15,7 @@ "hello", {"message": "hello!"}, create_model("Message", r=str)(r="hello!"), + [1, 2, 3], ), ) async def test_rpc(message, queue: RabbitQueue, broker: RabbitBroker): diff --git a/tests/brokers/redis/conftest.py b/tests/brokers/redis/conftest.py new file mode 100644 index 00000000..f5b6b7e4 --- /dev/null +++ b/tests/brokers/redis/conftest.py @@ -0,0 +1,44 @@ +from uuid import uuid4 + +import pytest +import pytest_asyncio +from pydantic import BaseSettings + +from propan import RedisBroker +from propan.test import TestRedisBroker + + +class Settings(BaseSettings): + url = "redis://localhost:6379" + + +@pytest.fixture +def channel_name(): + return str(uuid4()) + + +@pytest.fixture(scope="session") +def settings(): + return Settings() + + +@pytest_asyncio.fixture +@pytest.mark.redis +async def broker(settings): + broker = RedisBroker(settings.url, apply_types=False) + yield broker + await broker.close() + + +@pytest_asyncio.fixture +@pytest.mark.redis +async def full_broker(settings): + broker = RedisBroker(settings.url) + yield broker + await broker.close() + + +@pytest_asyncio.fixture +async def test_broker(): + broker = RedisBroker() + yield TestRedisBroker(broker) diff --git a/tests/brokers/redis/test_acc.py b/tests/brokers/redis/test_acc.py new file mode 100644 index 00000000..f7a461a3 --- /dev/null +++ b/tests/brokers/redis/test_acc.py @@ -0,0 +1,115 @@ +from asyncio import Event, wait_for + +import pytest + +from propan import RedisBroker + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_consume( + mock, + channel_name: str, + broker: RedisBroker, +): + consume = Event() + mock.side_effect = lambda *_: consume.set() # pragma: no branch + + async with broker: + broker.handle(channel_name)(mock) + await broker.start() + await broker.publish("hello", channel_name) + await wait_for(consume.wait(), 3) + + mock.assert_called_once() + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_pattern_consume( + mock, + channel_name: str, + broker: RedisBroker, +): + consume = Event() + mock.side_effect = lambda *_: consume.set() # pragma: no branch + + async with broker: + broker.handle("*", pattern=True)(mock) + await broker.start() + await broker.publish("hello", channel_name) + await wait_for(consume.wait(), 3) + + mock.assert_called_once() + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_consume_from_native_redis( + mock, + channel_name: str, + broker: RedisBroker, +): + consume = Event() + mock.side_effect = lambda *_: consume.set() # pragma: no branch + + async with broker: + broker.handle(channel_name)(mock) + await broker.start() + await broker._connection.publish(channel_name, "msg") + await wait_for(consume.wait(), 3) + + mock.assert_called_once() + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_consume_double( + mock, + channel_name: str, + broker: RedisBroker, +): + consume = Event() + mock.side_effect = lambda *_: consume.set() # pragma: no branch + + async with broker: + broker.handle(channel_name)(mock) + await broker.start() + + await broker.publish("hello", channel_name) + await wait_for(consume.wait(), 3) + + consume.clear() + await broker.publish("hello", channel_name) + await wait_for(consume.wait(), 3) + + assert mock.call_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_different_consume( + mock, + channel_name: str, + broker: RedisBroker, +): + first_consume = Event() + second_consume = Event() + + mock.method.side_effect = lambda *_: first_consume.set() # pragma: no branch + mock.method2.side_effect = lambda *_: second_consume.set() # pragma: no branch + + another_channel = channel_name + "1" + async with broker: + broker.handle(channel_name)(mock.method) + broker.handle(another_channel)(mock.method2) + await broker.start() + + await broker.publish("hello", channel_name) + await broker.publish("hello", another_channel) + + await wait_for(first_consume.wait(), 3) + await wait_for(second_consume.wait(), 3) + + mock.method.assert_called_once() + mock.method2.assert_called_once() diff --git a/tests/brokers/redis/test_connect.py b/tests/brokers/redis/test_connect.py new file mode 100644 index 00000000..1c77770f --- /dev/null +++ b/tests/brokers/redis/test_connect.py @@ -0,0 +1,26 @@ +import pytest + +from propan import RedisBroker + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_init_connect_by_url(broker): + assert await broker.connect() + await broker.close() + + +@pytest.mark.asyncio +@pytest.mark.rabbit +async def test_connect_by_url(settings): + broker = RedisBroker() + assert await broker.connect(settings.url) + await broker.close() + + +@pytest.mark.asyncio +@pytest.mark.rabbit +async def test_connect_by_url_priority(settings): + broker = RedisBroker("wrong_url") + assert await broker.connect(settings.url) + await broker.close() diff --git a/tests/brokers/redis/test_publish.py b/tests/brokers/redis/test_publish.py new file mode 100644 index 00000000..554e7a8e --- /dev/null +++ b/tests/brokers/redis/test_publish.py @@ -0,0 +1,70 @@ +import asyncio + +import pytest +from pydantic import create_model + +from propan import RedisBroker + + +@pytest.mark.asyncio +@pytest.mark.redis +@pytest.mark.parametrize( + "message", + ( + b"hello!", + "hello", + {"message": "hello!"}, + create_model("Message", r=str)(r="hello!"), + [1, 2, 3], + ), +) +async def test_rpc(message, channel_name: str, broker: RedisBroker): + @broker.handle(channel_name) + async def handler(m): + return m + + async with broker: + await broker.start() + r = await broker.publish(message, channel_name, callback=True) + + assert r == message + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_rpc_timeout_raises(channel_name: str, full_broker: RedisBroker): + @full_broker.handle(channel_name) + async def m(): # pragma: no cover + await asyncio.sleep(1) + + async with full_broker: + await full_broker.start() + + with pytest.raises(asyncio.TimeoutError): # pragma: no branch + await full_broker.publish( + message="hello", + channel=channel_name, + callback=True, + raise_timeout=True, + callback_timeout=0, + ) + + +@pytest.mark.asyncio +@pytest.mark.redis +async def test_rpc_timeout_none(channel_name: str, full_broker: RedisBroker): + @full_broker.handle(channel_name) + async def m(): # pragma: no cover + await asyncio.sleep(1) + + async with full_broker: + await full_broker.start() + + r = await full_broker.publish( + message="hello", + channel=channel_name, + callback=True, + callback_timeout=0, + ) + + assert r is None diff --git a/tests/brokers/redis/test_test_client.py b/tests/brokers/redis/test_test_client.py new file mode 100644 index 00000000..cb3df44f --- /dev/null +++ b/tests/brokers/redis/test_test_client.py @@ -0,0 +1,88 @@ +import asyncio + +import pytest +from pydantic import ValidationError, create_model + +from propan import RedisBroker +from propan.test.redis import build_message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "message", + ( + b"hello!", + "hello", + {"message": "hello!"}, + create_model("Message", r=str)(r="hello!"), + [1, 2, 3], + ), +) +async def test_rpc(message, channel_name: str, test_broker: RedisBroker): + @test_broker.handle(channel_name) + async def handler(m): + return m + + async with test_broker: + await test_broker.start() + r = await test_broker.publish(message, channel_name, callback=True) + + assert r == message + + +@pytest.mark.asyncio +async def test_rpc_timeout_raises(channel_name: str, test_broker: RedisBroker): + @test_broker.handle(channel_name) + async def m(): # pragma: no cover + await asyncio.sleep(1) + + async with test_broker: + await test_broker.start() + + with pytest.raises(asyncio.TimeoutError): + await test_broker.publish( + message="hello", + channel=channel_name, + callback=True, + raise_timeout=True, + callback_timeout=0, + ) + + +@pytest.mark.asyncio +async def test_pattern_consume(channel_name: str, test_broker: RedisBroker): + @test_broker.handle(".*", pattern=True) + async def m(): # pragma: no cover + return 1 + + async with test_broker: + await test_broker.start() + + assert ( + await test_broker.publish( + message="hello", + channel=channel_name, + callback=True, + ) + ) == 1 + + +@pytest.mark.asyncio +async def test_handler_calling(channel_name: str, test_broker: RedisBroker): + @test_broker.handle(channel_name) + async def handler(m: dict): + return m + + raw_msg = {"msg": "hello!"} + message = build_message(raw_msg, channel_name) + + wrong_msg = build_message("Hi!", channel_name) + + async with test_broker: + await test_broker.start() + assert raw_msg == await handler(message) + + await handler(wrong_msg) + + with pytest.raises(ValidationError): + await handler(wrong_msg, reraise_exc=True) diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 6ba84a3e..35ad22de 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -5,7 +5,7 @@ from propan import PropanApp from propan.brokers.rabbit import RabbitBroker -from propan.cli.startproject import create +from propan.cli.startproject.async_app.rabbit import create_rabbit @pytest.fixture @@ -34,6 +34,6 @@ def runner(): @pytest.fixture(scope="module") -def project_dir(version): +def rabbit_async_project(): with TemporaryDirectory() as dir: - yield create(dir, version) + yield create_rabbit(dir) diff --git a/tests/cli/test_app.py b/tests/cli/test_app.py index 172a3a9f..17d7f00f 100644 --- a/tests/cli/test_app.py +++ b/tests/cli/test_app.py @@ -78,12 +78,11 @@ def call2(): @pytest.mark.asyncio @needs_py38 async def test_startup_lifespan_before_broker_started(async_mock, app: PropanApp): + @app.on_startup async def call(): await async_mock() assert not async_mock.broker_start.called - app.on_startup(call) - with patch.object(app.broker, "start", async_mock.broker_start): await app._startup() @@ -94,12 +93,11 @@ async def call(): @pytest.mark.asyncio @needs_py38 async def test_shutdown_lifespan_after_broker_stopped(mock, async_mock, app: PropanApp): + @app.after_shutdown async def call(): await async_mock() assert async_mock.broker_stop.called - app.on_shutdown(call) - with patch.object(app.broker, "close", async_mock.broker_stop): await app._shutdown() diff --git a/tests/cli/test_creation.py b/tests/cli/test_creation.py index 0c82d701..d9700cd2 100644 --- a/tests/cli/test_creation.py +++ b/tests/cli/test_creation.py @@ -1,2 +1,2 @@ -def test_create_propject(project_dir): - assert project_dir.exists() +def test_create_propject(rabbit_async_project): + assert rabbit_async_project.exists() diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 6609f881..c2ba010c 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -10,8 +10,8 @@ @pytest.mark.rabbit @pytest.mark.slow -def test_run_correct(project_dir): - module, app = get_app_path(f'{project_dir / "app" / "serve"}:app') +def test_run_rabbit_correct(rabbit_async_project): + module, app = get_app_path(f'{rabbit_async_project / "app" / "serve"}:app') sys.path.insert(0, str(module.parent)) p = Process(target=_run, args=(module, app, {})) p.start() diff --git a/tests/fastapi/test_app.py b/tests/fastapi/test_app.py index d1e9b2a3..492db3f6 100644 --- a/tests/fastapi/test_app.py +++ b/tests/fastapi/test_app.py @@ -3,12 +3,12 @@ import pytest from fastapi import FastAPI -from propan.fastapi import RabbitRouter -from propan.test import TestRabbitBroker +from propan.fastapi import RabbitRouter, RedisRouter +from propan.test import TestRabbitBroker, TestRedisBroker @pytest.mark.asyncio -async def test_consume(): +async def test_rabbit(): name = str(uuid4()) name2 = name + "1" @@ -37,3 +37,33 @@ async def hello2(b: int): assert r == "2" await router.shutdown() + + +@pytest.mark.asyncio +async def test_redis(): + name = str(uuid4()) + name2 = name + "1" + + router = RedisRouter() + router.broker = TestRedisBroker(router.broker) + + app = FastAPI() + app.include_router(router) + + @router.event(name) + async def hello(): + return "1" + + @router.event(name2) + async def hello2(b: int): + return "2" + + await router.startup() + + r = await router.broker.publish("", name, callback=True, callback_timeout=0.5) + assert r == "1" + + r = await router.broker.publish("2", name2, callback=True, callback_timeout=0.5) + assert r == "2" + + await router.shutdown()