From 7e1ccb973bfff08612afcc81b677de8d3127317e Mon Sep 17 00:00:00 2001 From: Evan Rees <25933122+WiscEvan@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:06:28 +0200 Subject: [PATCH] :memo: Update task-queue docs in contributing --- automappa/pages/home/tasks/__init__.py | 2 +- .../{task_status_badge.py => status_badge.py} | 0 docs/CONTRIBUTING.md | 3 +- docs/task-queue.md | 56 ++++++++++++++++--- 4 files changed, 51 insertions(+), 10 deletions(-) rename automappa/pages/home/tasks/{task_status_badge.py => status_badge.py} (100%) diff --git a/automappa/pages/home/tasks/__init__.py b/automappa/pages/home/tasks/__init__.py index 0aefe56..ddc44e0 100644 --- a/automappa/pages/home/tasks/__init__.py +++ b/automappa/pages/home/tasks/__init__.py @@ -1,4 +1,4 @@ -from .task_status_badge import set_badge_color +from .status_badge import set_badge_color from .sample_cards import ( create_metagenome_model, initialize_refinement, diff --git a/automappa/pages/home/tasks/task_status_badge.py b/automappa/pages/home/tasks/status_badge.py similarity index 100% rename from automappa/pages/home/tasks/task_status_badge.py rename to automappa/pages/home/tasks/status_badge.py diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 66814f9..43584d3 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -22,6 +22,7 @@ - [Postgres](#postgres) - [RabbitMQ](#rabbitmq) - [Celery](#celery-task-queue) + - [task-queue docs](task-queue.md) - [Redis](#redis) - [Flower](#flower) - [Dev Resources](#development-resources) @@ -1117,7 +1118,7 @@ Automappa services (`redis`, `flower`, `web`, `celery`, `rabbitmq`) as tasks are registered with celery at instantiation and will not be 'hot-reloaded' like other parts of the app. -For more information on implementing new tasks see the [task docs](../automappa/pages/home/tasks/README.md) +For more information on implementing new tasks see the [task-queue docs](task-queue.md)
diff --git a/docs/task-queue.md b/docs/task-queue.md index b1a6f81..c732c9e 100644 --- a/docs/task-queue.md +++ b/docs/task-queue.md @@ -1,5 +1,13 @@ # Tasks +Simple steps to adding a task: + +1. [Add page's `tasks` module](#adding-a-new-task-module-discovery-path) +2. [Add page tasks module in `celeryconfig.py`](#add-in-tasks-module-in-celeryconfigpy) +3. [Define task in `tasks` module](#define-task-in-tasks-module) +4. [Import task in `__init__.py` w.r.t `tasks` module](#import-task-in-__init__py-within-respective-tasks-module) + + ## Adding a new task module discovery path To have the Celery task-queue register a page's tasks, they must be @@ -10,31 +18,63 @@ We have our page's tasks module: ```console automappa/pages/home/tasks -├── README.md +|... ├── __init__.py -└── task_status_badge.py +└── status_badge.py ``` +## Add in `tasks` module in `celeryconfig.py` + We configure celery for this module in the `celeryconfig.py` file. ```python -# contents of celeryconfig.py +# contents of automappa/conf/celeryconfig.py imports = ("automappa.pages.home.tasks", "automappa.tasks") ``` This ***almost*** takes care of celery checking for tasks in the module -Unfortunately, this is not all, we also need to update the +Unfortunately, this is not all... + +## Define task in `tasks` module + +Here is a simple example where we will simulate a long-running task with sleep and then set a badge color. + +> Notice we are importing our celery task-queue application (`queue`) from the base automappa directory. + +```python +# contents of /automappa/pages/home/tasks/status_badge.py +import time + +from automappa.tasks import queue + + +@queue.task(bind=True) +def set_badge_color(self, color: str) -> str: + time.sleep(15) + return color + + +if __name__ == "__main__": + pass + +``` + +Now with our task defined we can move on to importing it into our `tasks` module to be used by celery. + +## Import task in `__init__.py` within respective `tasks` module + +We also need to update the `automappa/pages/home/tasks/__init__.py` with all of our implemented tasks in the module for celery to recognize and register the tasks under the `automappa/pages/home/tasks` module. -For example we have a task `set_badge_color` defined in `task_status_badge.py`. +For example we have a task `set_badge_color` defined in `status_badge.py`. We would need to explicitly add this task to `__init__.py` like so: ```python # contents of `automappa/pages/home/tasks/__init__.py` -from .task_status_badge import set_badge_color +from .status_badge import set_badge_color # *Notice this is a relative import* __all__ = ["set_badge_color"] ``` @@ -62,7 +102,7 @@ Voilá, now celery will recognize the task on startup. It should look like this: .> celery exchange=celery(direct) key=celery [tasks] - . automappa.pages.home.tasks.task_status_badge.set_badge_color + . automappa.pages.home.tasks.status_badge.set_badge_color . automappa.tasks.aggregate_embeddings . automappa.tasks.count_kmer . automappa.tasks.embed_kmer @@ -97,4 +137,4 @@ but each page has its own `tasks` module with an `__init__.py` where any tasks implemented under this module must be imported into `__init__.py` and specified in the `__all__` dunder variable. -You can avoid alot of headache by recalling this... \ No newline at end of file +You can avoid alot of headache by recalling this...