Merge pull request #1972 from doccano/enhancement/flower

[Enhancement] Add Flower to monitor and manage Celery tasks

backend/cli.py

@@ -110,6 +110,18 @@ def command_run_task_queue(args):
     app.worker_main(argv=argv)
 
 
+def command_run_flower(args):
+    print("Starting flower.")
+    argv = [
+        "--app=config",
+        "--workdir={}".format(base),
+        "flower",
+    ]
+    if args.basic_auth:
+        argv.append("--basic_auth={}".format(args.basic_auth))
+    app.worker_main(argv=argv)
+
+
 def command_help(args):
     print(parser.parse_args([args.command, "--help"]))
 
@@ -143,9 +155,14 @@ def main():
     # Create a parser for task queue.
     parser_queue = subparsers.add_parser("task", help="see `task -h`")
     parser_queue.add_argument("--concurrency", type=int, default=2, help="concurrency")
-    parser_queue.add_argument("--env_file", type=str, help="read in a file of environment variables")
+    parser_queue.add_argument("--env_file", type=str, default="", help="read in a file of environment variables")
     parser_queue.set_defaults(handler=command_run_task_queue)
 
+    parser_flower = subparsers.add_parser("flower", help="see `flower -h`")
+    parser_flower.add_argument("--env_file", type=str, help="read in a file of environment variables")
+    parser_flower.add_argument("--basic_auth", type=str, help="username and password for basic authentication")
+    parser_flower.set_defaults(handler=command_run_flower)
+
     # Create a parser for help.
     parser_help = subparsers.add_parser("help", help="see `help -h`")
     parser_help.add_argument("command", help="command name which help is shown")

backend/pyproject.toml

@@ -64,6 +64,7 @@ django-storages = {extras = ["google"], version = "^1.12.3"}
 django-cleanup = "^6.0.0"
 filetype = "^1.0.10"
 pandas = "^1.4.2"
+flower = "^1.2.0"
 
 [tool.poetry.dev-dependencies]
 model-mommy = "^2.0.0"

docker/.env.example

@@ -3,12 +3,14 @@ ADMIN_USERNAME=admin
 ADMIN_PASSWORD=password
 ADMIN_EMAIL=admin@example.com
 
-
 # rabbit mq settings
 RABBITMQ_DEFAULT_USER=doccano
 RABBITMQ_DEFAULT_PASS=doccano
 
 # database settings
 POSTGRES_USER=doccano
 POSTGRES_PASSWORD=doccano
-POSTGRES_DB=doccano
+POSTGRES_DB=doccano
+
+# Flower settings
+FLOWER_BASIC_AUTH=""

docker/docker-compose.prod.yml

@@ -45,6 +45,26 @@ services:
     networks:
       - network-backend
 
+  flower:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.prod
+    image: doccano_flower:prod
+    entrypoint: ["/opt/bin/prod-flower.sh"]
+    environment:
+      PYTHONUNBUFFERED: "1"
+      CELERY_BROKER_URL: "amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq"
+      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable"
+      DJANGO_SETTINGS_MODULE: "config.settings.production"
+      FLOWER_BASIC_AUTH: "${FLOWER_BASIC_AUTH}" # Format "username:password"
+    depends_on:
+      - celery
+    ports:
+      - 5555:5555
+    networks:
+      - network-backend
+      - network-frontend
+
   rabbitmq:
     image: rabbitmq:3.10.7-alpine
     environment:

docs/developer_guide.md

@@ -79,6 +79,7 @@ The `tools` directory contains some shell scripts. They are mainly used in Docke
 | create-package.sh | This script creates doccano's Python package. Note that yarn and poetry must already be installed.                   |
 | heroku.sh         | This script is used to create django's superuser in Heroku.                                                          |
 | prod-celery.sh    | This script is used to run celery in `docker-compose.prod.yml`.                                                      |
+| prod-flower.sh    | This script is used to run Flower in `docker-compose.prod.yml`.                                                      |
 | prod-django.sh    | This script is used to run gunicorn in `docker-compose.prod.yml`. In addition, create roles, superuser, and migrate. |
 | run.sh            | This script is used in `Dockerfile`. After creating roles and superuser, run gunicorn and celery.                    |
 

docs/install_and_upgrade_doccano.md

@@ -9,8 +9,10 @@ Install doccano on local or in the cloud. Choose the installation method that wo
   - [Install with pip](#install-with-pip)
     - [Use PostgreSQL as a database](#use-postgresql-as-a-database)
     - [Use RabbitMQ as a message broker](#use-rabbitmq-as-a-message-broker)
+    - [Use Flower to monitor Celery tasks](#use-flower-to-monitor-celery-tasks)
   - [Install with Docker](#install-with-docker)
     - [Build a local image with Docker](#build-a-local-image-with-docker)
+    - [Use Flower](#use-flower)
   - [Install with Docker Compose](#install-with-docker-compose)
   - [Install from source](#install-from-source)
     - [Backend](#backend)
@@ -123,6 +125,16 @@ export CELERY_BROKER_URL='amqp://doccano_rabit:doccano_pass@localhost:5672//'
 
 That's it. Now you can start webserver and task queue by running the `doccano webserver` and `doccano task` command. Notice that the both commands needs `DATABASE_URL` and `CELERY_BROKER_URL` environment variables if you would change them.
 
+### Use Flower to monitor Celery tasks
+
+If you want to monitor and manage celery tasks, you can use [Flower](https://flower.readthedocs.io/en/latest/index.html). The `–basic_auth` option accepts _user:password_ pairs separated by a comma. If configured, any client trying to access this Flower instance will be prompted to provide the credentials specified in this argument:
+
+```bash
+doccano flower --basic_auth=user1:password1,user2:password2
+```
+
+Open <http://localhost:5555/>.
+
 ## Install with Docker
 
 doccano is also available as a [Docker](https://www.docker.com/) container. Make sure you have Docker installed on your machine.
@@ -156,6 +168,20 @@ If you want to build a local image, run:
 docker build -t doccano:latest . -f docker/Dockerfile
 ```
 
+### Use Flower
+
+Set `FLOWER_BASIC_AUTH` environment variable and open `5555` port. The variable accepts _user:password_ pairs separated by a comma.
+
+```bash
+docker container create --name doccano \
+  -e "ADMIN_USERNAME=admin" \
+  -e "ADMIN_EMAIL=admin@example.com" \
+  -e "ADMIN_PASSWORD=password" \
+  -e "FLOWER_BASIC_AUTH=username:password"
+  -v doccano-db:/data \
+  -p 8000:8000 -p 5555:5555 doccano/doccano
+```
+
 ## Install with Docker Compose
 
 You need to install Git and to clone the repository:

tools/prod-flower.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+
+cd /backend
+
+(
+  echo "Waiting for database"
+  python manage.py wait_for_db
+
+  echo "Starting flower"
+  if [[ -n "${FLOWER_BASIC_AUTH}" ]]; then
+    celery --app=config flower --basic_auth="${FLOWER_BASIC_AUTH}"
+  fi
+)

tools/run.sh

@@ -39,9 +39,15 @@ echo "Starting django"
 gunicorn --bind="0.0.0.0:${PORT:-8000}" --workers="${WORKERS:-1}" config.wsgi --timeout=300 &
 gunicorn_pid="$!"
 
+echo "Starting celery"
 celery --app=config worker --loglevel=INFO --concurrency="${CELERY_WORKERS:-1}" &
 celery_pid="$!"
 
+echo "Starting flower"
+if [[ -n "${FLOWER_BASIC_AUTH}" ]]; then
+  celery --app=config flower --basic_auth="${FLOWER_BASIC_AUTH}" &
+fi
+
 while :; do
   if [[ ! -e "/proc/${celery_pid}" ]]; then
     echo "celery crashed" >&2