docker-compose.yml

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# DOCKER COMPOSE FOR FLOWKIT
#

version: '3.9'

x-flowetl-common:
  &flowetl-common
  image: flowminder/flowetl:${CONTAINER_TAG:-latest}
  restart: always
  tty: true
  stdin_open: true

  volumes:
    - ${HOST_DAGS_DIR:?Must set HOST_DAGS_DIR env var}:/opt/airflow/dags:ro

  environment:
    &flowetl-envs
    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${SQL_ALCHEMY_CONN:?Must set SQL_ALCHEMY_CONN env var}
    AIRFLOW_CONN_FLOWDB: postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@flowdb:5432/flowdb
    AIRFLOW__CORE__FERNET_KEY: ${FLOWETL_FERNET_KEY:?Must set FLOWETL_FERNET_KEY env var}
    AIRFLOW__CELERY__RESULT_BACKEND: "db+${SQL_ALCHEMY_CONN:?}"
    AIRFLOW__CELERY__BROKER_URL: redis://:${FLOWETL_REDIS_PASSWORD:?Must set FLOWETL_REDIS_PASSWORD}@flowetl_redis:6379/0
    POSTGRES_HOST: flowetl_db
    FLOWETL_AIRFLOW_PG_POOL_SLOT_COUNT: 4

  depends_on:
    &flowetl-deps
    flowetl_db:
      condition: service_healthy
    flowetl_redis:
      condition: service_healthy
    flowetl_init:
      condition: service_completed_successfully


networks:
  db:
  redis:
  zero:
  flowetl:
  api:


services:

  flowdb:
    container_name: flowdb
    image: flowminder/flowdb:${CONTAINER_TAG:-latest}
    ports:
      - ${FLOWDB_PORT:?Must set FLOWDB_PORT env var}:5432

    environment:
      POSTGRES_USER: ${POSTGRES_USER:?Must set POSTGRES_USER env var}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Must set POSTGRES_PASSWORD env var}
      FLOWMACHINE_FLOWDB_PASSWORD: ${FLOWMACHINE_FLOWDB_PASSWORD:?Must set FLOWMACHINE_FLOWDB_PASSWORD env var}
      FLOWAPI_FLOWDB_USER: ${FLOWAPI_FLOWDB_USER:?Must set FLOWAPI_FLOWDB_USER env var}
      FLOWAPI_FLOWDB_PASSWORD: ${FLOWAPI_FLOWDB_PASSWORD:?Must set FLOWAPI_FLOWDB_PASSWORD env var}
      CACHE_SIZE: ${CACHE_SIZE:-""}
      CACHE_HALF_LIFE: ${CACHE_HALF_LIFE:?Must set CACHE_HALF_LIFE env var}
      FLOWDB_ENABLE_POSTGRES_DEBUG_MODE: ${FLOWDB_ENABLE_POSTGRES_DEBUG_MODE:?Must set FLOWDB_ENABLE_POSTGRES_DEBUG_MODE env var}
    shm_size: 1G
    tty: true
    stdin_open: true
    restart: always
    networks:
      db:
        aliases:
          - flowdb

  flowmachine:
    container_name: flowmachine
    image: flowminder/flowmachine:${CONTAINER_TAG:-latest}
    ports:
      - ${FLOWMACHINE_PORT:?Must set FLOWMACHINE_PORT env var}:5555
    depends_on:
      - flowmachine_query_locker
    tty: true
    stdin_open: true
    environment:
      - FLOWMACHINE_PORT=5555
      - FLOWMACHINE_LOG_LEVEL=${FLOWMACHINE_LOG_LEVEL:?Must set FLOWMACHINE_LOG_LEVEL env var}
      - FLOWMACHINE_SERVER_DEBUG_MODE=${FLOWMACHINE_SERVER_DEBUG_MODE:?Must set FLOWMACHINE_SERVER_DEBUG_MODE env var}
      - FLOWMACHINE_SERVER_DISABLE_DEPENDENCY_CACHING=${FLOWMACHINE_SERVER_DISABLE_DEPENDENCY_CACHING:?Must set FLOWMACHINE_SERVER_DISABLE_DEPENDENCY_CACHING env var}
      - FLOWDB_PORT=5432
      - FLOWDB_HOST=flowdb
      - FLOWMACHINE_FLOWDB_USER=${FLOWMACHINE_FLOWDB_USER:?Must set FLOWMACHINE_FLOWDB_USER env var}
      - FLOWMACHINE_FLOWDB_PASSWORD=${FLOWMACHINE_FLOWDB_PASSWORD:?Must set FLOWMACHINE_FLOWDB_PASSWORD env var}
      - REDIS_HOST=flowmachine_query_locker
      - REDIS_PORT=6379
      - REDIS_PASSWORD=${REDIS_PASSWORD:?Must set REDIS_PASSWORD env var}
    restart: always
    networks:
      - zero
      - db
      - redis

  worked_examples:
    container_name: worked_examples
    image: flowminder/flowkit-examples:${CONTAINER_TAG:-latest}
    ports:
      - ${WORKED_EXAMPLES_PORT:?Must set WORKED_EXAMPLES_PORT env var}:8888
    tty: true
    stdin_open: true
    environment:
      - FLOWDB_PORT=5432
      - FLOWDB_HOST=flowdb
      - FLOWMACHINE_FLOWDB_USER=${FLOWMACHINE_FLOWDB_USER:?Must set FLOWMACHINE_FLOWDB_USER env var}
      - FLOWMACHINE_FLOWDB_PASSWORD=${FLOWMACHINE_FLOWDB_PASSWORD:?Must set FLOWMACHINE_FLOWDB_PASSWORD env var}
      - REDIS_HOST=flowmachine_query_locker
      - REDIS_PORT=6379
      - REDIS_PASSWORD=${REDIS_PASSWORD:?Must set REDIS_PASSWORD env var}
      - FLOWAPI_URL=http://flowapi:9090
      - MAPBOX_ACCESS_TOKEN
    restart: always
    networks:
      - db
      - redis
    command:
      ["start.sh", "jupyter", "lab", "--LabApp.token=''"]

  flowapi:
    container_name: flowapi
    image: flowminder/flowapi:${CONTAINER_TAG:-latest}
    ports:
      - ${FLOWAPI_PORT:?Must set FLOWAPI_PORT env var}:9090
    environment:
      - FLOWMACHINE_HOST=flowmachine
      - FLOWMACHINE_PORT=5555
      - FLOWAPI_FLOWDB_USER=${FLOWAPI_FLOWDB_USER:?Must set FLOWAPI_FLOWDB_USER env var}
      - FLOWAPI_FLOWDB_PASSWORD=${FLOWAPI_FLOWDB_PASSWORD:?Must set FLOWAPI_FLOWDB_PASSWORD env var}
      - FLOWDB_HOST=flowdb
      - FLOWDB_PORT=5432
      - PUBLIC_JWT_SIGNING_KEY=${PUBLIC_JWT_SIGNING_KEY:?Must set PUBLIC_JWT_SIGNING_KEY env var}
      - FLOWAPI_LOG_LEVEL=${FLOWAPI_LOG_LEVEL:?Must set FLOWAPI_LOG_LEVEL env var}
      - FLOWAPI_IDENTIFIER=${FLOWAPI_IDENTIFIER:?Must set FLOWAPI_IDENTIFIER env var}
    tty: true
    stdin_open: true
    restart: always
    networks:
      - db
      - zero
      - api

  flowauth:
    container_name: flowauth
    image: flowminder/flowauth:${CONTAINER_TAG:-latest}
    ports:
      - ${FLOWAUTH_PORT:?Must set FLOWAUTH_PORT env var}:80
    environment:
      DEMO_MODE: ${DEMO_MODE:?Must set DEMO_MODE env var}
      PRIVATE_JWT_SIGNING_KEY: ${PRIVATE_JWT_SIGNING_KEY:?Must set PRIVATE_JWT_SIGNING_KEY env var}
      FLOWAUTH_FERNET_KEY: ${FLOWAUTH_FERNET_KEY:?Must set FLOWAUTH_FERNET_KEY env var}
      FLOWAUTH_ADMIN_PASSWORD: ${FLOWAUTH_ADMIN_PASSWORD:?Must set FLOWAUTH_ADMIN_PASSWORD env var}
      FLOWAUTH_ADMIN_USERNAME: ${FLOWAUTH_ADMIN_USERNAME:?Must set FLOWAUTH_ADMIN_USERNAME env var}
      SECRET_KEY: ${SECRET_KEY:?Must set SECRET_KEY env var}


  flowmachine_query_locker:
    container_name: flowmachine_query_locker
    image: bitnami/redis
    ports:
      - ${REDIS_PORT:?Must set REDIS_PORT env var}:6379
    environment:
      - REDIS_PASSWORD=${REDIS_PASSWORD:?Must set REDIS_PASSWORD env var}
    restart: always
    networks:
      redis:
        aliases:
          - redis

  # Flowetl components (scheduler, redis, worker(s), postgres db, webserver, triggerer,
  flowetl_scheduler:
    << : *flowetl-common
    container_name: flowetl_scheduler
    command: scheduler
    networks:
      - flowetl

  flowetl_db:
    image: postgres:11.0
    container_name: flowetl_db
    tty: true
    stdin_open: true
    restart: always
    ports:
      - ${FLOWETL_POSTGRES_PORT:?Must set FLOWETL_POSTGRES_PORT env var}:5432
    environment:
      POSTGRES_USER: ${FLOWETL_POSTGRES_USER:?Must set FLOWETL_POSTGRES_USER env var}
      POSTGRES_PASSWORD: ${FLOWETL_POSTGRES_PASSWORD:?Must set FLOWETL_POSTGRES_PASSWORD env var}
      POSTGRES_DB: ${FLOWETL_POSTGRES_DB:?Must set FLOWETL_POSTGRES_DB env var}
    healthcheck:
      test: ["CMD-SHELL", "pg_isready", "-U", "airflow"]
      interval: 5s
      timeout: 30s
      retries: 10
    networks:
      flowetl:
        aliases:
          - flowetl_db

  flowetl_redis:
    container_name: flowetl_redis
    image: bitnami/redis:latest
    environment:
      - REDIS_PASSWORD=${FLOWETL_REDIS_PASSWORD:?Must set FLOWETL_REDIS_PASSWORD env var}
    expose:
      - 6379
    healthcheck:
      test: [ "CMD", "redis-cli", "ping" ]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: always
    networks:
      flowetl:
        aliases:
          - flowetl_redis

  flowetl_worker:
    <<: *flowetl-common
    command: celery worker
    deploy:
      replicas: ${FLOWETL_WORKER_COUNT:-1}
    networks:
      - flowetl
      - db

  flowetl_webserver:
    container_name: flowetl_webserver
    <<: *flowetl-common
    ports:
      - ${FLOWETL_PORT:?Must set FLOWETL_PORT env var}:8080
    command: webserver
    networks:
      - flowetl
    environment:
      <<: *flowetl-envs

  flowetl_flower:
    container_name: flowetl_flower
    <<: *flowetl-common
    command: celery flower
    ports:
      - ${FLOWETL_CELERY_PORT:?Must set FLOWETL_CELERY_PORT env var}:5555
    restart: always
    networks:
      - flowetl

  flowetl_triggerer:
    container_name: flowetl_triggerer
    <<: *flowetl-common
    command: triggerer
    restart: always
    networks:
      - flowetl

  flowetl_init:
    container_name: flowetl_init
    <<: *flowetl-common
    environment:
      <<: *flowetl-envs
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_PASSWORD: ${FLOWETL_WEBSERVER_PASSWORD:?Must set FLOWETL_WEBSERVER_PASSWORD env var}
    command: version
    networks:
      - flowetl
    depends_on:
      - flowetl_db
      - flowetl_redis
    restart: on-failure