docker-compose-base.yml

# ------------------------------------------------------------------------------
# Base settings for these containers:
#
#  * Databases + Minio + Keycloak + NGINX containers (use same settings in all files)
#  * Aether Kernel
#  * Entity Extractor
#  * ODK Module
#  * Aether UI & UI Assets
#  * Zookeeper & Kafka
#  * Aether Kafka Producer
#
# These container will be extended in the other DC files with dependencies and networks.
# Volumes and environment variables can be overridden in those files too.
# See more in: https://docs.docker.com/compose/extends/
# ------------------------------------------------------------------------------

services:

  # ---------------------------------
  # Auxiliary containers
  # ---------------------------------

  postgres-base:
    image: postgres:15-alpine
    environment:
      PGUSER: postgres
      POSTGRES_HOST_AUTH_METHOD: trust
    healthcheck:
      test: pg_isready
      interval: 2s
      timeout: 3s
      retries: 30

  redis-base:
    image: redis:alpine
    command: >
      redis-server
      --requirepass ${REDIS_PASSWORD}
      --notify-keyspace-events KEA
      --appendonly yes
      --auto-aof-rewrite-percentage 100
      --auto-aof-rewrite-min-size 64mb
    healthcheck:
      test: redis-cli ping
      interval: 2s
      timeout: 3s
      retries: 30

  nginx-base:
    image: nginx:alpine
    volumes:
      # local setup
      - ./local-setup/nginx/nginx.conf:/etc/nginx/nginx.conf
      - ./local-setup/nginx/sites-enabled:/etc/nginx/sites-enabled

      # nginx log files
      - ./.persistent_data/nginx:/var/log/nginx

      # static folders per container
      - ./.persistent_data/static:/static
    ports:
      - 80:80
    extra_hosts:
      - ${NETWORK_DOMAIN}:127.0.0.1
    healthcheck:
      test: nginx -t
      interval: 5s
      timeout: 3s
      retries: 30

  minio-base:
    image: minio/minio:latest
    volumes:
      - ./.persistent_data/media:/data
    environment:
      MINIO_ACCESS_KEY: ${MINIO_STORAGE_ACCESS_KEY}
      MINIO_SECRET_KEY: ${MINIO_STORAGE_SECRET_KEY}
    command: minio server --quiet --address minio:9090 /data

  keycloak-base:
    image: jboss/keycloak:${KEYCLOAK_VERSION:-latest}
    environment:
      DB_VENDOR: POSTGRES
      DB_ADDR: db
      DB_DATABASE: keycloak
      DB_USER: keycloak
      DB_PASSWORD: ${KEYCLOAK_DB_PASSWORD}

      KEYCLOAK_USER: ${KEYCLOAK_ADMIN_USERNAME}
      KEYCLOAK_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}


  # ---------------------------------
  # Aether Kernel
  # ---------------------------------

  kernel-base:
    image: aether-kernel
    build: ./aether-kernel
    stdin_open: true
    tty: true
    environment:
      CSRF_COOKIE_DOMAIN: ${NETWORK_DOMAIN}
      DJANGO_SECRET_KEY: ${KERNEL_DJANGO_SECRET_KEY}
      LOGGING_FORMATTER: verbose
      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
      HTML_SELECT_CUTOFF: 10
      PROFILING_ENABLED: ${ENABLE_PROFILING}

      KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL}
      KEYCLOAK_CLIENT_ID: ${KEYCLOAK_AETHER_CLIENT}
      KEYCLOAK_BEHIND_SCENES: ${KEYCLOAK_BEHIND_SCENES}

      MULTITENANCY: ${MULTITENANCY}
      DEFAULT_REALM: ${DEFAULT_REALM}
      REALM_COOKIE: ${REALM_COOKIE}

      BUCKET_NAME: kernel
      DJANGO_STORAGE_BACKEND: minio
      MINIO_STORAGE_ACCESS_KEY: ${MINIO_STORAGE_ACCESS_KEY}
      MINIO_STORAGE_SECRET_KEY: ${MINIO_STORAGE_SECRET_KEY}
      MINIO_STORAGE_ENDPOINT: minio:9090
      MINIO_STORAGE_MEDIA_URL: http://minio:9090/kernel
      MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET: "true"
      MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY: "true"

      APP_NAME: Aether Kernel
      APP_URL: /kernel
      STATIC_URL: /static/kernel/

      ADMIN_USERNAME: ${KERNEL_ADMIN_USERNAME}
      ADMIN_PASSWORD: ${KERNEL_ADMIN_PASSWORD}
      ADMIN_TOKEN: ${KERNEL_ADMIN_TOKEN}

      DB_NAME: aether
      PGHOST: db
      PGPASSWORD: ${KERNEL_DB_PASSWORD}
      PGPORT: 5432
      PGUSER: postgres

      ENABLE_CONNECTION_POOL: ${ENABLE_DB_POOL}
      DB_POOL_INTERNAL: ${ENABLE_DB_POOL}

      DJANGO_USE_CACHE: ${ENABLE_CACHE}
      REDIS_DJANGO_CACHE: ${ENABLE_CACHE}
      REDIS_SESSION_CACHE: ${ENABLE_CACHE}
      REDIS_HOST: redis
      REDIS_PASSWORD: ${REDIS_PASSWORD}
      REDIS_PORT: 6379

      REDIS_DB: 0
      REDIS_DB_CACHEOPS: 1
      REDIS_DB_DJANGO: 2
      REDIS_DB_SESSION: 3

      EXPORT_CSV_ESCAPE: "\\"
      EXPORT_CSV_QUOTES: "\""
      EXPORT_CSV_SEPARATOR: ","
      EXPORT_DATA_FORMAT: split
      EXPORT_HEADER_CONTENT: labels
      EXPORT_HEADER_SEPARATOR: /
      EXPORT_HEADER_SHORTEN: "no"

      WRITE_ENTITIES_TO_REDIS: ${WRITE_ENTITIES_TO_REDIS}

      WEB_SERVER_PORT: 8100

      PYTHONPYCACHEPREFIX: "/var/tmp/cache/cpython/"
    volumes:
      - ./aether-kernel:/code
      # -------------------------------------------------------------
      # to speed up SDK development changes
      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/var/run/aether/venv/lib/python3.8/site-packages/aether/sdk
      # to speed up Aether Python Library development changes
      # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/var/run/aether/venv/lib/python3.8/site-packages/aether/python
    command: start_dev


  # ---------------------------------
  # Entity Extractor
  # ---------------------------------

  exm-base:
    image: aether-exm
    build: ./aether-entity-extraction-module
    stdin_open: true
    tty: true
    environment:
      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}

      DEFAULT_REALM: ${DEFAULT_REALM}
      REALM_COOKIE: ${REALM_COOKIE}

      # use internal url, this happens behind the curtains
      AETHER_KERNEL_URL: http://kernel:8100/kernel
      AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}

      REDIS_DB: 0
      REDIS_HOST: redis
      REDIS_PASSWORD: ${REDIS_PASSWORD}
      REDIS_PORT: 6379

      PYTHONPYCACHEPREFIX: "/var/tmp/cache/cpython/"
    volumes:
      - ./aether-entity-extraction-module:/code
      # to speed up Aether Python Library development changes
      # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/var/run/aether/venv/lib/python3.8/site-packages/aether/python
    command: start


  # ---------------------------------
  # ODK module
  # ---------------------------------

  odk-base:
    image: aether-odk
    build: ./aether-odk-module
    stdin_open: true
    tty: true
    environment:
      CSRF_COOKIE_DOMAIN: ${NETWORK_DOMAIN}
      DJANGO_SECRET_KEY: ${ODK_DJANGO_SECRET_KEY}
      LOGGING_FORMATTER: verbose
      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
      HTML_SELECT_CUTOFF: 10
      PROFILING_ENABLED: ${ENABLE_PROFILING}

      KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL}
      KEYCLOAK_CLIENT_ID: ${KEYCLOAK_AETHER_CLIENT}
      KEYCLOAK_BEHIND_SCENES: ${KEYCLOAK_BEHIND_SCENES}

      MULTITENANCY: ${MULTITENANCY}
      DEFAULT_REALM: ${DEFAULT_REALM}
      REALM_COOKIE: ${REALM_COOKIE}

      BUCKET_NAME: odk
      DJANGO_STORAGE_BACKEND: minio
      MINIO_STORAGE_ACCESS_KEY: ${MINIO_STORAGE_ACCESS_KEY}
      MINIO_STORAGE_SECRET_KEY: ${MINIO_STORAGE_SECRET_KEY}
      MINIO_STORAGE_ENDPOINT: minio:9090
      MINIO_STORAGE_MEDIA_URL: http://minio:9090/odk
      MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET: "true"
      MINIO_STORAGE_AUTO_CREATE_MEDIA_POLICY: "true"

      APP_NAME: Aether ODK
      APP_URL: /odk
      STATIC_URL: /static/odk/

      ADMIN_USERNAME: ${ODK_ADMIN_USERNAME}
      ADMIN_PASSWORD: ${ODK_ADMIN_PASSWORD}
      ADMIN_TOKEN: ${ODK_ADMIN_TOKEN}

      AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}
      AETHER_KERNEL_URL: http://${NETWORK_DOMAIN}/kernel
      AETHER_KERNEL_URL_TEST: http://kernel-test:9100

      DB_NAME: odk
      PGHOST: db
      PGPASSWORD: ${ODK_DB_PASSWORD}
      PGPORT: 5432
      PGUSER: postgres

      ENABLE_CONNECTION_POOL: ${ENABLE_DB_POOL}
      DB_POOL_INTERNAL: ${ENABLE_DB_POOL}

      DJANGO_USE_CACHE: ${ENABLE_CACHE}
      REDIS_DJANGO_CACHE: ${ENABLE_CACHE}
      REDIS_SESSION_CACHE: ${ENABLE_CACHE}
      REDIS_HOST: redis
      REDIS_PASSWORD: ${REDIS_PASSWORD}
      REDIS_PORT: 6379

      REDIS_DB: 4
      REDIS_DB_CACHEOPS: 5
      REDIS_DB_DJANGO: 6
      REDIS_DB_SESSION: 7

      WEB_SERVER_PORT: 8102

      PYTHONPYCACHEPREFIX: "/var/tmp/cache/cpython/"
    volumes:
      - ./aether-odk-module:/code
      # -------------------------------------------------------------
      # to speed up SDK development changes
      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/var/run/aether/venv/lib/python3.8/site-packages/aether/sdk
    command: start_dev


  # ---------------------------------
  # Aether UI
  # ---------------------------------

  ui-base:
    image: aether-ui
    build: ./aether-ui
    stdin_open: true
    tty: true
    environment:
      CSRF_COOKIE_DOMAIN: ${NETWORK_DOMAIN}
      DJANGO_SECRET_KEY: ${UI_DJANGO_SECRET_KEY}
      LOGGING_FORMATTER: verbose
      LOGGING_LEVEL: ${LOGGING_LEVEL:-ERROR}
      HTML_SELECT_CUTOFF: 10
      PROFILING_ENABLED: ${ENABLE_PROFILING}

      KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL}
      KEYCLOAK_CLIENT_ID: ${KEYCLOAK_AETHER_CLIENT}
      KEYCLOAK_BEHIND_SCENES: ${KEYCLOAK_BEHIND_SCENES}

      MULTITENANCY: ${MULTITENANCY}
      DEFAULT_REALM: ${DEFAULT_REALM}
      REALM_COOKIE: ${REALM_COOKIE}

      APP_NAME: Aether
      STATIC_URL: /static/ui/

      ADMIN_USERNAME: ${UI_ADMIN_USERNAME}
      ADMIN_PASSWORD: ${UI_ADMIN_PASSWORD}

      AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}
      AETHER_KERNEL_URL: http://${NETWORK_DOMAIN}/kernel
      AETHER_KERNEL_URL_TEST: http://kernel-test:9100

      DB_NAME: ui
      PGHOST: db
      PGPASSWORD: ${UI_DB_PASSWORD}
      PGPORT: 5432
      PGUSER: postgres

      ENABLE_CONNECTION_POOL: ${ENABLE_DB_POOL}
      DB_POOL_INTERNAL: ${ENABLE_DB_POOL}

      DJANGO_USE_CACHE: ${ENABLE_CACHE}
      REDIS_DJANGO_CACHE: ${ENABLE_CACHE}
      REDIS_SESSION_CACHE: ${ENABLE_CACHE}
      REDIS_HOST: redis
      REDIS_PASSWORD: ${REDIS_PASSWORD}
      REDIS_PORT: 6379

      REDIS_DB: 8
      REDIS_DB_CACHEOPS: 9
      REDIS_DB_DJANGO: 10
      REDIS_DB_SESSION: 11

      # use this value with "start_dev" command and webpack otherwise comment it out
      WEBPACK_STATS_FILE: /code/aether/ui/assets/bundles/webpack-stats.json

      WEB_SERVER_PORT: 8104

      PYTHONPYCACHEPREFIX: "/var/tmp/cache/cpython/"
    volumes:
      - ./aether-ui:/code
      # -------------------------------------------------------------
      # to speed up SDK development changes
      # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/var/run/aether/venv/lib/python3.8/site-packages/aether/sdk
    command: start_dev

  ui-assets-base:
    image: aether-ui-assets
    build: ./aether-ui/aether/ui/assets
    stdin_open: true
    tty: true
    volumes:
      - ./aether-ui/aether/ui/assets/:/code/assets
    command: start_dev


  # ---------------------------------
  # Kafka & Zookeeper
  # ---------------------------------

  zookeeper-base:
    image: confluentinc/cp-zookeeper:${CONFLUENTINC_VERSION:-latest}
    environment:
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_TICK_TIME: 2000

  kafka-base:
    image: confluentinc/cp-kafka:${CONFLUENTINC_VERSION:-latest}
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      ADVERTISED_HOST_NAME: kafka


  # ---------------------------------
  # Aether Kafka Producer
  # ---------------------------------

  producer-base:
    image: aether-producer
    build: ./aether-producer
    stdin_open: true
    environment:
      PYTHONUNBUFFERED: 1
      LOG_LEVEL: ${LOGGING_LEVEL:-ERROR}

      PRODUCER_ADMIN_USER: ${PRODUCER_ADMIN_USER}
      PRODUCER_ADMIN_PW: ${PRODUCER_ADMIN_PW}

      # These variables will override the ones indicated in the settings file
      AETHER_KERNEL_TOKEN: ${KERNEL_ADMIN_TOKEN}
      AETHER_KERNEL_URL: http://kernel:8100/kernel
      DEFAULT_REALM: ${DEFAULT_REALM}
      REALM_COOKIE: ${REALM_COOKIE}

      POSTGRES_HOST: db
      POSTGRES_DBNAME: aether
      POSTGRES_USER: postgres
      POSTGRES_PORT: 5432
      POSTGRES_PASSWORD: ${KERNEL_DB_PASSWORD}

      OFFSET_DB_HOST: db
      OFFSET_DB_USER: postgres
      OFFSET_DB_PORT: 5432
      OFFSET_DB_PASSWORD: ${PRODUCER_DB_PASSWORD}
      OFFSET_DB_NAME: producer_offset_db

      PYTHONPYCACHEPREFIX: "/var/tmp/cache/cpython/"
    volumes:
      - ./aether-producer:/code
    command: start