docker-compose.yml

services:
  postgres:
    image: postgres
    container_name: migsbyu_pgdb
    restart: always
    ports:
      - "5432:5432"
    environment:
      POSTGRES_USER: pguser
      POSTGRES_PASSWORD: password
      POSTGRES_DB: prod
    command: ['postgres', '-c', 'wal_level=logical']
    volumes:
      - ./postgres/data:/var/lib/postgresql/data
      - ./postgres/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
    networks:
      - migsbyu_network
  api:
    build:
      context: ./api
      dockerfile: Dockerfile
    ports:
      - "8000:8000"
    image: migsbyu_api
    restart: "unless-stopped"
    depends_on:
      - postgres
    networks:
      - migsbyu_network
  frontend:
    build:
      context: ./frontend
      dockerfile: Dockerfile
    ports:
      - "8501:8501"
    image: migsbyu_frontend
    restart: "unless-stopped"
    depends_on:
      - postgres
      - api
      - cdc_system
    volumes:
      - ./data_lake:/data_lake/
    networks:
      - migsbyu_network

  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.1
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - '2181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    healthcheck:
      test: echo srvr | nc zookeeper 2181 || exit 1
      start_period: 10s
      retries: 20
      interval: 10s
    networks:
      - migsbyu_network
  kafka:
    image: confluentinc/cp-kafka:7.3.1
    depends_on:
      zookeeper:
        condition: service_healthy
    environment:
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9991
    ports:
      - '9092:9092'
    networks:
      - migsbyu_network
  debezium:
    image: debezium/connect:latest
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      GROUP_ID: 1
      CONFIG_STORAGE_TOPIC: connect_configs
      STATUS_STORAGE_TOPIC: connect_statuses
      OFFSET_STORAGE_TOPIC: connect_offsets
      KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      ENABLE_DEBEZIUM_SCRIPTING: 'true'
    depends_on: [kafka]
    ports:
      - 8083:8083
    healthcheck:
      test:
        [
          'CMD',
          'curl',
          '--silent',
          '--fail',
          '-X',
          'GET',
          'http://localhost:8083/connectors',
        ]
      start_period: 10s
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - migsbyu_network
  debezium-setup:
    image: ubuntu:latest
    command: >
      sh -c "apt-get update && apt-get install -y curl && /debezium/init_debezium.sh"
    volumes:
      - ./debezium:/debezium/
    networks:
      - migsbyu_network
    depends_on:
      - debezium
  schema-registry:
    image: confluentinc/cp-schema-registry:7.3.1
    environment:
      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
      - SCHEMA_REGISTRY_HOST_NAME=schema-registry
      - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081
      - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
    ports:
      - 8081:8081
    depends_on: [zookeeper, kafka]
    networks:
      - migsbyu_network

  spark-master:
    image: bitnami/spark:latest
    command: bin/spark-class org.apache.spark.deploy.master.Master
    ports:
      - "9090:8080"
      - "7077:7077"
    networks:
      - migsbyu_network
  spark-worker:
    image: bitnami/spark:latest
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_MODE: worker
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_MASTER_URL: spark://spark-master:7077
    networks:
      - migsbyu_network

  cdc_system:
    build:
      context: ./cdc_system
      dockerfile: Dockerfile
    image: migsbyu_cdc_system
    restart: "unless-stopped"
    depends_on:
      - kafka
      - debezium
      - zookeeper
      - schema-registry
      - spark-master
      - spark-worker
    networks:
      - migsbyu_network
    volumes:
      - ./cdc_system:/app
      - ./data_lake:/data_lake/

networks:
  migsbyu_network:
    driver: bridge
    name: migsbyu_network