docker-compose.yaml

version: '3'

services:
  source_postgres:
    image: postgres:15
    ports:
      - "5433:5432"
    networks:
      - elt_network
    environment:
      POSTGRES_DB: source_db
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: secret
    volumes:
      - ./source_db_init/init.sql:/docker-entrypoint-initdb.d/init.sql

  destination_postgres:
    image: postgres:15
    ports:
      - "5434:5432"
    networks:
      - elt_network
    environment:
      POSTGRES_DB: destination_db
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: secret

  # elt_script:
  #   build:
  #     context: ./elt # Directory containing the Dockerfile and elt_script.py
  #     dockerfile: Dockerfile # Name of the Dockerfile, if it's something other than "Dockerfile", specify here
  #   command: [ "python", "elt_script.py" ]
  #   networks:
  #     - elt_network
  #   depends_on:
  #     - source_postgres
  #     - destination_postgres
  
  # dbt:
  #   image: ghcr.io/dbt-labs/dbt-postgres:1.4.7
  #   command: 
  #     [
  #       "run",
  #       "--profiles-dir",
  #       "/root",
  #       "--project-dir",
  #       "/dbt"
  #     ]
  #   networks:
  #     - elt_network
  #   volumes:
  #     - ./custom_postgres:/dbt
  #     - ~/.dbt:/root
  #   depends_on:
  #     - elt_script
  #   environment:
  #     DBT_PROFILE: default
  #     DBT_TARGET: dev

  postgres:
    image: postgres:latest
    networks:
      - elt_network
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow

  init-airflow:
    image: apache/airflow:latest
    depends_on:
      - postgres
    networks:
      - elt_network
    environment:
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
    command: >
      bash -c "airflow db init && 
               airflow users create --username airflow --password password --firstname John --lastname Doe --role Admin --email admin@example.com"
  webserver:
    build: 
      context: .
      dockerfile: Dockerfile
    depends_on:
      - postgres
    networks:
      - elt_network
    extra_hosts:
      - "host.docker.internal:host-gateway"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./elt_script:/opt/airflow/elt_script
      - ./postgres_transformations:/opt/dbt
      - ~/.dbt:/root/.dbt
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - LOAD_EX=n
      - EXECUTOR=Local
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW_CONN_DESTINATION_POSTGRES=postgres://postgres:secret@destination_postgres:5434/destination_db
      - AIRFLOW__CORE__FERNET_KEY=h_OR946T_EuJD6LDMDD4zNOpUpjn4PVhd18h_91jICs=
      - AIRFLOW__WEBSERVER__DEFAULT_USER_USERNAME=airflow
      - AIRFLOW__WEBSERVER__DEFAULT_USER_PASSWORD=password
      - AIRFLOW_WWW_USER_USERNAME=airflow
      - AIRFLOW_WWW_USER_PASSWORD=password
      - AIRFLOW__WEBSERVER__SECRET_KEY=secret
    ports:
      - "8080:8080"
    command: webserver

  scheduler:
    build: 
      context: .
      dockerfile: Dockerfile
    depends_on:
      - postgres
    networks:
      - elt_network
    extra_hosts:
      - "host.docker.internal:host-gateway"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./elt_script:/opt/airflow/elt_script
      - ./postgres_transformations:/opt/dbt
      - ~/.dbt:/root/.dbt
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - LOAD_EX=n
      - EXECUTOR=Local
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW_CONN_DESTINATION_POSTGRES=postgres://postgres:secret@destination_postgres:5434/destination_db
      - AIRFLOW__CORE__FERNET_KEY=h_OR946T_EuJD6LDMDD4zNOpUpjn4PVhd18h_91jICs=
      - AIRFLOW__WEBSERVER__DEFAULT_USER_USERNAME=airflow
      - AIRFLOW__WEBSERVER__DEFAULT_USER_PASSWORD=password
      - AIRFLOW_WWW_USER_USERNAME=airflow
      - AIRFLOW_WWW_USER_PASSWORD=password
      - AIRFLOW__WEBSERVER__SECRET_KEY=secret
    command: scheduler



networks:
  elt_network:
    driver: bridge