docker-compose.yml

version: '3.7'
services:

    postgres:                                           # create postgres container
        image: postgres:9.6
        container_name: postgres_container
        environment:
            - POSTGRES_USER=airflow
            - POSTGRES_PASSWORD=airflow
            - POSTGRES_DB=airflow

    airflow:                                            # create airflow container
        build: './airflow_docker'                       # construct the container along the Dockerfile in this folder
        container_name: airflow_container
        restart: always
        depends_on:
            - postgres
        environment:
            - LOAD_EX=n
            - EXECUTOR=Local
        volumes:                                        # mount the following local folders
            - ./dags:/usr/local/airflow/dags
            - ./data:/usr/local/airflow/data
        ports:
            - "8080:8080"                               # expose port
        command: webserver
        healthcheck:
            test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
            interval: 30s
            timeout: 30s
            retries: 3

    zookeeper:                                          # create zookeeper container
        image: wurstmeister/zookeeper
        container_name: zookeeper_container
        ports:
            - "2181:2181"                               # expose port

    kafka:                                              # create an instance of a Kafka broker in a container
        image: wurstmeister/kafka
        container_name: kafka_container
        ports:
            - "9092:9092"                               # expose port
        environment:
            KAFKA_ADVERTISED_HOST_NAME: kafka                               # specify the docker host IP at which other containers can reach the broker
            KAFKA_CREATE_TOPICS: "transactions:1:1,locations:1:1"           # create a 2 topics  with 1 partition and 1 replica
            KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181                         # specify where the broker can reach Zookeeper
            KAFKA_LISTENERS: PLAINTEXT://kafka:9092                         # the list of addresses on which the Kafka broker will listen on for incoming connections.
            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092              # Kafka sends the value of this variable to clients during their connection. After receiving that value, the clients use it for sending/consuming records to/from the Kafka broker.y connect to it.
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock