docker-compose-CeleryExecutor.yml

version: '2.1'
services:
    redis:
        image: 'redis:5.0.5'
        # command: redis-server --requirepass redispass

    postgres:
        image: postgres:9.6
        environment:
            - POSTGRES_USER=airflow
            - POSTGRES_PASSWORD=airflow
            - POSTGRES_DB=airflow
        # Uncomment these lines to persist data on the local filesystem.
        #     - PGDATA=/var/lib/postgresql/data/pgdata
        # volumes:
        #     - ./pgdata:/var/lib/postgresql/data/pgdata

    webserver:
        image: zdp-airflow:zmaster
        depends_on:
            - postgres
            - redis
            - initdb
        environment:
            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
            - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
            - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
            - AIRFLOW__CORE__LOAD_EXAMPLES=False
        volumes:
            - ./dags:/home/airflow/airflow/dags
            # Uncomment to include custom plugins
        ports:
            - "8080:8080"
        command: webserver -w 16
        healthcheck:
            test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
            interval: 30s
            timeout: 30s
            retries: 3

    initdb:
        image: zdp-airflow:zmaster
        depends_on:
            - postgres
            - redis
        volumes:
            - ./dags:/home/airflow/airflow/dags
            # Uncomment to include custom plugins
            # - ./plugins:/usr/local/airflow/plugins
        environment:
            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
            - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
            - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
            - AIRFLOW__CORE__LOAD_EXAMPLES=False
        command: initdb

    flower:
        image: zdp-airflow:zmaster
        restart: always
        depends_on:
            - redis
        environment:
            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
            - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
            - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
            - AIRFLOW__CORE__LOAD_EXAMPLES=False
        ports:
            - "5555:5555"
        command: flower

    scheduler:
        image: zdp-airflow:zmaster
        restart: always
        depends_on:
            - webserver
        volumes:
            - ./dags:/home/airflow/airflow/dags
            # Uncomment to include custom plugins
            # - ./plugins:/usr/local/airflow/plugins
        environment:
            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
            - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
            - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
            - AIRFLOW__CORE__LOAD_EXAMPLES=False
        command: scheduler -r 30

    worker:
        image: zdp-airflow:zmaster
        restart: always
        depends_on:
            - scheduler
        volumes:
            - ./dags:/home/airflow/airflow/dags
            # Uncomment to include custom plugins
            # - ./plugins:/usr/local/airflow/plugins
        environment:
            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/airflow
            - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
            - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
            - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
            - AIRFLOW__CORE__LOAD_EXAMPLES=False
        command: worker