diff --git a/ansible/deploy_dockercloud.yml b/ansible/deploy_dockercloud.yml index dcd1ba8..07395ff 100644 --- a/ansible/deploy_dockercloud.yml +++ b/ansible/deploy_dockercloud.yml @@ -15,7 +15,7 @@ - name: update Docker Cloud stack shell: docker-cloud stack update --sync $DOCKERCLOUD_STACK -f {{ dockercloud_conf }} environment: - EXECUTOR: Local + EXECUTOR: Celery AIRFLOW_ENABLE_AUTH: True STATSD_ENABLED: True REMOTE_BASE_LOG_FOLDER: "{{ remote_base_log_folder }}" diff --git a/ansible/deploy_local.yml b/ansible/deploy_local.yml index 25e7c7b..3a3b088 100644 --- a/ansible/deploy_local.yml +++ b/ansible/deploy_local.yml @@ -13,7 +13,7 @@ - docker-compose -f {{ compose_conf[0] }} -f {{ compose_conf[1] }} down -v --remove-orphans - docker-compose -f {{ compose_conf[0] }} -f {{ compose_conf[1] }} up -d environment: - EXECUTOR: Local + EXECUTOR: Celery AIRFLOW_ENABLE_AUTH: False STATSD_ENABLED: False FERNET_KEY: 46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= diff --git a/ansible/files/docker-cloud.yml b/ansible/files/docker-cloud.yml index e746e4e..679910c 100644 --- a/ansible/files/docker-cloud.yml +++ b/ansible/files/docker-cloud.yml @@ -1,3 +1,7 @@ +redis: + image: 'redis:3.2.7' + restart: always + webserver: image: 'opentrials/opentrials-airflow:latest' restart: always @@ -19,6 +23,7 @@ webserver: - '80:8080' links: - datadog-agent:statsd + - redis command: webserver scheduler: @@ -42,7 +47,58 @@ scheduler: - SMTP_PASSWORD links: - datadog-agent:statsd - command: scheduler + - redis + command: scheduler --num-runs 10 + +flower: + image: 'opentrials/opentrials-airflow:latest' + restart: always + environment: + - EXECUTOR + - AIRFLOW_ENABLE_AUTH + - STATSD_ENABLED + - REMOTE_BASE_LOG_FOLDER + - REMOTE_LOG_CONN_ID + - FERNET_KEY + - DB_URI + - DB_USER + - DB_PASSWORD + - DB_TABLE + - SMTP_HOST + - SMTP_USER + - SMTP_PASSWORD + links: + - datadog-agent:statsd + - redis + ports: + - '5555:5555' + command: flower + +worker: + image: 'opentrials/opentrials-airflow:latest' + restart: always + ports: + - '8793:8793' + volumes: + - '/var/run/docker.sock:/var/run/docker.sock' + environment: + - EXECUTOR + - AIRFLOW_ENABLE_AUTH + - STATSD_ENABLED + - REMOTE_BASE_LOG_FOLDER + - REMOTE_LOG_CONN_ID + - FERNET_KEY + - DB_URI + - DB_USER + - DB_PASSWORD + - DB_TABLE + - SMTP_HOST + - SMTP_USER + - SMTP_PASSWORD + links: + - datadog-agent:statsd + - redis + command: worker datadog-agent: image: 'datadog/docker-dd-agent:latest' diff --git a/ansible/files/docker-compose-local.yml b/ansible/files/docker-compose-local.yml index ed8e65b..3f90a81 100644 --- a/ansible/files/docker-compose-local.yml +++ b/ansible/files/docker-compose-local.yml @@ -1,21 +1,29 @@ version: '2' services: postgres: - image: postgres:9.5 - restart: always - volumes: - - /var/lib/postgresql/data:/var/lib/postgresql/data - ports: - - "5432:5432" - environment: - POSTGRES_USER: airflow - POSTGRES_PASSWORD: airflow - POSTGRES_DB: airflow + image: postgres:9.5 + restart: always + volumes: + - /var/lib/postgresql/data:/var/lib/postgresql/data + ports: + - '5432:5432' + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow webserver: - links: - - postgres:postgres + links: + - postgres scheduler: - links: - - postgres:postgres + links: + - postgres + + worker: + links: + - postgres + + flower: + links: + - postgres diff --git a/ansible/files/docker-compose.yml b/ansible/files/docker-compose.yml index 65e9c91..b04a977 100644 --- a/ansible/files/docker-compose.yml +++ b/ansible/files/docker-compose.yml @@ -1,8 +1,14 @@ version: '2' services: + redis: + image: 'redis:3.2.7' + restart: always + webserver: - image: opentrials/opentrials-airflow + image: opentrials/opentrials-airflow:latest restart: always + depends_on: + - redis environment: EXECUTOR: AIRFLOW_ENABLE_AUTH: @@ -18,16 +24,60 @@ services: SMTP_USER: SMTP_PASSWORD: ports: - - "8080:8080" + - '8080:8080' command: webserver scheduler: - image: opentrials/opentrials-airflow + image: opentrials/opentrials-airflow:latest + restart: always + depends_on: + - redis + environment: + EXECUTOR: + AIRFLOW_ENABLE_AUTH: + STATSD_ENABLED: + REMOTE_BASE_LOG_FOLDER: + REMOTE_LOG_CONN_ID: + FERNET_KEY: + DB_URI: + DB_USER: + DB_PASSWORD: + DB_TABLE: + SMTP_HOST: + SMTP_USER: + SMTP_PASSWORD: + command: ['scheduler', '--num_runs', '10'] + + flower: + image: opentrials/opentrials-airflow:latest + restart: always + depends_on: + - redis + environment: + EXECUTOR: + AIRFLOW_ENABLE_AUTH: + STATSD_ENABLED: + REMOTE_BASE_LOG_FOLDER: + REMOTE_LOG_CONN_ID: + FERNET_KEY: + DB_URI: + DB_USER: + DB_PASSWORD: + DB_TABLE: + SMTP_HOST: + SMTP_USER: + SMTP_PASSWORD: + ports: + - '5555:5555' + command: flower + + worker: + image: opentrials/opentrials-airflow:latest restart: always volumes: - '/var/run/docker.sock:/var/run/docker.sock' depends_on: - - webserver + - scheduler environment: EXECUTOR: AIRFLOW_ENABLE_AUTH: @@ -42,4 +92,6 @@ services: SMTP_HOST: SMTP_USER: SMTP_PASSWORD: - command: ['scheduler', '--num_runs', '10'] + ports: + - '8793:8793' + command: worker