[#812] Revert back to using CeleryExecutor instead of LocalExecutor

We solved the bug https://issues.apache.org/jira/browse/AIRFLOW-1131 by
creating a new DockerCLIOperator, so we can return using the CeleryExecutor.

Reverts 99106c3.

opentrials/opentrials#812

ansible/deploy_dockercloud.yml

@@ -15,7 +15,7 @@
     - name: update Docker Cloud stack
       shell: docker-cloud stack update --sync $DOCKERCLOUD_STACK -f {{ dockercloud_conf }}
       environment:
-        EXECUTOR: Local
+        EXECUTOR: Celery
         AIRFLOW_ENABLE_AUTH: True
         STATSD_ENABLED: True
         REMOTE_BASE_LOG_FOLDER: "{{ remote_base_log_folder }}"

ansible/deploy_local.yml

@@ -13,7 +13,7 @@
         - docker-compose -f {{ compose_conf[0] }} -f {{ compose_conf[1] }} down -v --remove-orphans
         - docker-compose -f {{ compose_conf[0] }} -f {{ compose_conf[1] }} up -d
       environment:
-        EXECUTOR: Local
+        EXECUTOR: Celery
         AIRFLOW_ENABLE_AUTH: False
         STATSD_ENABLED: False
         FERNET_KEY: 46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=

ansible/files/docker-cloud.yml

@@ -1,3 +1,7 @@
+redis:
+  image: 'redis:3.2.7'
+  restart: always
+
 webserver:
   image: 'opentrials/opentrials-airflow:latest'
   restart: always
@@ -19,6 +23,7 @@ webserver:
     - '80:8080'
   links:
     - datadog-agent:statsd
+    - redis
   command: webserver
 
 scheduler:
@@ -42,7 +47,58 @@ scheduler:
     - SMTP_PASSWORD
   links:
     - datadog-agent:statsd
-  command: scheduler
+    - redis
+  command: scheduler --num-runs 10
+
+flower:
+  image: 'opentrials/opentrials-airflow:latest'
+  restart: always
+  environment:
+    - EXECUTOR
+    - AIRFLOW_ENABLE_AUTH
+    - STATSD_ENABLED
+    - REMOTE_BASE_LOG_FOLDER
+    - REMOTE_LOG_CONN_ID
+    - FERNET_KEY
+    - DB_URI
+    - DB_USER
+    - DB_PASSWORD
+    - DB_TABLE
+    - SMTP_HOST
+    - SMTP_USER
+    - SMTP_PASSWORD
+  links:
+    - datadog-agent:statsd
+    - redis
+  ports:
+    - '5555:5555'
+  command: flower
+
+worker:
+  image: 'opentrials/opentrials-airflow:latest'
+  restart: always
+  ports:
+    - '8793:8793'
+  volumes:
+    - '/var/run/docker.sock:/var/run/docker.sock'
+  environment:
+    - EXECUTOR
+    - AIRFLOW_ENABLE_AUTH
+    - STATSD_ENABLED
+    - REMOTE_BASE_LOG_FOLDER
+    - REMOTE_LOG_CONN_ID
+    - FERNET_KEY
+    - DB_URI
+    - DB_USER
+    - DB_PASSWORD
+    - DB_TABLE
+    - SMTP_HOST
+    - SMTP_USER
+    - SMTP_PASSWORD
+  links:
+    - datadog-agent:statsd
+    - redis
+  command: worker
 
 datadog-agent:
   image: 'datadog/docker-dd-agent:latest'

ansible/files/docker-compose-local.yml

@@ -1,21 +1,29 @@
 version: '2'
 services:
   postgres:
-      image: postgres:9.5
-      restart: always
-      volumes:
-          - /var/lib/postgresql/data:/var/lib/postgresql/data
-      ports:
-          - "5432:5432"
-      environment:
-          POSTGRES_USER: airflow
-          POSTGRES_PASSWORD: airflow
-          POSTGRES_DB: airflow
+    image: postgres:9.5
+    restart: always
+    volumes:
+      - /var/lib/postgresql/data:/var/lib/postgresql/data
+    ports:
+      - '5432:5432'
+    environment:
+      POSTGRES_USER: airflow
+      POSTGRES_PASSWORD: airflow
+      POSTGRES_DB: airflow
 
   webserver:
-      links:
-          - postgres:postgres
+    links:
+      - postgres
 
   scheduler:
-      links:
-          - postgres:postgres
+    links:
+      - postgres
+
+  worker:
+    links:
+      - postgres
+
+  flower:
+    links:
+      - postgres

ansible/files/docker-compose.yml

@@ -1,8 +1,14 @@
 version: '2'
 services:
+    redis:
+      image: 'redis:3.2.7'
+      restart: always
+
     webserver:
-        image: opentrials/opentrials-airflow
+        image: opentrials/opentrials-airflow:latest
         restart: always
+        depends_on:
+            - redis
         environment:
             EXECUTOR:
             AIRFLOW_ENABLE_AUTH:
@@ -18,16 +24,60 @@ services:
             SMTP_USER:
             SMTP_PASSWORD:
         ports:
-            - "8080:8080"
+            - '8080:8080'
         command: webserver
 
     scheduler:
-        image: opentrials/opentrials-airflow
+        image: opentrials/opentrials-airflow:latest
+        restart: always
+        depends_on:
+            - redis
+        environment:
+            EXECUTOR:
+            AIRFLOW_ENABLE_AUTH:
+            STATSD_ENABLED:
+            REMOTE_BASE_LOG_FOLDER:
+            REMOTE_LOG_CONN_ID:
+            FERNET_KEY:
+            DB_URI:
+            DB_USER:
+            DB_PASSWORD:
+            DB_TABLE:
+            SMTP_HOST:
+            SMTP_USER:
+            SMTP_PASSWORD:
+        command: ['scheduler', '--num_runs', '10']
+
+    flower:
+        image: opentrials/opentrials-airflow:latest
+        restart: always
+        depends_on:
+            - redis
+        environment:
+            EXECUTOR:
+            AIRFLOW_ENABLE_AUTH:
+            STATSD_ENABLED:
+            REMOTE_BASE_LOG_FOLDER:
+            REMOTE_LOG_CONN_ID:
+            FERNET_KEY:
+            DB_URI:
+            DB_USER:
+            DB_PASSWORD:
+            DB_TABLE:
+            SMTP_HOST:
+            SMTP_USER:
+            SMTP_PASSWORD:
+        ports:
+            - '5555:5555'
+        command: flower
+
+    worker:
+        image: opentrials/opentrials-airflow:latest
         restart: always
         volumes:
             - '/var/run/docker.sock:/var/run/docker.sock'
         depends_on:
-            - webserver
+            - scheduler
         environment:
             EXECUTOR:
             AIRFLOW_ENABLE_AUTH:
@@ -42,4 +92,6 @@ services:
             SMTP_HOST:
             SMTP_USER:
             SMTP_PASSWORD:
-        command: ['scheduler', '--num_runs', '10']
+        ports:
+            - '8793:8793'
+        command: worker