Move from rabbitmq to Redis

Dockerfile

@@ -1,4 +1,4 @@
-# VERSION 1.7.1.3-5
+# VERSION 1.7.1.3-6
 # AUTHOR: Matthieu "Puckel_" Roisil
 # DESCRIPTION: Basic Airflow container
 # BUILD: docker build --rm -t puckel/docker-airflow .
@@ -33,6 +33,7 @@ RUN set -ex \
         build-essential \
         libblas-dev \
         liblapack-dev \
+        libpq-dev \
     ' \
     && echo "deb http://http.debian.net/debian jessie-backports main" >/etc/apt/sources.list.d/backports.list \
     && apt-get update -yqq \
@@ -43,22 +44,20 @@ RUN set -ex \
         curl \
         netcat \
         locales \
-    && apt-get install -yqq -t jessie-backports python-requests libpq-dev \
+    && apt-get install -yqq -t jessie-backports python-requests \
     && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
     && locale-gen \
     && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
     && useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
+    && python -m pip install -U pip \
     && pip install Cython \
     && pip install pytz==2015.7 \
-    && pip install cryptography \
     && pip install pyOpenSSL \
     && pip install ndg-httpsclient \
     && pip install pyasn1 \
-    && pip install psycopg2 \
-    && pip install pandas==0.18.1 \
-    && pip install celery==3.1.23 \
-    && pip install airflow[celery,postgres,hive,hdfs,jdbc]==$AIRFLOW_VERSION \
-    && apt-get remove --purge -yqq $buildDeps libpq-dev \
+    && pip install airflow[crypto,celery,postgres,hive,hdfs,jdbc]==$AIRFLOW_VERSION \
+    && pip install celery[redis]==3.1.17 \
+    && apt-get remove --purge -yqq $buildDeps \
     && apt-get clean \
     && rm -rf \
         /var/lib/apt/lists/* \

README.md

@@ -8,7 +8,7 @@ This repository contains **Dockerfile** of [airflow](https://github.com/apache/i
 
 ## Informations
 
-* Based on Debian Jessie official Image [debian:jessie](https://registry.hub.docker.com/_/debian/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [RabbitMQ](https://hub.docker.com/_/rabbitmq/) as queue
+* Based on Debian Jessie official Image [debian:jessie](https://registry.hub.docker.com/_/debian/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue
 * Install [Docker](https://www.docker.com/)
 * Install [Docker Compose](https://docs.docker.com/compose/install/)
 * Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/airflow)
@@ -48,8 +48,8 @@ NB : If you don't want to have DAGs example loaded (default=True), you've to set
         docker run -d -p 8080:8080 -e LOAD_EX=n puckel/docker-airflow
 
 If you want to use Ad hoc query, make sure you've configured connections:
-Go to Admin -> Connections and Edit "mysql_default" set this values (equivalent to values in airflow.cfg/docker-compose.yml) :
-- Host : mysql
+Go to Admin -> Connections and Edit "postgres_default" set this values (equivalent to values in airflow.cfg/docker-compose*.yml) :
+- Host : postgres
 - Schema : airflow
 - Login : airflow
 - Password : airflow
@@ -71,7 +71,6 @@ Check [Airflow Documentation](https://pythonhosted.org/airflow/)
 
 - Airflow: [localhost:8080](http://localhost:8080/)
 - Flower: [localhost:5555](http://localhost:5555/)
-- RabbitMQ: [localhost:15672](http://localhost:15672/)
 
 When using OSX with boot2docker, use: open http://$(boot2docker ip):8080
 

config/airflow.cfg

@@ -151,10 +151,10 @@ worker_log_server_port = 8793
 # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
 # a sqlalchemy database. Refer to the Celery documentation for more
 # information.
-broker_url = amqp://airflow:airflow@rabbitmq:5672/airflow
+broker_url = redis://redis:6379/1
 
 # Another key Celery setting
-celery_result_backend = amqp://airflow:airflow@rabbitmq:5672/airflow
+celery_result_backend = redis://redis:6379/1
 
 # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
 # it `airflow flower`. This defines the port that Celery Flower runs on

docker-compose-CeleryExecutor.yml

@@ -1,15 +1,7 @@
 version: '2'
 services:
-    rabbitmq:
-        image: rabbitmq:3.6-management
-        restart: always
-        ports:
-            - "15672:15672"
-            - "5672:5672"
-        environment:
-            - RABBITMQ_DEFAULT_USER=airflow
-            - RABBITMQ_DEFAULT_PASS=airflow
-            - RABBITMQ_DEFAULT_VHOST=airflow
+    redis:
+        image: 'redis:3.2.7'
 
     postgres:
         image: postgres:9.6
@@ -19,11 +11,11 @@ services:
             - POSTGRES_DB=airflow
 
     webserver:
-        image: puckel/docker-airflow:1.7.1.3-5
+        image: puckel/docker-airflow:1.7.1.3-6
         restart: always
         depends_on:
             - postgres
-            - rabbitmq
+            - redis
         environment:
             # - LOAD_EX=n
             - FERNET_KEY=9IoTgQ_EJ0hCsamBU3Mctc7F9OkODnndOKCwkwXCAA
@@ -35,18 +27,18 @@ services:
         command: webserver
 
     flower:
-        image: puckel/docker-airflow:1.7.1.3-5
+        image: puckel/docker-airflow:1.7.1.3-6
         restart: always
         depends_on:
-            - rabbitmq
+            - redis
         environment:
             - EXECUTOR=Celery
         ports:
             - "5555:5555"
         command: flower
 
     scheduler:
-        image: puckel/docker-airflow:1.7.1.3-5
+        image: puckel/docker-airflow:1.7.1.3-6
         restart: always
         depends_on:
             - webserver
@@ -59,7 +51,7 @@ services:
         command: scheduler -n 5
 
     worker:
-        image: puckel/docker-airflow:1.7.1.3-5
+        image: puckel/docker-airflow:1.7.1.3-6
         restart: always
         depends_on:
             - scheduler

docker-compose-LocalExecutor.yml

@@ -8,7 +8,7 @@ services:
             - POSTGRES_DB=airflow
 
     webserver:
-        image: puckel/docker-airflow:1.7.1.3-5
+        image: puckel/docker-airflow:1.7.1.3-6
         restart: always
         depends_on:
             - postgres
@@ -23,7 +23,7 @@ services:
         command: webserver
 
     scheduler:
-        image: puckel/docker-airflow:1.7.1.3-5
+        image: puckel/docker-airflow:1.7.1.3-6
         restart: always
         depends_on:
             - webserver

script/entrypoint.sh

@@ -5,10 +5,9 @@ CMD="airflow"
 TRY_LOOP="10"
 POSTGRES_HOST="postgres"
 POSTGRES_PORT="5432"
-RABBITMQ_HOST="rabbitmq"
-RABBITMQ_CREDS="airflow:airflow"
+REDIS_HOST="redis"
+REDIS_PORT="6379"
 : ${FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print FERNET_KEY")}
-# FERNET_KEY=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print FERNET_KEY")
 
 # Load DAGs exemples (default: Yes)
 if [ "x$LOAD_EX" = "xn" ]; then
@@ -33,7 +32,7 @@ if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] ; the
       exit 1
     fi
     echo "$(date) - waiting for ${POSTGRES_HOST}:${POSTGRES_PORT}... $i/$TRY_LOOP"
-    sleep 5
+    sleep 10
   done
   if [ "$1" = "webserver" ]; then
     echo "Initialize database..."
@@ -42,19 +41,18 @@ if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] ; the
   sleep 5
 fi
 
-# If we use docker-compose, we use Celery (rabbitmq container).
+# If we use docker-compose, we use Celery.
 if [ "x$EXECUTOR" = "xCelery" ]
 then
-# wait for rabbitmq
   if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] || [ "$1" = "flower" ] ; then
     j=0
-    while ! curl -sI -u $RABBITMQ_CREDS http://$RABBITMQ_HOST:15672/api/whoami |grep '200 OK'; do
+    while ! nc -z $REDIS_HOST $REDIS_PORT >/dev/null 2>&1 < /dev/null; do
       j=$((j+1))
       if [ $j -ge $TRY_LOOP ]; then
-        echo "$(date) - $RABBITMQ_HOST still not reachable, giving up"
+        echo "$(date) - $REDIS_HOST still not reachable, giving up"
         exit 1
       fi
-      echo "$(date) - waiting for RabbitMQ... $j/$TRY_LOOP"
+      echo "$(date) - waiting for Redis... $j/$TRY_LOOP"
       sleep 5
     done
   fi