[#763] Save Airflow logs to S3

This sets the remote logs URL to a S3 bucket, making sure our logs persist even
if Airflow's host machine is destroyed. There's a caveat, though: we can't use
buckets with dots in the name (e.g. "datastore.opentrials.net"). This is
because Airflow still uses the older boto (not boto3) that has this issue (see
boto/boto#2836 and
https://issues.apache.org/jira/browse/AIRFLOW-115).

Fixes opentrials/opentrials#763

ansible/deploy_dockercloud.yml

@@ -15,6 +15,8 @@
     - name: update Docker Cloud stack
       shell: docker-cloud stack update --sync $DOCKERCLOUD_STACK -f {{ dockercloud_conf }}
       environment:
+        REMOTE_BASE_LOG_FOLDER: "{{ remote_base_log_folder }}"
+        REMOTE_LOG_CONN_ID: "{{ remote_log_conn_id }}"
         DB_URI: "{{ db_uri }}"
         DB_USER: "{{ db_user }}"
         DB_PASSWORD: "{{ db_password }}"

ansible/deploy_local.yml

@@ -15,6 +15,8 @@
       environment:
         AIRFLOW_ENABLE_AUTH: False
         FERNET_KEY: 46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
+        REMOTE_BASE_LOG_FOLDER: "{{ remote_base_log_folder }}"
+        REMOTE_LOG_CONN_ID: "{{ remote_log_conn_id }}"
         DB_URI: "{{ db_uri }}"
         DB_USER: "{{ db_user }}"
         DB_PASSWORD: "{{ db_password }}"

ansible/envs/dev.yml

@@ -2,5 +2,7 @@ db_uri: postgres
 db_user: airflow
 db_password: airflow
 db_table: airflow
+remote_base_log_folder: ""
+remote_log_conn_id: ""
 
 airflow_enable_auth: false

ansible/envs/production.yml

@@ -1,26 +1,30 @@
 $ANSIBLE_VAULT;1.1;AES256
-36316137353232663236663733333061396261376537633938646232316664633761353834616139
-6236663231396237343864623134393239656231336333380a323931663361313030303636346534
-34313262303639363335653931333364333064376131643933636235376539363035653738653165
-6432343539376130390a666130393265613035303736343933353035643636306234363261303139
-65373433633263343663373331306366616563613663613835626237373264343362313538373563
-62373534633036343935313464636664363366653738313163303861633538376431383865393234
-61313963643066613531346439626431643131663132333862356338653132623238393834396131
-37346665623937626264643764653836636566613861333734373233633036616431366466336464
-65663435653732326531323232623264336239363463666665353534323062396235326438306335
-33306633663337346636656466396335386266306137323237366333313364366239636234353734
-34383364376663313664393262393562393137383535366138633765353839623064646565626564
-65323635306636363039613233313130306334356162663365366136613531383966383466373634
-36326632323635316135356639633165336461376664613137333465356566323037333836383664
-34356462303836633964643166313133336437303335336333303365343264616162303761373836
-36306633656332343530303831353263333663343939393137303639313338306236653634616437
-64663039333832336666393563353165323439646535653735636363313738626365303732613764
-34316265613635303166373561376337373132646336333533616466633561633663386163646239
-35666430373336313163646236626466656163396637333638616566323730386664653165323637
-61313736353535303432323765373533643939353531313963363230663066623936653835653866
-37323661323737613664356139306536373763663130326337336333613266363661393338373432
-35316634346338313334616431363062376334646434656461356532336262313062653264646532
-63613837383663373665666265633139383038623532396536396665336431343733373037333238
-33626165343031383839373465386639303563636465376431303766343635303630613934336562
-38356132326535323362383930616134653338373833623561313962643336333438613931366337
-3662
+37623235646262393932656565376130316238346537666562303837376566336136363361393366
+3230313065393437393666303534306264376663383631300a666633396333353961393135633764
+66356632353162386131383735306262653138666133376337393734663231386538366664373066
+3336613830353334640a613062306336333538353533386164336366316132343366636561343937
+64613132393532393031643138376561386430616633313136336565666232363464353566343637
+36626436343962343834373733396564323265373364663866356431316465656663653230643064
+38656236303631383863396530313130346636623437656263393862653637663031303131363061
+30386263363763333031646633376161626137303630303435633962613963356630346235366132
+36653663616666336239366238303139383132613635633335663731333935643264323665396664
+32666361623265616535613566323863376366313161376163623164626233353366623535316631
+66383061373736313230663630303434613938666133326233663263306138336133373334396264
+39363262353233306264356364663038613462663635356561656565316334363865313939323765
+36373436633137303339613364356462336133633832303232366638323535336634613664313633
+33626531313533623534643964623939353465363037366330626362393736623932323563376366
+38653731633261306533373734653238646462313037376439623432323865373531656331633631
+33613134363161353230316430373130666633373136313030646337373265623961303965396639
+36393937633434646536313735393635313735393061363363633032613535376430636536326138
+66396134343032356632326232383934313865663138373763313238353235653762333864393934
+61363161616265373538303439313137323234613139376566636232313131363166666666393136
+38613766353333333361633732376232666466316262396639333334653439633662343235353238
+35363431646436333164323234643632653638363562623439646638666130303166353139646133
+61376363636466323135363864396234653430343564333565383437613239353433356335626535
+32626133663035653438663962373531653066303739643533323730383562613064373930613266
+35663130393762613566386235616334366162633635303139633339333763616132636134393061
+38623838643065613730643065366538366535653739623032393564633433613733363134663630
+34393863643930633030626431663532356564636363386530623132663239666365636335616635
+39343939616361646665316565666139303862633836333630346539643132393332336634306530
+35663731356435373039656539306464636633353066326362366230656632653039653330353433
+65313065373236663636303963353165643361393963643135643835623165643432

ansible/files/airflow/airflow.cfg

@@ -9,6 +9,15 @@ dags_folder = /usr/local/airflow/dags
 # The folder where airflow should store its log files. This location
 base_log_folder = /usr/local/airflow/logs
 
+# Airflow can store logs remotely in AWS S3 or Google Cloud Storage. Users
+# must supply a remote location URL (starting with either 's3://...' or
+# 'gs://...') and an Airflow connection id that provides access to the storage
+# location.
+remote_base_log_folder = ${REMOTE_BASE_LOG_FOLDER}
+remote_log_conn_id = ${REMOTE_LOG_CONN_ID}
+# Use server-side encryption for logs stored in S3
+encrypt_s3_logs = False
+
 # The executor class that airflow should use. Choices include
 # SequentialExecutor, LocalExecutor, CeleryExecutor
 executor = CeleryExecutor

ansible/files/docker-cloud.yml

@@ -8,6 +8,8 @@ webserver:
   environment:
     - EXECUTOR=Celery
     - AIRFLOW_ENABLE_AUTH
+    - REMOTE_BASE_LOG_FOLDER
+    - REMOTE_LOG_CONN_ID
     - FERNET_KEY
     - DB_URI
     - DB_USER
@@ -28,6 +30,8 @@ flower:
   environment:
     - EXECUTOR=Celery
     - AIRFLOW_ENABLE_AUTH
+    - REMOTE_BASE_LOG_FOLDER
+    - REMOTE_LOG_CONN_ID
     - FERNET_KEY
     - DB_URI
     - DB_USER
@@ -48,6 +52,8 @@ scheduler:
   environment:
     - EXECUTOR=Celery
     - AIRFLOW_ENABLE_AUTH
+    - REMOTE_BASE_LOG_FOLDER
+    - REMOTE_LOG_CONN_ID
     - FERNET_KEY
     - DB_URI
     - DB_USER
@@ -70,6 +76,8 @@ worker:
   environment:
     - EXECUTOR=Celery
     - AIRFLOW_ENABLE_AUTH
+    - REMOTE_BASE_LOG_FOLDER
+    - REMOTE_LOG_CONN_ID
     - FERNET_KEY
     - DB_URI
     - DB_USER

ansible/files/docker-compose.yml

@@ -11,6 +11,8 @@ services:
         environment:
             EXECUTOR: Celery
             AIRFLOW_ENABLE_AUTH:
+            REMOTE_BASE_LOG_FOLDER:
+            REMOTE_LOG_CONN_ID:
             FERNET_KEY:
             DB_URI:
             DB_USER:
@@ -31,6 +33,8 @@ services:
         environment:
             EXECUTOR: Celery
             AIRFLOW_ENABLE_AUTH:
+            REMOTE_BASE_LOG_FOLDER:
+            REMOTE_LOG_CONN_ID:
             FERNET_KEY:
             DB_URI:
             DB_USER:
@@ -51,6 +55,8 @@ services:
         environment:
             EXECUTOR: Celery
             AIRFLOW_ENABLE_AUTH:
+            REMOTE_BASE_LOG_FOLDER:
+            REMOTE_LOG_CONN_ID:
             FERNET_KEY:
             DB_URI:
             DB_USER:
@@ -71,6 +77,8 @@ services:
         environment:
             EXECUTOR: Celery
             AIRFLOW_ENABLE_AUTH:
+            REMOTE_BASE_LOG_FOLDER:
+            REMOTE_LOG_CONN_ID:
             FERNET_KEY:
             DB_URI:
             DB_USER:

requirements.in

@@ -1,5 +1,5 @@
--e git+git://github.com/apache/incubator-airflow.git@35e43f5067f4741640278b765c0e54e4fd45ffa3#egg=airflow[async,password,celery,crypto,postgres,hive,hdfs,jdbc]
-docker-py==1.10.4
+-e git+git://github.com/apache/incubator-airflow.git@35e43f5067f4741640278b765c0e54e4fd45ffa3#egg=airflow[async,password,celery,redis,crypto,postgres,hive,hdfs,jdbc,s3]
+docker-py
 boto3==1.4.1
 retrying==1.3.3
 requests==2.13.0

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
--e git+git://github.com/apache/incubator-airflow.git@35e43f5067f4741640278b765c0e54e4fd45ffa3#egg=airflow[async,password,celery,crypto,postgres,hive,hdfs,jdbc]
+-e git+git://github.com/apache/incubator-airflow.git@35e43f5067f4741640278b765c0e54e4fd45ffa3#egg=airflow[async,password,celery,redis,crypto,postgres,hive,hdfs,jdbc,s3]
 alembic==0.8.10
 amqp==2.1.4               # via kombu
 appdirs==1.4.3            # via setuptools
@@ -16,6 +16,7 @@ billiard==3.5.0.2         # via celery
 bitarray==0.8.1           # via impyla
 bleach==2.0.0
 boto3==1.4.1
+boto==2.46.1
 botocore==1.4.93          # via boto3, s3transfer
 celery==4.0.2             # via flower
 cffi==1.10.0              # via bcrypt, cryptography
@@ -24,18 +25,19 @@ configparser==3.5.0
 croniter==0.3.16
 cryptography==1.8.1
 dill==0.2.6
-docker-py==1.10.4
+docker-py==1.10.6
 docker-pycreds==0.2.1     # via docker-py
 docutils==0.13.1          # via botocore, python-daemon
 enum-compat==0.0.2        # via eventlet
 eventlet==0.21.0
+filechunkio==1.8
 flask-admin==1.4.1
 flask-bcrypt==0.7.1
 flask-cache==0.13.1
 flask-login==0.2.11
 flask-swagger==0.2.13
 flask-wtf==0.12
-Flask==0.11.1             # via flask-admin, flask-bcrypt, flask-cache, flask-login, flask-swagger, flask-wtf
+flask==0.11.1             # via flask-admin, flask-bcrypt, flask-cache, flask-login, flask-swagger, flask-wtf
 flower==0.9.1
 funcsigs==1.0.0
 future==0.16.0            # via pyhive
@@ -51,15 +53,15 @@ idna==2.5                 # via cryptography
 impyla==0.14.0
 itsdangerous==0.24        # via flask
 jaydebeapi==1.1.1
-Jinja2==2.8.1             # via flask, python-nvd3
+jinja2==2.8.1             # via flask, python-nvd3
 jmespath==0.9.2           # via boto3, botocore
-JPype1==0.6.2             # via jaydebeapi
+jpype1==0.6.2             # via jaydebeapi
 kombu==4.0.2              # via celery
 lockfile==0.12.2          # via python-daemon
 lxml==3.7.3
-Mako==1.0.6               # via alembic
+mako==1.0.6               # via alembic
 markdown==2.6.8
-MarkupSafe==1.0           # via jinja2, mako
+markupsafe==1.0           # via jinja2, mako
 numpy==1.12.1             # via pandas
 ordereddict==1.1          # via funcsigs
 packaging==16.8           # via cryptography, setuptools
@@ -79,7 +81,8 @@ python-editor==1.0.3      # via alembic
 python-nvd3==0.14.2
 python-slugify==1.1.4     # via python-nvd3
 pytz==2017.2              # via babel, celery, flower, pandas
-PyYAML==3.12              # via flask-swagger
+pyyaml==3.12              # via flask-swagger
+redis==2.10.5
 requests==2.13.0
 retrying==1.3.3
 s3transfer==0.1.10        # via boto3
@@ -94,11 +97,11 @@ thrift==0.9.3
 thriftpy==0.3.9           # via impyla
 tornado==4.2.0            # via flower
 unicodecsv==0.14.1
-Unidecode==0.4.20         # via python-slugify
+unidecode==0.4.20         # via python-slugify
 vine==1.1.3               # via amqp
 webencodings==0.5.1       # via html5lib
 websocket-client==0.40.0  # via docker-py
-Werkzeug==0.12.1          # via flask, flask-wtf
+werkzeug==0.12.1          # via flask, flask-wtf
 wtforms==2.1              # via flask-admin, flask-wtf
 zope.deprecation==4.2.0