Add alias for env var in docker compose

Barski-lab · Nov 24, 2020 · 5b86305 · 5b86305
1 parent 922a6e5
commit 5b86305
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 31 deletions.
diff --git a/packaging/docker_compose/local_executor/.env b/packaging/docker_compose/local_executor/.env
@@ -1,6 +1,6 @@
 # We always read configuration from AIRFLOW_HOME/airflow.cfg
 # We expect that your AIRFLOW_HOME folder has the following
-# structure
+# structure. If not, it will be created this way.
 #
 #  ├── dags
 #  ├── logs
@@ -21,9 +21,11 @@
 # 
 # as these parameters will be overwritten by the environment
 # variables with their defaults for Airflow values.
-# We can't read it from airflow.cfg because, for example,
+# We can't read them from airflow.cfg because, for example,
 # dags_folder should be mounted to the container before it was
-# started and could possibly read the airflow.cfg file.
+# started. All log related parameters should be also redefined
+# because in the airflow.cfg they might have locations on the
+# host system.
 
 
 # Volumes

diff --git a/packaging/docker_compose/local_executor/docker-compose.yml b/packaging/docker_compose/local_executor/docker-compose.yml
@@ -1,6 +1,19 @@
 version: "3"
 
 
+x-aliases:               # if it starts with x- it's ignored by docker-compose
+    - &airflow_env_var
+        environment:
+            - AIRFLOW_HOME=/opt/airflow
+            - AIRFLOW__CORE__EXECUTOR=LocalExecutor
+            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql:3306/${MYSQL_DATABASE}
+            - AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
+            - AIRFLOW__CORE__BASE_LOG_FOLDER=/opt/airflow/logs
+            - AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
+            - AIRFLOW__CORE__PLUGINS_FOLDER=/opt/airflow/plugins
+            - AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/opt/airflow/logs/scheduler
+
+
 services:
 
     scheduler:
@@ -15,16 +28,8 @@ services:
             - ${CWL_PICKLE_FOLDER}:/opt/airflow/cwl_pickle_folder
         privileged: true
         restart: always
-        environment:
-            - AIRFLOW_HOME=/opt/airflow
-            - AIRFLOW__CORE__EXECUTOR=LocalExecutor
-            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql:3306/${MYSQL_DATABASE}
-            - AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
-            - AIRFLOW__CORE__BASE_LOG_FOLDER=/opt/airflow/logs
-            - AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
-            - AIRFLOW__CORE__PLUGINS_FOLDER=/opt/airflow/plugins
-            - AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/opt/airflow/logs/scheduler
-        command: bash -c "cwl-airflow init && airflow scheduler"
+        <<: *airflow_env_var
+        command: bash -c "cwl-airflow init --upgrade && airflow scheduler"  # always run init, as it's safe
         depends_on:
             - mysql
 
@@ -40,15 +45,7 @@ services:
             - ${CWL_OUTPUTS_FOLDER}:/opt/airflow/cwl_outputs_folder       # why does webserver need it?
             - ${CWL_PICKLE_FOLDER}:/opt/airflow/cwl_pickle_folder
         restart: always
-        environment:
-            - AIRFLOW_HOME=/opt/airflow
-            - AIRFLOW__CORE__EXECUTOR=LocalExecutor
-            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql:3306/${MYSQL_DATABASE}
-            - AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
-            - AIRFLOW__CORE__BASE_LOG_FOLDER=/opt/airflow/logs
-            - AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
-            - AIRFLOW__CORE__PLUGINS_FOLDER=/opt/airflow/plugins
-            - AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/opt/airflow/logs/scheduler
+        <<: *airflow_env_var
         command: airflow webserver
         depends_on:
             - mysql
@@ -66,15 +63,7 @@ services:
             - ${CWL_OUTPUTS_FOLDER}:/opt/airflow/cwl_outputs_folder       # why does apiserver need it?
             - ${CWL_PICKLE_FOLDER}:/opt/airflow/cwl_pickle_folder
         restart: always
-        environment:
-            - AIRFLOW_HOME=/opt/airflow
-            - AIRFLOW__CORE__EXECUTOR=LocalExecutor
-            - AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql:3306/${MYSQL_DATABASE}
-            - AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
-            - AIRFLOW__CORE__BASE_LOG_FOLDER=/opt/airflow/logs
-            - AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
-            - AIRFLOW__CORE__PLUGINS_FOLDER=/opt/airflow/plugins
-            - AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/opt/airflow/logs/scheduler
+        <<: *airflow_env_var
         command: cwl-airflow api --host 0.0.0.0  # default --host 127.0.0.1 won't work from inside the docker container
         depends_on:
             - mysql