Merge pull request #28 from Dorianteffo/dev_dorian

dbt executable, dockerfile airflow

airflow/Dockerfile

@@ -1,15 +1,10 @@
-# FROM apache/airflow:2.8.3
-
-FROM apache/airflow:2.8.3-python3.10
+FROM apache/airflow:2.8.3
 
 COPY requirements.txt /
 
 RUN pip install apache-airflow==${AIRFLOW_VERSION} astronomer-cosmos
 
-# # Create the virtual environment using virtualenv
-# RUN python -m venv dbt_venv
-
-# # Activate the virtual environment
-# RUN source dbt_venv/bin/activate && \
-#     pip install --no-cache-dir --ignore-installed dbt-snowflake && \
-#     deactivate
+# install dbt-snowflake in a virtual env
+RUN export PIP_USER=false && python -m venv dbt_venv && source dbt_venv/bin/activate && \
+    pip install --no-cache-dir --ignore-installed dbt-snowflake && \
+    deactivate

airflow/dags/cosmos_dbt_dag.py

@@ -1,12 +1,12 @@
 from cosmos import DbtTaskGroup, ProjectConfig, ProfileConfig, ExecutionConfig
 from cosmos.profiles import SnowflakeUserPasswordProfileMapping
-import os
 from datetime import datetime 
 from airflow.decorators import dag
 
 DBT_PROJECT_PATH = "/opt/airflow/dags/dbt/dbt_transformation"
-# DBT_EXECUTABLE_PATH = "/opt/airflow/dbt_venv/bin/dbt"
+DBT_EXECUTABLE_PATH = "/opt/airflow/dbt_venv/bin/dbt"
 
+# dev environment 
 profile_config_dev = ProfileConfig(
     profile_name="modern_warehouse",
     target_name="dev",
@@ -16,7 +16,7 @@
     ),
 )
 
-
+# profile for the prod env
 profile_config_prod = ProfileConfig(
     profile_name="modern_warehouse",
     target_name="prod",
@@ -38,7 +38,7 @@ def dbt_dag():
         group_id = "dbt_dev",
         project_config=ProjectConfig(DBT_PROJECT_PATH),
         operator_args={"install_deps": True},
-        # execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
+        execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
         profile_config=profile_config_dev,
         default_args={"retries": 2}
     )
@@ -48,7 +48,7 @@ def dbt_dag():
         group_id = "dbt_prod",
         project_config=ProjectConfig(DBT_PROJECT_PATH),
         operator_args={"install_deps": True},
-        # execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
+        execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
         profile_config=profile_config_prod,
         default_args={"retries": 2}
     )

airflow/docker-compose.yaml

@@ -74,7 +74,6 @@ x-airflow-common:
     - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
     - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config
     - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
-    - ${AIRFLOW_PROJ_DIR:-.}dbt_venv:/opt/airflow/dbt_venv
   user: "${AIRFLOW_UID:-50000}:0"
   depends_on:
     &airflow-common-depends-on

airflow/requirements.txt

@@ -1,3 +1,2 @@
-# Astro Runtime includes the following pre-installed providers packages: https://docs.astronomer.io/astro/runtime-image-architecture#provider-packages
 astronomer-cosmos
 apache-airflow-providers-snowflake