Merge pull request #27 from Dorianteffo/dev_dorian

remove executable path cosmos
Dorianteffo · Mar 21, 2024 · a00d55c · a00d55c
2 parents 0451b9f + 66a7181
commit a00d55c
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 35 deletions.
diff --git a/.github/workflows/cd_airflow.yml b/.github/workflows/cd_airflow.yml
@@ -27,12 +27,6 @@ jobs:
                 SCRIPT_AFTER: |
                     cd airflow
                     echo -e "AIRFLOW_UID=$(id -u)\nAIRFLOW_GID=0" > .env
-                    sudo apt install python3-pip  
-                    pip3 install virtualenv 
-                    virtualenv dbt_venv
-                    source myenv/bin/activate
-                    pip install dbt-snowflake
-                    deactivate
                     docker build -t apache-airflow:dbt-snowflake-dorian . 
                     docker compose up airflow-init
                     docker compose up -d 

diff --git a/airflow/Dockerfile b/airflow/Dockerfile
@@ -1,4 +1,6 @@
-FROM apache/airflow:2.8.3
+# FROM apache/airflow:2.8.3
+
+FROM apache/airflow:2.8.3-python3.10
 
 COPY requirements.txt /
 

diff --git a/airflow/dags/cosmos_dbt_dag.py b/airflow/dags/cosmos_dbt_dag.py
@@ -5,7 +5,7 @@
 from airflow.decorators import dag
 
 DBT_PROJECT_PATH = "/opt/airflow/dags/dbt/dbt_transformation"
-DBT_EXECUTABLE_PATH = "/opt/airflow/dbt_venv/Lib/site-packages"
+# DBT_EXECUTABLE_PATH = "/opt/airflow/dbt_venv/bin/dbt"
 
 profile_config_dev = ProfileConfig(
     profile_name="modern_warehouse",
@@ -38,7 +38,7 @@ def dbt_dag():
         group_id = "dbt_dev",
         project_config=ProjectConfig(DBT_PROJECT_PATH),
         operator_args={"install_deps": True},
-        execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
+        # execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
         profile_config=profile_config_dev,
         default_args={"retries": 2}
     )
@@ -48,7 +48,7 @@ def dbt_dag():
         group_id = "dbt_prod",
         project_config=ProjectConfig(DBT_PROJECT_PATH),
         operator_args={"install_deps": True},
-        execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
+        # execution_config = ExecutionConfig(dbt_executable_path = DBT_EXECUTABLE_PATH),
         profile_config=profile_config_prod,
         default_args={"retries": 2}
     )

diff --git a/airflow/dags/dbt/dbt_transformation/profiles.yml b/airflow/dags/dbt/dbt_transformation/profiles.yml