Deploy

Dockerfile

@@ -1,7 +1,7 @@
 FROM mageai/mageai:alpha
 
 ARG PROJECT_NAME=mlops
-ARG MAGE_CODE_PATH=/home/src
+ARG MAGE_CODE_PATH=/home/mage_code
 ARG USER_CODE_PATH=${MAGE_CODE_PATH}/${PROJECT_NAME}
 
 WORKDIR ${MAGE_CODE_PATH}

docker-compose.yml

@@ -8,9 +8,9 @@ services:
     ports:
       - 6789:6789
     volumes:
-      - .:/$MAGE_CODE_PATH # /home/src by default
+      - .:/$MAGE_CODE_PATH # /home/mage_code by default
       # Store the data output on local machine to easily debug
-      - ~/.mage_data:/$MAGE_CODE_PATH/mage_data # /home/src/mage_data by default
+      - ~/.mage_data:/$MAGE_CODE_PATH/mage_data # /home/mage_code/mage_data by default
     restart: on-failure:5
     networks:
       - app-network

mlops/unit_2_training/pipelines/data_preparation/metadata.yaml

@@ -90,7 +90,6 @@ variables:
   split_on_feature: lpep_pickup_datetime
   split_on_feature_value: "2024-02-01"
   target: duration
-variables_dir: /home/src/mage_data/unit_1_data_preparation
 widgets:
   - all_upstream_blocks_executed: true
     color: null

mlops/unit_2_training/pipelines/sklearn_training/metadata.yaml

@@ -102,5 +102,4 @@ spark_config: {}
 tags: []
 type: python
 uuid: sklearn_training
-variables_dir: /home/src/mage_data/unit_2_training
 widgets: []

mlops/unit_2_training/pipelines/xgboost_training/metadata.yaml

@@ -106,5 +106,4 @@ variables:
   early_stopping_rounds: 1
   max_depth: 1
   max_evaluations: 1
-variables_dir: /home/src/mage_data/unit_4_triggering
 widgets: []

mlops/unit_3_observability/pipelines/xgboost_training/metadata.yaml

@@ -106,5 +106,4 @@ variables:
   early_stopping_rounds: 1
   max_depth: 1
   max_evaluations: 1
-variables_dir: /home/src/mage_data/unit_4_triggering
 widgets: []

mlops/unit_4_triggering/pipelines/automatic_retraining/metadata.yaml

@@ -1,64 +1,64 @@
 blocks:
-- all_upstream_blocks_executed: true
-  color: null
-  configuration:
-    file_source:
-      path: unit_4_triggering/sensors/detect_new_data.py
-  downstream_blocks:
-  - retrain/sklearn
-  - retrain/xgboost
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: Detect new data
-  retry_config: null
-  status: executed
-  timeout: null
-  type: sensor
-  upstream_blocks: []
-  uuid: detect_new_data
-- all_upstream_blocks_executed: true
-  color: teal
-  configuration:
-    file_source:
-      path: unit_4_triggering/custom/retrain/sklearn.py
-  downstream_blocks: []
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: retrain/sklearn
-  retry_config: null
-  status: executed
-  timeout: null
-  type: custom
-  upstream_blocks:
-  - detect_new_data
-  uuid: retrain/sklearn
-- all_upstream_blocks_executed: true
-  color: yellow
-  configuration:
-    file_source:
-      path: unit_4_triggering/custom/retrain/xgboost.py
-  downstream_blocks: []
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: retrain/XGBoost
-  retry_config: null
-  status: updated
-  timeout: null
-  type: custom
-  upstream_blocks:
-  - detect_new_data
-  uuid: retrain/xgboost
+  - all_upstream_blocks_executed: true
+    color: null
+    configuration:
+      file_source:
+        path: unit_4_triggering/sensors/detect_new_data.py
+    downstream_blocks:
+      - retrain/sklearn
+      - retrain/xgboost
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: Detect new data
+    retry_config: null
+    status: executed
+    timeout: null
+    type: sensor
+    upstream_blocks: []
+    uuid: detect_new_data
+  - all_upstream_blocks_executed: true
+    color: teal
+    configuration:
+      file_source:
+        path: unit_4_triggering/custom/retrain/sklearn.py
+    downstream_blocks: []
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: retrain/sklearn
+    retry_config: null
+    status: executed
+    timeout: null
+    type: custom
+    upstream_blocks:
+      - detect_new_data
+    uuid: retrain/sklearn
+  - all_upstream_blocks_executed: true
+    color: yellow
+    configuration:
+      file_source:
+        path: unit_4_triggering/custom/retrain/xgboost.py
+    downstream_blocks: []
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: retrain/XGBoost
+    retry_config: null
+    status: updated
+    timeout: null
+    type: custom
+    upstream_blocks:
+      - detect_new_data
+    uuid: retrain/xgboost
 cache_block_output_in_memory: false
 callbacks: []
 concurrency_config: {}
 conditionals: []
-created_at: '2024-05-10 20:43:05.240091+00:00'
+created_at: "2024-05-10 20:43:05.240091+00:00"
 data_integration: null
 description: null
 executor_config: {}
@@ -76,5 +76,4 @@ spark_config: {}
 tags: []
 type: python
 uuid: automatic_retraining
-variables_dir: /home/src/mage_data/unit_4_triggering
 widgets: []

mlops/unit_4_triggering/pipelines/predict/metadata.yaml

@@ -1,67 +1,67 @@
 blocks:
-- all_upstream_blocks_executed: true
-  color: null
-  configuration:
-    global_data_product:
-      uuid: training_set
-  downstream_blocks:
-  - inference
-  - model
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: Training data
-  retry_config: null
-  status: executed
-  timeout: null
-  type: global_data_product
-  upstream_blocks: []
-  uuid: training_data
-- all_upstream_blocks_executed: true
-  color: null
-  configuration:
-    file_source:
-      path: unit_4_triggering/global_data_products/model.py
-    global_data_product:
-      uuid: xgboost
-  downstream_blocks:
-  - inference
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: Model
-  retry_config: null
-  status: executed
-  timeout: null
-  type: global_data_product
-  upstream_blocks: []
-  uuid: model
-- all_upstream_blocks_executed: true
-  color: teal
-  configuration:
-    file_source:
-      path: null
-  downstream_blocks: []
-  executor_config: null
-  executor_type: local_python
-  has_callback: false
-  language: python
-  name: inference
-  retry_config: null
-  status: failed
-  timeout: null
-  type: custom
-  upstream_blocks:
-  - training_data
-  - model
-  uuid: inference
+  - all_upstream_blocks_executed: true
+    color: null
+    configuration:
+      global_data_product:
+        uuid: training_set
+    downstream_blocks:
+      - inference
+      - model
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: Training data
+    retry_config: null
+    status: executed
+    timeout: null
+    type: global_data_product
+    upstream_blocks: []
+    uuid: training_data
+  - all_upstream_blocks_executed: true
+    color: null
+    configuration:
+      file_source:
+        path: unit_4_triggering/global_data_products/model.py
+      global_data_product:
+        uuid: xgboost
+    downstream_blocks:
+      - inference
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: Model
+    retry_config: null
+    status: executed
+    timeout: null
+    type: global_data_product
+    upstream_blocks: []
+    uuid: model
+  - all_upstream_blocks_executed: true
+    color: teal
+    configuration:
+      file_source:
+        path: null
+    downstream_blocks: []
+    executor_config: null
+    executor_type: local_python
+    has_callback: false
+    language: python
+    name: inference
+    retry_config: null
+    status: failed
+    timeout: null
+    type: custom
+    upstream_blocks:
+      - training_data
+      - model
+    uuid: inference
 cache_block_output_in_memory: false
 callbacks: []
 concurrency_config: {}
 conditionals: []
-created_at: '2024-05-09 02:45:15.656239+00:00'
+created_at: "2024-05-09 02:45:15.656239+00:00"
 data_integration: null
 description: Online inference pipeline.
 executor_config: {}
@@ -80,6 +80,5 @@ tags: []
 type: python
 uuid: predict
 variables:
-  model_dir: ''
-variables_dir: /home/src/mage_data/unit_4_triggering
+  model_dir: ""
 widgets: []

mlops/unit_4_triggering/pipelines/xgboost_training/metadata.yaml

@@ -106,5 +106,4 @@ variables:
   early_stopping_rounds: 1
   max_depth: 1
   max_evaluations: 1
-variables_dir: /home/src/mage_data/unit_4_triggering
 widgets: []

terraform/aws/scripts/create-github-actions-workflow.sh

@@ -7,7 +7,7 @@ ECS_SERVICE="$4"
 ECS_TASK_DEFINITION="$5"
 CONTAINER_NAME="$6"
 
-WORKFLOW_PATH="../../.github/workflows/deploy_to_ecs.yml"
+WORKFLOW_PATH="../../.github/workflows/build_and_deploy_to_aws_ecs.yml"
 WORKFLOW_DIR=$(dirname "$WORKFLOW_PATH")
 
 mkdir -p "$WORKFLOW_DIR"