diff --git a/.github/workflows/deploy_to_ecs.yml b/.github/workflows/build_and_deploy_to_aws_ecs.yml similarity index 100% rename from .github/workflows/deploy_to_ecs.yml rename to .github/workflows/build_and_deploy_to_aws_ecs.yml diff --git a/Dockerfile b/Dockerfile index cfcf2919..452b6a95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM mageai/mageai:alpha ARG PROJECT_NAME=mlops -ARG MAGE_CODE_PATH=/home/src +ARG MAGE_CODE_PATH=/home/mage_code ARG USER_CODE_PATH=${MAGE_CODE_PATH}/${PROJECT_NAME} WORKDIR ${MAGE_CODE_PATH} diff --git a/docker-compose.yml b/docker-compose.yml index 64f88b48..bb0b8a16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,9 @@ services: ports: - 6789:6789 volumes: - - .:/$MAGE_CODE_PATH # /home/src by default + - .:/$MAGE_CODE_PATH # /home/mage_code by default # Store the data output on local machine to easily debug - - ~/.mage_data:/$MAGE_CODE_PATH/mage_data # /home/src/mage_data by default + - ~/.mage_data:/$MAGE_CODE_PATH/mage_data # /home/mage_code/mage_data by default restart: on-failure:5 networks: - app-network diff --git a/mlops/unit_2_training/pipelines/data_preparation/metadata.yaml b/mlops/unit_2_training/pipelines/data_preparation/metadata.yaml index 91eccf01..367cf7a2 100755 --- a/mlops/unit_2_training/pipelines/data_preparation/metadata.yaml +++ b/mlops/unit_2_training/pipelines/data_preparation/metadata.yaml @@ -90,7 +90,6 @@ variables: split_on_feature: lpep_pickup_datetime split_on_feature_value: "2024-02-01" target: duration -variables_dir: /home/src/mage_data/unit_1_data_preparation widgets: - all_upstream_blocks_executed: true color: null diff --git a/mlops/unit_2_training/pipelines/sklearn_training/metadata.yaml b/mlops/unit_2_training/pipelines/sklearn_training/metadata.yaml index 620ba657..589d1fb6 100755 --- a/mlops/unit_2_training/pipelines/sklearn_training/metadata.yaml +++ b/mlops/unit_2_training/pipelines/sklearn_training/metadata.yaml @@ -102,5 +102,4 @@ spark_config: {} tags: [] type: python uuid: sklearn_training -variables_dir: /home/src/mage_data/unit_2_training widgets: [] diff --git a/mlops/unit_2_training/pipelines/xgboost_training/metadata.yaml b/mlops/unit_2_training/pipelines/xgboost_training/metadata.yaml index ce093075..3bc6ed14 100755 --- a/mlops/unit_2_training/pipelines/xgboost_training/metadata.yaml +++ b/mlops/unit_2_training/pipelines/xgboost_training/metadata.yaml @@ -106,5 +106,4 @@ variables: early_stopping_rounds: 1 max_depth: 1 max_evaluations: 1 -variables_dir: /home/src/mage_data/unit_4_triggering widgets: [] diff --git a/mlops/unit_3_observability/pipelines/xgboost_training/metadata.yaml b/mlops/unit_3_observability/pipelines/xgboost_training/metadata.yaml index ce093075..3bc6ed14 100755 --- a/mlops/unit_3_observability/pipelines/xgboost_training/metadata.yaml +++ b/mlops/unit_3_observability/pipelines/xgboost_training/metadata.yaml @@ -106,5 +106,4 @@ variables: early_stopping_rounds: 1 max_depth: 1 max_evaluations: 1 -variables_dir: /home/src/mage_data/unit_4_triggering widgets: [] diff --git a/mlops/unit_4_triggering/pipelines/automatic_retraining/metadata.yaml b/mlops/unit_4_triggering/pipelines/automatic_retraining/metadata.yaml index ef1fbebd..263168f3 100755 --- a/mlops/unit_4_triggering/pipelines/automatic_retraining/metadata.yaml +++ b/mlops/unit_4_triggering/pipelines/automatic_retraining/metadata.yaml @@ -1,64 +1,64 @@ blocks: -- all_upstream_blocks_executed: true - color: null - configuration: - file_source: - path: unit_4_triggering/sensors/detect_new_data.py - downstream_blocks: - - retrain/sklearn - - retrain/xgboost - executor_config: null - executor_type: local_python - has_callback: false - language: python - name: Detect new data - retry_config: null - status: executed - timeout: null - type: sensor - upstream_blocks: [] - uuid: detect_new_data -- all_upstream_blocks_executed: true - color: teal - configuration: - file_source: - path: unit_4_triggering/custom/retrain/sklearn.py - downstream_blocks: [] - executor_config: null - executor_type: local_python - has_callback: false - language: python - name: retrain/sklearn - retry_config: null - status: executed - timeout: null - type: custom - upstream_blocks: - - detect_new_data - uuid: retrain/sklearn -- all_upstream_blocks_executed: true - color: yellow - configuration: - file_source: - path: unit_4_triggering/custom/retrain/xgboost.py - downstream_blocks: [] - executor_config: null - executor_type: local_python - has_callback: false - language: python - name: retrain/XGBoost - retry_config: null - status: updated - timeout: null - type: custom - upstream_blocks: - - detect_new_data - uuid: retrain/xgboost + - all_upstream_blocks_executed: true + color: null + configuration: + file_source: + path: unit_4_triggering/sensors/detect_new_data.py + downstream_blocks: + - retrain/sklearn + - retrain/xgboost + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: Detect new data + retry_config: null + status: executed + timeout: null + type: sensor + upstream_blocks: [] + uuid: detect_new_data + - all_upstream_blocks_executed: true + color: teal + configuration: + file_source: + path: unit_4_triggering/custom/retrain/sklearn.py + downstream_blocks: [] + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: retrain/sklearn + retry_config: null + status: executed + timeout: null + type: custom + upstream_blocks: + - detect_new_data + uuid: retrain/sklearn + - all_upstream_blocks_executed: true + color: yellow + configuration: + file_source: + path: unit_4_triggering/custom/retrain/xgboost.py + downstream_blocks: [] + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: retrain/XGBoost + retry_config: null + status: updated + timeout: null + type: custom + upstream_blocks: + - detect_new_data + uuid: retrain/xgboost cache_block_output_in_memory: false callbacks: [] concurrency_config: {} conditionals: [] -created_at: '2024-05-10 20:43:05.240091+00:00' +created_at: "2024-05-10 20:43:05.240091+00:00" data_integration: null description: null executor_config: {} @@ -76,5 +76,4 @@ spark_config: {} tags: [] type: python uuid: automatic_retraining -variables_dir: /home/src/mage_data/unit_4_triggering widgets: [] diff --git a/mlops/unit_4_triggering/pipelines/predict/metadata.yaml b/mlops/unit_4_triggering/pipelines/predict/metadata.yaml index 550a4e23..907e8421 100755 --- a/mlops/unit_4_triggering/pipelines/predict/metadata.yaml +++ b/mlops/unit_4_triggering/pipelines/predict/metadata.yaml @@ -1,67 +1,67 @@ blocks: -- all_upstream_blocks_executed: true - color: null - configuration: - global_data_product: - uuid: training_set - downstream_blocks: - - inference - - model - executor_config: null - executor_type: local_python - has_callback: false - language: python - name: Training data - retry_config: null - status: executed - timeout: null - type: global_data_product - upstream_blocks: [] - uuid: training_data -- all_upstream_blocks_executed: true - color: null - configuration: - file_source: - path: unit_4_triggering/global_data_products/model.py - global_data_product: - uuid: xgboost - downstream_blocks: - - inference - executor_config: null - executor_type: local_python - has_callback: false - language: python - name: Model - retry_config: null - status: executed - timeout: null - type: global_data_product - upstream_blocks: [] - uuid: model -- all_upstream_blocks_executed: true - color: teal - configuration: - file_source: - path: null - downstream_blocks: [] - executor_config: null - executor_type: local_python - has_callback: false - language: python - name: inference - retry_config: null - status: failed - timeout: null - type: custom - upstream_blocks: - - training_data - - model - uuid: inference + - all_upstream_blocks_executed: true + color: null + configuration: + global_data_product: + uuid: training_set + downstream_blocks: + - inference + - model + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: Training data + retry_config: null + status: executed + timeout: null + type: global_data_product + upstream_blocks: [] + uuid: training_data + - all_upstream_blocks_executed: true + color: null + configuration: + file_source: + path: unit_4_triggering/global_data_products/model.py + global_data_product: + uuid: xgboost + downstream_blocks: + - inference + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: Model + retry_config: null + status: executed + timeout: null + type: global_data_product + upstream_blocks: [] + uuid: model + - all_upstream_blocks_executed: true + color: teal + configuration: + file_source: + path: null + downstream_blocks: [] + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: inference + retry_config: null + status: failed + timeout: null + type: custom + upstream_blocks: + - training_data + - model + uuid: inference cache_block_output_in_memory: false callbacks: [] concurrency_config: {} conditionals: [] -created_at: '2024-05-09 02:45:15.656239+00:00' +created_at: "2024-05-09 02:45:15.656239+00:00" data_integration: null description: Online inference pipeline. executor_config: {} @@ -80,6 +80,5 @@ tags: [] type: python uuid: predict variables: - model_dir: '' -variables_dir: /home/src/mage_data/unit_4_triggering + model_dir: "" widgets: [] diff --git a/mlops/unit_4_triggering/pipelines/xgboost_training/metadata.yaml b/mlops/unit_4_triggering/pipelines/xgboost_training/metadata.yaml index ce093075..3bc6ed14 100755 --- a/mlops/unit_4_triggering/pipelines/xgboost_training/metadata.yaml +++ b/mlops/unit_4_triggering/pipelines/xgboost_training/metadata.yaml @@ -106,5 +106,4 @@ variables: early_stopping_rounds: 1 max_depth: 1 max_evaluations: 1 -variables_dir: /home/src/mage_data/unit_4_triggering widgets: [] diff --git a/terraform/aws/scripts/create-github-actions-workflow.sh b/terraform/aws/scripts/create-github-actions-workflow.sh index 64c35bb7..dac08c7a 100755 --- a/terraform/aws/scripts/create-github-actions-workflow.sh +++ b/terraform/aws/scripts/create-github-actions-workflow.sh @@ -7,7 +7,7 @@ ECS_SERVICE="$4" ECS_TASK_DEFINITION="$5" CONTAINER_NAME="$6" -WORKFLOW_PATH="../../.github/workflows/deploy_to_ecs.yml" +WORKFLOW_PATH="../../.github/workflows/build_and_deploy_to_aws_ecs.yml" WORKFLOW_DIR=$(dirname "$WORKFLOW_PATH") mkdir -p "$WORKFLOW_DIR"