Simplify configuration and disable mlflow

forecasting_model/conf/base/catalog.yml

@@ -1,10 +1,6 @@
-# TODO: raw setting is needed only for vertexai integration - this could be moved to cloud config
-
 order_book:
-  type: pandas.CSVDataSet
-  filepath: data/01_raw/order_book.csv
-  layer: raw
-
-processed_order_book:
-  type: pandas.CSVDataSet
-  filepath: data/05_model_input/order_book.csv
+  type: pandas.GBQTableDataSet
+  dataset: order_book_568de2
+  # Temporarily taking only live data which is much smaller dataset
+  table_name: live
+  project: mlops-383318

forecasting_model/conf/base/logging.yml

@@ -13,25 +13,8 @@ handlers:
     formatter: simple
     stream: ext://sys.stdout
 
-  info_file_handler:
-    class: logging.handlers.RotatingFileHandler
-    level: INFO
-    formatter: simple
-    filename: logs/info.log
-    maxBytes: 10485760 # 10MB
-    backupCount: 20
-    encoding: utf8
-    delay: True
-
-  error_file_handler:
-    class: logging.handlers.RotatingFileHandler
-    level: ERROR
-    formatter: simple
-    filename: logs/errors.log
-    maxBytes: 10485760 # 10MB
-    backupCount: 20
-    encoding: utf8
-    delay: True
+  gcp:
+    class: google.cloud.logging_v2.handlers.structured_log.StructuredLogHandler
 
   rich:
     class: rich.logging.RichHandler
@@ -44,4 +27,4 @@ loggers:
     level: INFO
 
 root:
-  handlers: [rich, info_file_handler, error_file_handler]
+  handlers: [rich]

forecasting_model/conf/base/vertexai.yml

@@ -1,88 +1,24 @@
 
-# Configuration used to run the pipeline
-project_id: i-2r8v-y9cvkuz6-7kgq4ai8iwk1ie
+project_id: mlops-383318
 region: europe-west1
 run_config:
-  # Name of the image to run as the pipeline steps
-  image: eu.gcr.io/i-2r8v-y9cvkuz6-7kgq4ai8iwk1ie/forecasting_model-tutorial-mb:20230328
-
-  # Pull policy to be used for the steps. Use Always if you push the images
-  # on the same tag, or Never if you use only local images
+  image: europe-west1-docker.pkg.dev/mlops-383318/model/training:latest
   image_pull_policy: IfNotPresent
-
-  # Location of Vertex AI GCS root
-  root: my_workshop_temp_123/mlops-webinar-demo
-
-  # Prefix of Vertex AI pipeline run
-  experiment_name: forecasting_model
-
-  # Name of the scheduled run, templated with the schedule parameters
-  scheduled_run_name: forecasting_model
-
-  # Optional service account to run vertex AI Pipeline with
-  # service_account: pipelines-account@my-project.iam.gserviceaccount.com
-
-  # Optional pipeline description
-  # description: "Very Important Pipeline"
-
-  # How long to keep underlying Argo workflow (together with pods and data
-  # volume after pipeline finishes) [in seconds]. Default: 1 week
+  root: experimentation-data-568de2
+  experiment_name: forecasting
+  scheduled_run_name: forecasting
   ttl: 604800
 
-  # Optional network configuration
-  # network:
-
-    # Name of the vpc to use for running Vertex Pipeline
-    # vpc: my-vpc
-
-    # Hosts aliases to be placed in /etc/hosts when pipeline is executed
-    # host_aliases:
-    #  - ip: 127.0.0.1
-    #    hostnames:
-    #     - me.local
-
-  # What Kedro pipeline should be run as the last step regardless of the
-  # pipeline status. Used to send notifications or raise the alerts
-  # on_exit_pipeline: notify_via_slack
-
-  # Optional section allowing adjustment of the resources, reservations and limits
-  # for the nodes. When not provided they're set to 500m cpu and 1024Mi memory.
-  # If you don't want to specify pipeline resources set both to None in __default__.
   resources:
-
-    # For nodes that require more RAM you can increase the "memory"
-    data_import_step:
-      memory: 4Gi
-
-    # Training nodes can utilize more than one CPU if the algoritm
-    # supports it
-    model_training:
+    # XGBoost can utilize more than one CPU core
+    train_model:
       cpu: 8
-      memory: 8Gi
-      gpu: 1
+      memory: 16Gi
+      gpu: 0
 
     # Default settings for the nodes
     __default__:
-      cpu: 1000m
+      cpu: 1
       memory: 2048Mi
 
-  node_selectors:
-    model_training:
-      cloud.google.com/gke-accelerator: NVIDIA_TESLA_T4
-
-  # Optional section allowing to generate config files at runtime,
-  # useful e.g. when you need to obtain credentials dynamically and store them in credentials.yaml
-  # but the credentials need to be refreshed per-node
-  # (which in case of Vertex AI would be a separate container / machine)
-  # Example:
-  # dynamic_config_providers:
-  #  - cls: kedro_vertexai.auth.gcp.MLFlowGoogleOAuthCredentialsProvider
-  #    params:
-  #      client_id: iam-client-id
-
   dynamic_config_providers: []
-
-  # Additional configuration for MLflow request header providers, e.g. to generate access tokens at runtime
-  # mlflow:
-  #   request_header_provider_params:
-  #       key: value

forecasting_model/conf/cloud/logging.yml

@@ -0,0 +1,3 @@
+root:
+  # GCP handler isn't fully ready
+  handlers: [console]

forecasting_model/src/forecasting_model/pipelines/data_science/nodes.py

@@ -1,5 +1,6 @@
 import logging
-import mlflow
+
+# import mlflow
 from typing import Any, Dict, Tuple
 
 import pandas as pd
@@ -73,7 +74,7 @@ def evaluate_model(
         stage,
     )
 
-    mlflow.log_metric(f"{stage}_max_error", max_error)
-    mlflow.log_metric(f"{stage}_median_error", median_error)
-    mlflow.log_metric(f"{stage}_mse_error", mse_error)
-    mlflow.log_metric(f"{stage}_explained_variance", explained_variance)
+    # mlflow.log_metric(f"{stage}_max_error", max_error)
+    # mlflow.log_metric(f"{stage}_median_error", median_error)
+    # mlflow.log_metric(f"{stage}_mse_error", mse_error)
+    # mlflow.log_metric(f"{stage}_explained_variance", explained_variance)

forecasting_model/src/forecasting_model/settings.py

@@ -35,3 +35,21 @@
 # Class that manages the Data Catalog.
 # from kedro.io import DataCatalog
 # DATA_CATALOG_CLASS = DataCatalog
+
+# Overwriting Rich exception hook
+import sys
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def handle_exception(exc_type: Any, exc_value: Any, exc_traceback: Any) -> None:
+    if issubclass(exc_type, KeyboardInterrupt):
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+        return
+
+    logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+
+sys.excepthook = handle_exception

forecasting_model/src/requirements.txt

@@ -1,18 +1,21 @@
 ipython>=7.31.1, <8.0
 isort~=5.0
+
 jupyter~=1.0
 jupyterlab_server>=2.11.1, <2.16.0
 jupyterlab~=3.0
+
 kedro~=0.18.4
-kedro-datasets[pandas.CSVDataSet, pandas.ExcelDataSet, pandas.ParquetDataSet]~=1.0.0
+kedro-datasets[pandas.CSVDataSet, pandas.ExcelDataSet, pandas.ParquetDataSet,pandas.GBQTableDataSet]~=1.0.0
 kedro-telemetry~=0.2.0
 kedro-viz~=5.0
+kedro-vertexai==0.8.1
+
 nbstripout~=0.4
 pytest-cov~=3.0
 pytest-mock>=1.7.1, <2.0
 pytest~=6.2
 scikit-learn~=1.0
 xgboost~=1.6.2
-kedro-vertexai==0.8.1
-kedro-docker==0.3.1
-kedro-mlflow~=0.11.8
+
+google-cloud-logging~=3.5.0