SNOW-2256092: Release snowflake-ml-python 1.11.0

CHANGELOG.md

@@ -1,11 +1,31 @@
 # Release History
 
-## 1.10.0
+## 1.11.0
 
 ### Bug Fixes
 
+* ML Job: Fix `Error: Unable to retrieve head IP address` if not all instances start within the timeout.
+* ML Job: Fix `TypeError: SnowflakeCursor.execute() got an unexpected keyword argument '_force_qmark_paramstyle'`
+  when running inside Stored Procedures.
+
+### Behavior Changes
+
+### New Features
+
+* `ModelVersion.create_service()`: Made `image_repo` argument optional. By
+  default it will use a default image repo, which is
+  being rolled out in server version 9.22+.
+* Experiment Tracking (PrPr): Automatically log the model, metrics, and parameters while training Keras models with
+  `snowflake.ml.experiment.callback.keras.SnowflakeKerasCallback`.
+
+## 1.10.0
+
 ### Behavior Changes
 
+* Experiment Tracking (PrPr): The import paths for the auto-logging callbacks have changed to
+  `snowflake.ml.experiment.callback.xgboost.SnowflakeXgboostCallback` and
+  `snowflake.ml.experiment.callback.lightgbm.SnowflakeLightgbmCallback`.
+
 ### New Features
 
 * Registry: add progress bars for `ModelVersion.create_service` and `ModelVersion.log_model`.
@@ -26,13 +46,13 @@
 
 ```python
 from snowflake.ml.experiment import ExperimentTracking
+from snowflake.ml.experiment.callback import SnowflakeXgboostCallback, SnowflakeLightgbmCallback
 
 exp = ExperimentTracking(session=sp_session, database_name="ML", schema_name="PUBLIC")
 
 exp.set_experiment("MY_EXPERIMENT")
 
 # XGBoost
-from snowflake.ml.experiment.callback.xgboost import SnowflakeXgboostCallback
 callback = SnowflakeXgboostCallback(
   exp, log_model=True, log_metrics=True, log_params=True, model_name="model_name", model_signature=sig
 )
@@ -41,7 +61,6 @@ with exp.start_run():
   model.fit(X, y, eval_set=[(X_test, y_test)])
 
 # LightGBM
-from snowflake.ml.experiment.callback.lightgbm import SnowflakeLightgbmCallback
 callback = SnowflakeLightgbmCallback(
   exp, log_model=True, log_metrics=True, log_params=True, model_name="model_name", model_signature=sig
 )

bazel/environments/conda-env-all.yml

@@ -12,7 +12,7 @@ dependencies:
   - anyio==4.2.0
   - boto3==1.34.69
   - cachetools==5.3.3
-  - catboost==1.2.0
+  - catboost==1.2.8
   - cloudpickle==2.2.1
   - coverage==7.2.2
   - cryptography==41.0.3

bazel/environments/conda-env-ml.yml

@@ -12,7 +12,7 @@ dependencies:
   - anyio==4.2.0
   - boto3==1.34.69
   - cachetools==5.3.3
-  - catboost==1.2.0
+  - catboost==1.2.8
   - cloudpickle==2.2.1
   - coverage==7.2.2
   - cryptography==41.0.3

bazel/environments/requirements_ml.txt

@@ -8,7 +8,7 @@ anyio==4.2.0
 boto3==1.34.69
 build==0.10.0
 cachetools==5.3.3
-catboost==1.2.0
+catboost==1.2.8
 cloudpickle==2.2.1
 coverage==7.2.2
 cryptography==41.0.3

ci/RunBazelAction.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # DESCRIPTION: Utility Shell script to run bazel action for snowml repository
 #
-# RunBazelAction.sh <test|coverage> [-b <bazel_path>] [-m merge_gate|continuous_run|quarantined|local_unittest|local_all] [-t <target>] [-c <path_to_coverage_report>] [--tags <tags>]
+# RunBazelAction.sh <test|coverage> [-b <bazel_path>] [-m merge_gate|continuous_run|quarantined|local_unittest|local_all] [-t <target>] [-c <path_to_coverage_report>] [--tags <tags>] [--with-spcs-image]
 #
 # Args:
 #   action: bazel action, choose from test and coverage
@@ -18,6 +18,7 @@
 #   -c: specify the path to the coverage report dat file.
 #   -e: specify the environment, used to determine.
 #   --tags: specify bazel test tag filters (e.g., "feature:jobs,feature:data")
+#   --with-spcs-image: use spcs image for testing.
 #
 
 set -o pipefail
@@ -40,6 +41,7 @@ help() {
     echo ""
     echo "Options:"
     echo "  --tags <tags>       Specify bazel tag filters (comma-separated)"
+    echo "  --with-spcs-image   Use spcs image for testing."
     echo ""
     echo "Examples:"
     echo "  ${PROG} test --tags 'feature:jobs'"
@@ -109,7 +111,7 @@ fi
 action_env=()
 
 if [[ "${WITH_SPCS_IMAGE}" = true ]]; then
-    export SKIP_GRYPE=true
+    export RUN_GRYPE=false
     source model_container_services_deployment/ci/build_and_push_images.sh
     action_env=("--action_env=BUILDER_IMAGE_PATH=${BUILDER_IMAGE_PATH}" "--action_env=BASE_CPU_IMAGE_PATH=${BASE_CPU_IMAGE_PATH}" "--action_env=BASE_GPU_IMAGE_PATH=${BASE_GPU_IMAGE_PATH}" "--action_env=IMAGE_BUILD_SIDECAR_CPU_PATH=${IMAGE_BUILD_SIDECAR_CPU_PATH}" "--action_env=IMAGE_BUILD_SIDECAR_GPU_PATH=${IMAGE_BUILD_SIDECAR_GPU_PATH}" "--action_env=PROXY_IMAGE_PATH=${PROXY_IMAGE_PATH}" "--action_env=VLLM_IMAGE_PATH=${VLLM_IMAGE_PATH}")
 fi

ci/build_and_run_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Usage
-# build_and_run_tests.sh <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run] [--with-snowpark] [--with-spcs-image] [--report <report_path>]
+# build_and_run_tests.sh <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run] [--with-snowpark] [--with-spcs-image] [--run-grype] [--report <report_path>]
 #
 # Args
 # workspace: path to the workspace, SnowML code should be in snowml directory.
@@ -15,6 +15,7 @@
 #   quarantined: run all quarantined tests.
 # with-snowpark: Build and test with snowpark in snowpark-python directory in the workspace.
 # with-spcs-image: Build and test with spcs-image in spcs-image directory in the workspace.
+# run-grype: Run grype security scanning on SPCS images. Only valid with --with-spcs-image.
 # snowflake-env: The environment of the snowflake, use to determine the test quarantine list
 # report: Path to xml test report
 #
@@ -30,7 +31,7 @@ PROG=$0
 
 help() {
     local exit_code=$1
-    echo "Usage: ${PROG} <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run|quarantined] [--with-snowpark] [--with-spcs-image] [--snowflake-env <sf_env>] [--report <report_path>]"
+    echo "Usage: ${PROG} <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run|quarantined] [--with-snowpark] [--with-spcs-image] [--run-grype] [--snowflake-env <sf_env>] [--report <report_path>]"
     exit "${exit_code}"
 }
 
@@ -39,6 +40,7 @@ BAZEL="bazel"
 ENV="pip"
 WITH_SNOWPARK=false
 WITH_SPCS_IMAGE=false
+RUN_GRYPE=false
 MODE="continuous_run"
 PYTHON_VERSION=3.9
 PYTHON_ENABLE_SCRIPT="bin/activate"
@@ -91,6 +93,9 @@ while (($#)); do
     --with-spcs-image)
         WITH_SPCS_IMAGE=true
         ;;
+    --run-grype)
+        RUN_GRYPE=true
+        ;;
     -h | --help)
         help 0
         ;;
@@ -101,6 +106,12 @@ while (($#)); do
     shift
 done
 
+# Validate flag combinations
+if [ "${RUN_GRYPE}" = true ] && [ "${WITH_SPCS_IMAGE}" = false ]; then
+    echo "Error: --run-grype flag requires --with-spcs-image to be set"
+    help 1
+fi
+
 echo "Running build_and_run_tests with PYTHON_VERSION ${PYTHON_VERSION}"
 
 EXT=""
@@ -180,6 +191,25 @@ trap 'rm -rf "${TEMP_BIN}"' EXIT
 # Install micromamba
 _MICROMAMBA_BIN="micromamba${EXT}"
 if [ "${ENV}" = "conda" ]; then
+    CONDA="/mnt/jenkins/home/jenkins/miniforge3/condabin/conda"
+
+    # Check if miniforge is already installed
+    if [ -x "${CONDA}" ]; then
+        echo "Miniforge exists at ${CONDA}."
+    else
+        echo "Downloading miniforge ..."
+        curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+
+        echo "Installing miniforge ..."
+        /bin/bash "Miniforge3-$(uname)-$(uname -m).sh" -b -u
+    fi
+
+    echo "Using ${CONDA} ..."
+
+    echo "Installing conda-build ..."
+    ${CONDA} install conda-build --yes
+
+    echo "Installing micromamba ..."
     if ! command -v "${_MICROMAMBA_BIN}" &>/dev/null; then
         curl -Lsv "https://github.com/mamba-org/micromamba-releases/releases/latest/download/micromamba-${MICROMAMBA_PLATFORM}-${MICROMAMBA_ARCH}" -o "${TEMP_BIN}/micromamba${EXT}" && chmod +x "${TEMP_BIN}/micromamba${EXT}"
         _MICROMAMBA_BIN="${TEMP_BIN}/micromamba${EXT}"
@@ -264,30 +294,31 @@ if [ "${ENV}" = "pip" ]; then
     cp "$("${BAZEL}" "${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]+"${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]}"}" info bazel-bin)/dist/snowflake_ml_python-${VERSION}-py3-none-any.whl" "${WORKSPACE}"
     popd
 else
-    # Clean conda cache
-    conda clean --all --force-pkgs-dirs -y
+    echo "Cleaning conda cache ..."
+    ${CONDA} clean --all --force-pkgs-dirs -y
 
-    # Clean conda build workspace
+    echo "Cleaning conda build workspace ..."
     rm -rf "${WORKSPACE}/conda-bld"
 
-    # Build Snowpark
+    echo "Building snowpark-python conda package ..."
     if [ "${WITH_SNOWPARK}" = true ]; then
         pushd ${SNOWPARK_DIR}
-        conda build recipe/ --python=${PYTHON_VERSION} --numpy=1.16 --croot "${WORKSPACE}/conda-bld"
+        ${CONDA} build recipe/ --python=${PYTHON_VERSION} --numpy=1.16 --croot "${WORKSPACE}/conda-bld"
         popd
     fi
 
-    # Build SnowML
     pushd ${SNOWML_DIR}
-    # Build conda package
-    conda build -c conda-forge --override-channels --prefix-length 50 --python=${PYTHON_VERSION} --croot "${WORKSPACE}/conda-bld" ci/conda_recipe
-    conda build purge
+
+    echo "Building snowflake-ml-python conda package ..."
+    ${CONDA} build -c conda-forge --override-channels --prefix-length 50 --python=${PYTHON_VERSION} --croot "${WORKSPACE}/conda-bld" ci/conda_recipe
+    ${CONDA} build purge
     popd
 fi
 
 if [[ "${WITH_SPCS_IMAGE}" = true ]]; then
     pushd ${SNOWML_DIR}
-    # Build SPCS Image
+    echo "Building SPCS Image ..."
+    export RUN_GRYPE
     source model_container_services_deployment/ci/build_and_push_images.sh
     popd
 fi
@@ -361,7 +392,7 @@ for i in "${!groups[@]}"; do
             COMMON_PYTEST_FLAG+=(-m "not conda_incompatible")
         fi
         # Create local conda channel
-        conda index "${WORKSPACE}/conda-bld"
+        ${CONDA} index "${WORKSPACE}/conda-bld"
 
         # Clean conda cache
         "${_MICROMAMBA_BIN}" clean --all --force-pkgs-dirs -y
@@ -384,7 +415,7 @@ for i in "${!groups[@]}"; do
 
         # Run integration tests
         set +e
-        TEST_SRCDIR="${TEMP_TEST_DIR}" conda run -p ./testenv --no-capture-output python -m pytest "${COMMON_PYTEST_FLAG[@]}" tests/integ/
+        TEST_SRCDIR="${TEMP_TEST_DIR}" ${CONDA} run -p ./testenv --no-capture-output python -m pytest "${COMMON_PYTEST_FLAG[@]}" tests/integ/
         group_exit_codes[$i]=$?
         set -e
 

ci/conda_recipe/meta.yaml

@@ -17,7 +17,7 @@ build:
   noarch: python
 package:
   name: snowflake-ml-python
-  version: 1.10.0
+  version: 1.11.0
 requirements:
   build:
     - python

ci/targets/quarantine/prod3.txt

@@ -1,7 +1,7 @@
 //tests/integ/snowflake/ml/extra_tests:xgboost_external_memory_training_test
 //tests/integ/snowflake/ml/extra_tests:pipeline_with_ohe_and_xgbr_test
-//tests/integ/snowflake/ml/lineage:lineage_integ_test
 //tests/integ/snowflake/ml/modeling/manifold:spectral_embedding_test
 //tests/integ/snowflake/ml/modeling/linear_model:logistic_regression_test
 //tests/integ/snowflake/ml/registry/services:registry_huggingface_pipeline_model_deployment_test
 //tests/integ/snowflake/ml/registry/services:registry_sentence_transformers_model_deployment_test
+//tests/integ/snowflake/ml/jobs:jobs_integ_test

requirements.yml

@@ -84,7 +84,7 @@
 - name: boto3
   dev_version: 1.34.69
 - name: catboost
-  dev_version: 1.2.0
+  dev_version: 1.2.8
   version_requirements: '>=1.2.0, <2'
   requirements_extra_tags:
     - catboost

snowflake/ml/experiment/callback/BUILD.bazel

@@ -11,6 +11,29 @@ py_library(
     ],
 )
 
+py_library(
+    name = "keras",
+    srcs = ["keras.py"],
+    deps = [
+        "//snowflake/ml/experiment:experiment_tracking",
+        "//snowflake/ml/experiment:utils",
+        "//snowflake/ml/model:model_signature",
+    ],
+)
+
+py_test(
+    name = "keras_test",
+    srcs = ["test/keras_test.py"],
+    optional_dependencies = [
+        "keras",
+    ],
+    tags = ["feature:observability"],
+    deps = [
+        ":keras",
+        ":test_base",
+    ],
+)
+
 py_library(
     name = "lightgbm",
     srcs = ["lightgbm.py"],
@@ -56,6 +79,7 @@ py_test(
 py_library(
     name = "callback",
     deps = [
+        ":keras",
         ":lightgbm",
         ":xgboost",
     ],

snowflake/ml/experiment/callback/keras.py

@@ -0,0 +1,63 @@
+import json
+from typing import TYPE_CHECKING, Any, Optional
+from warnings import warn
+
+import keras
+
+from snowflake.ml.experiment import utils
+
+if TYPE_CHECKING:
+    from snowflake.ml.experiment.experiment_tracking import ExperimentTracking
+    from snowflake.ml.model.model_signature import ModelSignature
+
+
+class SnowflakeKerasCallback(keras.callbacks.Callback):
+    def __init__(
+        self,
+        experiment_tracking: "ExperimentTracking",
+        log_model: bool = True,
+        log_metrics: bool = True,
+        log_params: bool = True,
+        log_every_n_epochs: int = 1,
+        model_name: Optional[str] = None,
+        model_signature: Optional["ModelSignature"] = None,
+    ) -> None:
+        self._experiment_tracking = experiment_tracking
+        self.log_model = log_model
+        self.log_metrics = log_metrics
+        self.log_params = log_params
+        if log_every_n_epochs < 1:
+            raise ValueError("`log_every_n_epochs` must be positive.")
+        self.log_every_n_epochs = log_every_n_epochs
+        self.model_name = model_name
+        self.model_signature = model_signature
+
+    def on_train_begin(self, logs: Optional[dict[str, Any]] = None) -> None:
+        if self.log_params:
+            params = json.loads(self.model.to_json())
+            self._experiment_tracking.log_params(utils.flatten_nested_params(params))
+
+    def on_epoch_end(self, epoch: int, logs: Optional[dict[str, Any]] = None) -> None:
+        if self.log_metrics and logs and epoch % self.log_every_n_epochs == 0:
+            for key, value in logs.items():
+                try:
+                    value = float(value)
+                except Exception:
+                    pass
+                else:
+                    self._experiment_tracking.log_metric(key=key, value=value, step=epoch)
+
+    def on_train_end(self, logs: Optional[dict[str, Any]] = None) -> None:
+        if self.log_model:
+            if not self.model_signature:
+                warn(
+                    "Model will not be logged because model signature is missing. "
+                    "To autolog the model, please specify `model_signature` when constructing SnowflakeKerasCallback."
+                )
+                return
+            model_name = self.model_name or self._experiment_tracking._get_or_set_experiment().name + "_model"
+            self._experiment_tracking.log_model(  # type: ignore[call-arg]
+                model=self.model,
+                model_name=model_name,
+                signatures={"predict": self.model_signature},
+            )