diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala index 9c4fd67493..b293c77a0b 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala @@ -31,7 +31,7 @@ object DatabricksUtilities { val Region = "eastus" val PoolName = "synapseml-build-10.4" val GpuPoolName = "synapseml-build-10.4-gpu" - val AdbRuntime = "10.4.x-scala2.12" + val AdbRuntime = "12.2.x-scala2.12" val AdbGpuRuntime = "10.4.x-gpu-ml-scala2.12" val NumWorkers = 5 val AutoTerminationMinutes = 15 @@ -57,7 +57,11 @@ object DatabricksUtilities { Map("pypi" -> Map("package" -> "onnxmltools==1.7.0")), Map("pypi" -> Map("package" -> "lightgbm")), Map("pypi" -> Map("package" -> "mlflow")), - Map("pypi" -> Map("package" -> "openai")) + Map("pypi" -> Map("package" -> "openai")), + Map("pypi" -> Map("package" -> "raiwidgets")), + Map("pypi" -> Map("package" -> "itsdangerous==2.0.1")), + Map("pypi" -> Map("package" -> "hyperopt")), + Map("pypi" -> Map("package" -> "scikit-learn")) ).toJson.compactPrint // TODO: install synapse.ml.dl wheel package here diff --git a/notebooks/features/hyperparameter_tuning/HyperOpt-SynapseML.ipynb b/notebooks/features/hyperparameter_tuning/HyperOpt-SynapseML.ipynb index 0911458fdb..f8952e8a1c 100644 --- a/notebooks/features/hyperparameter_tuning/HyperOpt-SynapseML.ipynb +++ b/notebooks/features/hyperparameter_tuning/HyperOpt-SynapseML.ipynb @@ -32,9 +32,7 @@ "outputs": [], "source": [ "# %pip install hyperopt\n", - "import os\n", - "\n", - "os.system(\"pip install hyperopt\")" + "# %pip install scikit-learn" ] }, { @@ -54,8 +52,7 @@ "outputs": [], "source": [ "# version >= 1.28.0 supports reading logModelAllowlistFile from url\n", - "# %pip install mlflow==1.29.0\n", - "os.system(\"pip install mlflow==1.29.0\")" + "# %pip install mlflow" ] }, { @@ -71,12 +68,15 @@ "\n", "if running_on_synapse_internal():\n", " experiment_name = \"hyperopt-synapseml\"\n", + " os.system(\"pip install mlflow\")\n", "elif running_on_synapse():\n", " experiment_name = \"hyperopt-synapseml\"\n", " # from notebookutils.visualization import display # use this display on interactive notebook\n", " from synapse.ml.core.platform import (\n", " materializing_display as display,\n", " ) # display for pipeline testing\n", + "\n", + " os.system(\"pip install mlflow\")\n", "else:\n", " experiment_name = \"/Shared/hyperopt-synapseml\"" ] diff --git a/notebooks/features/isolation_forest/IsolationForest - Multivariate Anomaly Detection.ipynb b/notebooks/features/isolation_forest/IsolationForest - Multivariate Anomaly Detection.ipynb index 3f288df84e..624eeebbfa 100644 --- a/notebooks/features/isolation_forest/IsolationForest - Multivariate Anomaly Detection.ipynb +++ b/notebooks/features/isolation_forest/IsolationForest - Multivariate Anomaly Detection.ipynb @@ -27,6 +27,27 @@ " - If you are running it on Synapse, you'll need to [create an AML workspace and set up linked Service](https://microsoft.github.io/SynapseML/docs/next/mlflow/installation/). \n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we install required dependencies and import the packages and define some UDFs we will need later.\n", + "When running Synapse Analytics, please follow instructions here [Package management - Azure Synapse Analytics | Microsoft Docs](https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries) to install [\"raiwidgets\"](https://pypi.org/project/raiwidgets/) and [\"interpret-community\"](https://pypi.org/project/interpret-community/) packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# %pip install raiwidgets\n", + "# %pip install itsdangerous==2.0.1\n", + "# %pip install interpret-community" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/notebooks/features/responsible_ai/Interpretability - Explanation Dashboard.ipynb b/notebooks/features/responsible_ai/Interpretability - Explanation Dashboard.ipynb index 134564dc48..eb0e937004 100644 --- a/notebooks/features/responsible_ai/Interpretability - Explanation Dashboard.ipynb +++ b/notebooks/features/responsible_ai/Interpretability - Explanation Dashboard.ipynb @@ -15,7 +15,27 @@ "\n", "In this example, similar to the \"Interpretability - Tabular SHAP explainer\" notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from https://github.com/microsoft/responsible-ai-widgets.\n", "\n", - "First we import the packages and define some UDFs we will need later." + "First we install required dependencies and import the packages and define some UDFs we will need later." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When running Synapse Analytics, please follow instructions here [Package management - Azure Synapse Analytics | Microsoft Docs](https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries) to install [\"raiwidgets\"](https://pypi.org/project/raiwidgets/) and [\"interpret-community\"](https://pypi.org/project/interpret-community/) packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# %pip install raiwidgets\n", + "# %pip install itsdangerous==2.0.1\n", + "# %pip install interpret-community" ] }, { @@ -312,36 +332,6 @@ " converted_importance_values.append(converted_list)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When running Synapse Analytics, please follow instructions here [Package management - Azure Synapse Analytics | Microsoft Docs](https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries) to install [\"raiwidgets\"](https://pypi.org/project/raiwidgets/) and [\"interpret-community\"](https://pypi.org/project/interpret-community/) packages." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "!pip install --upgrade raiwidgets\n", - "!pip install itsdangerous==2.0.1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "!pip install --upgrade interpret-community" - ] - }, { "cell_type": "code", "execution_count": null,