example-2/example-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7bfbba62-429c-4091-b0a1-8797f46bb4a5"
   },
   "source": [
    "# Example 2: Invoke deployed models/functions from Watson Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3c11bf6c-be7f-4186-9ab6-7d1ec08dacd3"
   },
   "source": [
    "## Use case\n",
    "\n",
    "\n",
    "A complex/advance model is developed by the Data Science team and deployed on Watson Machine Learning.\n",
    "\n",
    "The business users could then visually apply business rules and preprocess the data as required, before finally making a prediction using the deployed model.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d63e95e1-3d0d-48ff-b0fd-8ba0b8cc17c3"
   },
   "source": [
    "# Build a model in Watson Studio and deploy in Watson Machine Learning (Data Scientist persona)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "9aedb0a9-f35c-4c7b-be6d-75c09c0f9cc8",
    "tags": []
   },
   "outputs": [],
   "source": [
    "! pip install --quiet optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "52de31c5-c2f8-462b-ae0e-d8f271d9c77f",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import optuna\n",
    "import sklearn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import ibm_watson_machine_learning\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.compose import make_column_selector\n",
    "from sklearn.compose import make_column_transformer\n",
    "\n",
    "import itc_utils.flight_service as itcfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "c0c97df6-9a89-4fe8-a4b8-a94ef3581340",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'SUCCESS'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SPACE_ID = \"5d163188-8cf0-405c-94e6-077fae0bd306\"\n",
    "\n",
    "wml_credentials = {\n",
    "    \"token\": os.environ[\"USER_ACCESS_TOKEN\"],\n",
    "    \"url\": os.environ[\"RUNTIME_ENV_APSX_URL\"],\n",
    "    \"instance_id\": \"openshift\",\n",
    "    \"version\": \"4.5\"\n",
    "}\n",
    "\n",
    "wml_client = ibm_watson_machine_learning.APIClient(wml_credentials)\n",
    "wml_client.set.default_space(SPACE_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "06362577-e43c-4645-84fa-d61f5829bfa8",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Income</th>\n",
       "      <th>AppliedOnline</th>\n",
       "      <th>Residence</th>\n",
       "      <th>YearCurrentAddress</th>\n",
       "      <th>YearsCurrentEmployer</th>\n",
       "      <th>NumberOfCards</th>\n",
       "      <th>CCDebt</th>\n",
       "      <th>Loans</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>SalePrice</th>\n",
       "      <th>Location</th>\n",
       "      <th>MortgageDefault</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43593.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2315.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12820.0</td>\n",
       "      <td>180000</td>\n",
       "      <td>130</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>45706.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>17.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>373.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7275.0</td>\n",
       "      <td>145000</td>\n",
       "      <td>100</td>\n",
       "      <td>YES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44756.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>19.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2117.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10760.0</td>\n",
       "      <td>145000</td>\n",
       "      <td>110</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44202.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>748.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10455.0</td>\n",
       "      <td>170000</td>\n",
       "      <td>100</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>45715.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>8.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>772.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12985.0</td>\n",
       "      <td>137000</td>\n",
       "      <td>100</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>43800.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>725.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7340.0</td>\n",
       "      <td>259000</td>\n",
       "      <td>100</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>45049.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Public Housing</td>\n",
       "      <td>6.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1345.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9085.0</td>\n",
       "      <td>280000</td>\n",
       "      <td>110</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>44974.0</td>\n",
       "      <td>YES</td>\n",
       "      <td>Public Housing</td>\n",
       "      <td>14.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2772.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9515.0</td>\n",
       "      <td>264000</td>\n",
       "      <td>130</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>44956.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>Owner Occupier</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>806.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8150.0</td>\n",
       "      <td>170000</td>\n",
       "      <td>100</td>\n",
       "      <td>YES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>56087.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>Public Housing</td>\n",
       "      <td>27.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4818.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8910.0</td>\n",
       "      <td>95000</td>\n",
       "      <td>101</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Income AppliedOnline       Residence  YearCurrentAddress  \\\n",
       "0  43593.0           YES  Owner Occupier                13.0   \n",
       "1  45706.0           YES  Owner Occupier                17.0   \n",
       "2  44756.0           YES  Owner Occupier                19.0   \n",
       "3  44202.0           YES  Owner Occupier                 8.0   \n",
       "4  45715.0           YES  Owner Occupier                 8.0   \n",
       "5  43800.0           YES  Owner Occupier                 0.0   \n",
       "6  45049.0           YES  Public Housing                 6.0   \n",
       "7  44974.0           YES  Public Housing                14.0   \n",
       "8  44956.0            NO  Owner Occupier                21.0   \n",
       "9  56087.0            NO  Public Housing                27.0   \n",
       "\n",
       "   YearsCurrentEmployer  NumberOfCards  CCDebt  Loans  LoanAmount  SalePrice  \\\n",
       "0                   0.0            1.0  2315.0    0.0     12820.0     180000   \n",
       "1                  16.0            2.0   373.0    1.0      7275.0     145000   \n",
       "2                   6.0            1.0  2117.0    1.0     10760.0     145000   \n",
       "3                   0.0            2.0   748.0    0.0     10455.0     170000   \n",
       "4                  14.0            2.0   772.0    1.0     12985.0     137000   \n",
       "5                   4.0            2.0   725.0    0.0      7340.0     259000   \n",
       "6                  16.0            2.0  1345.0    1.0      9085.0     280000   \n",
       "7                  18.0            2.0  2772.0    0.0      9515.0     264000   \n",
       "8                  21.0            2.0   806.0    1.0      8150.0     170000   \n",
       "9                  18.0            1.0  4818.0    1.0      8910.0      95000   \n",
       "\n",
       "   Location MortgageDefault  \n",
       "0       130              NO  \n",
       "1       100             YES  \n",
       "2       110              NO  \n",
       "3       100              NO  \n",
       "4       100              NO  \n",
       "5       100              NO  \n",
       "6       110              NO  \n",
       "7       130              NO  \n",
       "8       100             YES  \n",
       "9       101              NO  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_data_request = {\n",
    "    \"data_name\": \"mortgage-default.csv\",\n",
    "    \"interaction_properties\": {\"infer_schema\": \"true\"}\n",
    "}\n",
    "\n",
    "flightInfo = itcfs.get_flight_info(itcfs.get_flight_client(), nb_data_request=nb_data_request)\n",
    "\n",
    "df = itcfs.read_pandas_and_concat(itcfs.get_flight_client(), flightInfo)\n",
    "df = df.drop(\"ID\", axis=1)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "1e85b620-d86b-48dd-b29a-8fbc9e95eb22",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = \"MortgageDefault\"\n",
    "\n",
    "y = df[target].map({\"YES\": 1, \"NO\": 0})\n",
    "X = df.drop(target, axis=1)\n",
    "\n",
    "ct = make_column_transformer(\n",
    "    (OneHotEncoder(), make_column_selector(dtype_include=object)),\n",
    "    remainder=\"passthrough\"\n",
    ")\n",
    "X_transformed = ct.fit_transform(X)\n",
    "\n",
    "params = {\n",
    "    \"objective\": \"binary\",\n",
    "    \"metric\": \"auc\",\n",
    "    \"verbosity\": -1,\n",
    "    \"boosting_type\": \"gbdt\",\n",
    "}\n",
    "data = optuna.integration.lightgbm.Dataset(X_transformed, label=y)\n",
    "tuner = optuna.integration.lightgbm.LightGBMTunerCV(\n",
    "    params, data, verbose_eval=None, callbacks=[lgb.early_stopping(stopping_rounds=10, verbose=False)], folds=sklearn.model_selection.KFold(n_splits=3), optuna_seed=12345\n",
    ")\n",
    "tuner.run()\n",
    "best_params = tuner.best_params\n",
    "param_names = {\n",
    "    \"feature_fraction\": \"colsample_bytree\",\n",
    "    \"lambda_l1\": \"reg_alpha\",\n",
    "    \"bagging_fraction\": \"subsample\",\n",
    "    \"lambda_l2\": \"reg_lambda\",\n",
    "    \"bagging_freq\": \"subsample_freq\",\n",
    "}\n",
    "for k, v in param_names.items():\n",
    "    best_params[v] = best_params.pop(k)\n",
    "\n",
    "pipeline = Pipeline(steps=[(\"transform\", ct), (\"clf\", lgb.LGBMClassifier())])\n",
    "\n",
    "pipeline.fit(X, y)\n",
    "\n",
    "pipeline.score(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "56fcf5cf-199e-4d69-adbf-be8212024eed",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "#######################################################################################\n",
      "\n",
      "Synchronous deployment creation for uid: '1384f077-50aa-48d2-b789-a76b27577ce5' started\n",
      "\n",
      "#######################################################################################\n",
      "\n",
      "\n",
      "initializing\n",
      "Note: online_url is deprecated and will be removed in a future release. Use serving_urls instead.\n",
      "..\n",
      "ready\n",
      "\n",
      "\n",
      "------------------------------------------------------------------------------------------------\n",
      "Successfully finished deployment creation, deployment_uid='17ad92c5-cd7a-4914-ad49-14adc5c919ca'\n",
      "------------------------------------------------------------------------------------------------\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'17ad92c5-cd7a-4914-ad49-14adc5c919ca'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_props = {\n",
    "    wml_client.repository.ModelMetaNames.NAME: \"mortgage model\",\n",
    "    wml_client.repository.ModelMetaNames.TYPE: \"scikit-learn_1.0\",\n",
    "    wml_client.repository.ModelMetaNames.SOFTWARE_SPEC_UID: wml_client.software_specifications.get_uid_by_name(\"runtime-22.1-py3.9\")\n",
    "}\n",
    "\n",
    "model_details = wml_client.repository.store_model(model=pipeline, meta_props=meta_props)\n",
    "model_uid = wml_client.repository.get_model_id(model_details)\n",
    "meta_props = {wml_client.deployments.ConfigurationMetaNames.NAME: \"mortgage model deployment\", wml_client.deployments.ConfigurationMetaNames.ONLINE: {}}\n",
    "deployment_details = wml_client.deployments.create(model_uid, meta_props=meta_props)\n",
    "deployment_id = wml_client.deployments.get_id(deployment_details)\n",
    "deployment_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "ebd46b4b-2897-47b1-9bca-b4dab9ee2816",
    "tags": []
   },
   "outputs": [],
   "source": [
    "payload = {\"input_data\": [{\"fields\": X.columns.tolist(), \"values\": X.sample(10)}]}\n",
    "response = wml_client.deployments.score(deployment_id, payload)\n",
    "response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5155d6a5-2086-4d03-93da-56d11f4e4c4b"
   },
   "source": [
    "# Make predictions from the deployed model in SPSS Modeler flow (Power Business User persona)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "11cff1f2-eba3-4384-b8ba-4faf83b62979",
    "tags": []
   },
   "source": [
    "## Build Syntax in Extension Model node\n",
    "\n",
    "Copy and the below code to the Build Syntax section of Extension Model node "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cd49f959-540c-4fd3-866d-bd775371a911"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import subprocess\n",
    "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"pandas\", \"ibm-watson-machine-learning\", \"--no-input\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2dc01585-5e2a-4905-9a41-dde90d8dd706"
   },
   "source": [
    "## Score Syntax in Extension Model node\n",
    "\n",
    "Copy and the below code to the Build Syntax section of Extension Model node "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "45d05adf-f4a4-4400-bfec-07083dd31a90"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import subprocess\n",
    "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"pandas\", \"ibm-watson-machine-learning\", \"--no-input\"])\n",
    "\n",
    "import spss.pyspark.runtime\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.sql import SQLContext\n",
    "import numpy as np\n",
    "import ibm_watson_machine_learning\n",
    "\n",
    "SPACE_ID = \"5d163188-8cf0-405c-94e6-077fae0bd306\"\n",
    "DEPLOYMENT_ID = \"17ad92c5-cd7a-4914-ad49-14adc5c919ca\"\n",
    "\n",
    "wml_credentials = {\n",
    "    \"username\": \"admin\",\n",
    "    \"password\": \"QJJj0GSt2dxZ\",\n",
    "    \"url\": \"https://cpd-cpd.itzroks-550003aw18-kv78z8-6ccd7f378ae819553d37d5f2ee142bd6-0000.au-syd.containers.appdomain.cloud\",\n",
    "    \"instance_id\": \"openshift\",\n",
    "    \"version\": \"4.5\",\n",
    "}\n",
    "\n",
    "wml_client = ibm_watson_machine_learning.APIClient(wml_credentials)\n",
    "wml_client.set.default_space(SPACE_ID)\n",
    "\n",
    "cxt = spss.pyspark.runtime.getContext()\n",
    "sqlContext = cxt.getSparkSQLContext()\n",
    "\n",
    "target = \"MortgageDefault\"\n",
    "prediction = f\"$PRED-{target}\"\n",
    "probability = f\"$PROB-{target}\"\n",
    "\n",
    "fieldList = [StructField(x.name, x.dataType, x.nullable) for x in cxt.getSparkInputSchema()]\n",
    "fieldList.append(StructField(prediction, StringType(), nullable=False))\n",
    "fieldList.append(StructField(probability, FloatType(), nullable=False))\n",
    "outputSchema = StructType(fieldList)\n",
    "cxt.setSparkOutputSchema(outputSchema)\n",
    "\n",
    "if not cxt.isComputeDataModelOnly():\n",
    "    df = cxt.getSparkInputData().toPandas()\n",
    "    payload = {\"input_data\": [{\"fields\": df.columns.tolist(), \"values\": df}]}\n",
    "    response = wml_client.deployments.score(DEPLOYMENT_ID, payload)\n",
    "    pred, prob = zip(*response[\"predictions\"][0][\"values\"])\n",
    "    df[prediction] = pred\n",
    "    df[prediction] = df[prediction].map({1: \"YES\", 0: \"NO\"})\n",
    "    df[probability] = np.array(prob)[:,-1]\n",
    "    cxt.setSparkOutputData(sqlContext.createDataFrame(df))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "e4cce46d6be9934fbd27f9ca0432556941ea5bdf741d4f4d64c6cd7f8dfa8fba"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}