benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "radical-wisconsin",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import timeit\n",
    "\n",
    "#models\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "\n",
    "#datasets\n",
    "from sklearn.datasets import load_boston, load_digits, load_wine, load_breast_cancer, load_iris\n",
    "\n",
    "#pmml libs\n",
    "from sklearn2pmml import PMMLPipeline, sklearn2pmml\n",
    "from pypmml import Model\n",
    "from sklearn_pmml_model.linear_model import PMMLLogisticRegression, PMMLRidgeClassifier\n",
    "from sklearn_pmml_model.naive_bayes import PMMLGaussianNB\n",
    "from sklearn_pmml_model.tree import PMMLTreeClassifier\n",
    "from sklearn_pmml_model.ensemble import PMMLForestClassifier, PMMLGradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "earlier-macintosh",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    'Linear model': LogisticRegression(), \n",
    "    'Naive Bayes': GaussianNB(),\n",
    "    'Decision tree': DecisionTreeClassifier(), \n",
    "    'Random Forest': RandomForestClassifier(), \n",
    "    'Gradient boosting': GradientBoostingClassifier()\n",
    "}\n",
    "pmmlModels = {\n",
    "    'Linear model': PMMLLogisticRegression, \n",
    "    'Naive Bayes': PMMLGaussianNB,\n",
    "    'Decision tree': PMMLTreeClassifier, \n",
    "    'Random Forest': PMMLForestClassifier, \n",
    "    'Gradient boosting': PMMLGradientBoostingClassifier\n",
    "}\n",
    "datasets = {\n",
    "#    'Iris': load_iris(as_frame=True), \n",
    "#    'Digits': load_digits(as_frame=True),\n",
    "    'Wine': load_wine(as_frame=True), \n",
    "    'Breast cancer': load_breast_cancer(as_frame=True)\n",
    "}\n",
    "algorithms = ['PyPMML', 'sklearn-pmml-model']\n",
    "\n",
    "load_times = pd.DataFrame(index=algorithms, columns=pd.MultiIndex.from_product([datasets, models]))\n",
    "pred_times = pd.DataFrame(index=algorithms, columns=pd.MultiIndex.from_product([datasets, models]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "every-anthropology",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wine - Linear model\n",
      "Wine - Naive Bayes\n",
      "Wine - Decision tree\n",
      "Wine - Random Forest\n",
      "Wine - Gradient boosting\n",
      "Breast cancer - Linear model\n",
      "Breast cancer - Naive Bayes\n",
      "Breast cancer - Decision tree\n",
      "Breast cancer - Random Forest\n",
      "Breast cancer - Gradient boosting\n"
     ]
    }
   ],
   "source": [
    "repeat = 100  # Note: this will take a long time\n",
    "\n",
    "for dataset_name, dataset in datasets.items():\n",
    "    X = StandardScaler().fit_transform(dataset.data)\n",
    "    X = pd.DataFrame(data=X, columns=dataset.data.columns)\n",
    "    y = dataset.target\n",
    "    \n",
    "    for model_name, model in models.items():\n",
    "        print(f\"{dataset_name} - {model_name}\")\n",
    "        \n",
    "        if isinstance(model, LogisticRegression):\n",
    "            model.multi_class = 'ovr' if len(np.unique(y)) == 2 else 'multinomial'\n",
    "\n",
    "        # Create PMML\n",
    "        pipeline = PMMLPipeline([\n",
    "          (\"classifier\", model)\n",
    "        ])\n",
    "        pipeline.fit(X, y)\n",
    "        sklearn2pmml(pipeline, \"test.pmml\", with_repr=True)\n",
    "        \n",
    "        # Record load times\n",
    "        pypmml_time_load = np.mean(timeit.repeat(lambda: Model.load(\"test.pmml\"), number=1, repeat=repeat))\n",
    "        spm_time_load = np.mean(timeit.repeat(lambda: pmmlModels[model_name](pmml=\"test.pmml\"), number=1, repeat=repeat))\n",
    "        load_times[(dataset_name, model_name)][algorithms[0]] = pypmml_time_load\n",
    "        load_times[(dataset_name, model_name)][algorithms[1]] = spm_time_load\n",
    "        \n",
    "        pypmml = Model.load(\"test.pmml\")\n",
    "        spm = pmmlModels[model_name](pmml=\"test.pmml\")\n",
    "        \n",
    "        # Record predict times\n",
    "        pypmml_time_predict = np.mean(timeit.repeat(lambda: pypmml.predict(X), number=1, repeat=repeat))\n",
    "        spm_time_predict = np.mean(timeit.repeat(lambda: spm.predict_proba(X), number=1, repeat=repeat))\n",
    "        pred_times[(dataset_name, model_name)][algorithms[0]] = pypmml_time_predict\n",
    "        pred_times[(dataset_name, model_name)][algorithms[1]] = spm_time_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fleet-allocation",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "declared-midwest",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">Wine</th>\n",
       "      <th colspan=\"5\" halign=\"left\">Breast cancer</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Linear model</th>\n",
       "      <th>Naive Bayes</th>\n",
       "      <th>Decision tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Gradient boosting</th>\n",
       "      <th>Linear model</th>\n",
       "      <th>Naive Bayes</th>\n",
       "      <th>Decision tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Gradient boosting</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PyPMML</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.002453</td>\n",
       "      <td>0.002322</td>\n",
       "      <td>0.042778</td>\n",
       "      <td>0.046875</td>\n",
       "      <td>0.00208</td>\n",
       "      <td>0.003304</td>\n",
       "      <td>0.001954</td>\n",
       "      <td>0.067751</td>\n",
       "      <td>0.018387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sklearn-pmml-model</th>\n",
       "      <td>0.001441</td>\n",
       "      <td>0.00161</td>\n",
       "      <td>0.001309</td>\n",
       "      <td>0.097291</td>\n",
       "      <td>0.120141</td>\n",
       "      <td>0.002286</td>\n",
       "      <td>0.003365</td>\n",
       "      <td>0.001482</td>\n",
       "      <td>0.133827</td>\n",
       "      <td>0.042554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Wine                                          \\\n",
       "                   Linear model Naive Bayes Decision tree Random Forest   \n",
       "PyPMML                    0.002    0.002453      0.002322      0.042778   \n",
       "sklearn-pmml-model     0.001441     0.00161      0.001309      0.097291   \n",
       "\n",
       "                                     Breast cancer                            \\\n",
       "                   Gradient boosting  Linear model Naive Bayes Decision tree   \n",
       "PyPMML                      0.046875       0.00208    0.003304      0.001954   \n",
       "sklearn-pmml-model          0.120141      0.002286    0.003365      0.001482   \n",
       "\n",
       "                                                    \n",
       "                   Random Forest Gradient boosting  \n",
       "PyPMML                  0.067751          0.018387  \n",
       "sklearn-pmml-model      0.133827          0.042554  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "practical-amount",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">Wine</th>\n",
       "      <th colspan=\"5\" halign=\"left\">Breast cancer</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Linear model</th>\n",
       "      <th>Naive Bayes</th>\n",
       "      <th>Decision tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Gradient boosting</th>\n",
       "      <th>Linear model</th>\n",
       "      <th>Naive Bayes</th>\n",
       "      <th>Decision tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Gradient boosting</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PyPMML</th>\n",
       "      <td>0.771291</td>\n",
       "      <td>0.771387</td>\n",
       "      <td>0.775104</td>\n",
       "      <td>0.852426</td>\n",
       "      <td>0.85548</td>\n",
       "      <td>3.847776</td>\n",
       "      <td>3.875145</td>\n",
       "      <td>3.834276</td>\n",
       "      <td>4.09583</td>\n",
       "      <td>4.119273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sklearn-pmml-model</th>\n",
       "      <td>0.004373</td>\n",
       "      <td>0.004747</td>\n",
       "      <td>0.001384</td>\n",
       "      <td>0.011591</td>\n",
       "      <td>0.001682</td>\n",
       "      <td>0.013437</td>\n",
       "      <td>0.007913</td>\n",
       "      <td>0.001325</td>\n",
       "      <td>0.012407</td>\n",
       "      <td>0.001462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Wine                                          \\\n",
       "                   Linear model Naive Bayes Decision tree Random Forest   \n",
       "PyPMML                 0.771291    0.771387      0.775104      0.852426   \n",
       "sklearn-pmml-model     0.004373    0.004747      0.001384      0.011591   \n",
       "\n",
       "                                     Breast cancer                            \\\n",
       "                   Gradient boosting  Linear model Naive Bayes Decision tree   \n",
       "PyPMML                       0.85548      3.847776    3.875145      3.834276   \n",
       "sklearn-pmml-model          0.001682      0.013437    0.007913      0.001325   \n",
       "\n",
       "                                                    \n",
       "                   Random Forest Gradient boosting  \n",
       "PyPMML                   4.09583          4.119273  \n",
       "sklearn-pmml-model      0.012407          0.001462  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "signed-family",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">Wine</th>\n",
       "      <th colspan=\"5\" halign=\"left\">Breast cancer</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Linear model</th>\n",
       "      <th>Naive Bayes</th>\n",
       "      <th>Decision tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Gradient boosting</th>\n",
       "      <th>Linear model</th>\n",
       "      <th>Naive Bayes</th>\n",
       "      <th>Decision tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>Gradient boosting</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PyPMML</th>\n",
       "      <td>0.773291</td>\n",
       "      <td>0.77384</td>\n",
       "      <td>0.777425</td>\n",
       "      <td>0.895204</td>\n",
       "      <td>0.902355</td>\n",
       "      <td>3.849855</td>\n",
       "      <td>3.878448</td>\n",
       "      <td>3.83623</td>\n",
       "      <td>4.16358</td>\n",
       "      <td>4.13766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sklearn-pmml-model</th>\n",
       "      <td>0.005813</td>\n",
       "      <td>0.006357</td>\n",
       "      <td>0.002693</td>\n",
       "      <td>0.108882</td>\n",
       "      <td>0.121823</td>\n",
       "      <td>0.015723</td>\n",
       "      <td>0.011278</td>\n",
       "      <td>0.002807</td>\n",
       "      <td>0.146234</td>\n",
       "      <td>0.044016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Wine                                          \\\n",
       "                   Linear model Naive Bayes Decision tree Random Forest   \n",
       "PyPMML                 0.773291     0.77384      0.777425      0.895204   \n",
       "sklearn-pmml-model     0.005813    0.006357      0.002693      0.108882   \n",
       "\n",
       "                                     Breast cancer                            \\\n",
       "                   Gradient boosting  Linear model Naive Bayes Decision tree   \n",
       "PyPMML                      0.902355      3.849855    3.878448       3.83623   \n",
       "sklearn-pmml-model          0.121823      0.015723    0.011278      0.002807   \n",
       "\n",
       "                                                    \n",
       "                   Random Forest Gradient boosting  \n",
       "PyPMML                   4.16358           4.13766  \n",
       "sklearn-pmml-model      0.146234          0.044016  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_times + pred_times"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}