Reporting (#24) (#25)

* added model diagnostic charts
* added results conversion for BayesOptimization and Optuna libraries

docs/examples/examples_index.rst

@@ -2,6 +2,7 @@
    Interactive experiment run comparison <interactive_compare_experiments>
    Hyper parameter comparison <explore_hyperparams_skopt>
    Run skopt/hyperopt hyperparameter sweep <run_hyperparameter_search>
+   Log model diagnostics <log_model_diagnostics>
    Monitor lightGBM training <monitor_lgbm>
    Monitor fast.ai training <monitor_fastai>
    Log matplotlib charts to neptune <log_matplotlib>

docs/examples/explore_hyperparams_skopt.ipynb

@@ -104,7 +104,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You can use `df2result` helper function from `neptunecontrib.viz`."
+    "You can use `df2result` helper function from `neptunecontrib.hpo.utils`."
    ]
   },
   {
@@ -125,7 +125,7 @@
     }
    ],
    "source": [
-    "from neptunecontrib.viz.utils import df2result\n",
+    "from neptunecontrib.hpo.utils import df2result\n",
     "\n",
     "result = df2result(hyper_df, \n",
     "                   metric_col='ROC_AUC', \n",
@@ -209,7 +209,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "neptunecontrib_py36",
+   "display_name": "scraping",
    "language": "python",
    "name": "python3"
   },

docs/examples/log_matplotlib.ipynb

@@ -105,7 +105,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "scraping",
    "language": "python",
    "name": "python3"
   },

docs/examples/log_model_diagnostics.ipynb

@@ -0,0 +1,188 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Log model diagnostics to Neptune\n",
+    "## Train your model and run predictions\n",
+    "Let's train a model on a synthetic problem predict on test data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.datasets import make_classification\n",
+    "from sklearn.ensemble import RandomForestClassifier\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.metrics import classification_report\n",
+    "\n",
+    "X, y = make_classification(n_samples=2000)\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
+    "\n",
+    "model = RandomForestClassifier()\n",
+    "model.fit(X_train, y_train)\n",
+    "\n",
+    "y_test_pred = model.predict_proba(X_test)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Instantiate Neptune"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import neptune\n",
+    "\n",
+    "ctx = neptune.Context()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Send classification report to Neptune"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from neptunecontrib.monitoring.reporting import send_binary_classification_report\n",
+    "\n",
+    "send_binary_classification_report(ctx, y_test, y_test_pred, threshold=0.5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is now safely logged in Neptune\n",
+    "\n",
+    "![image1](https://gist.githubusercontent.com/jakubczakon/f754769a39ea6b8fa9728ede49b9165c/raw/a1386b3a5edddc0eecb478a81d497336156b5b19/clf_report1.png)\n",
+    "\n",
+    "## Send confusion matrix to Neptune"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from neptunecontrib.monitoring.reporting import send_confusion_matrix\n",
+    "\n",
+    "send_confusion_matrix(ctx, y_test, y_test_pred[:, 1] > 0.5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is now safely logged in Neptune\n",
+    "\n",
+    "![image2](https://gist.githubusercontent.com/jakubczakon/f754769a39ea6b8fa9728ede49b9165c/raw/a1386b3a5edddc0eecb478a81d497336156b5b19/clf_report4.png)\n",
+    "\n",
+    "## Send ROC AUC curve to Neptune"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from neptunecontrib.monitoring.reporting import send_roc_auc_curve\n",
+    "\n",
+    "send_roc_auc_curve(ctx, y_test, y_test_pred)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is now safely logged in Neptune\n",
+    "\n",
+    "![image3](https://gist.githubusercontent.com/jakubczakon/f754769a39ea6b8fa9728ede49b9165c/raw/a1386b3a5edddc0eecb478a81d497336156b5b19/clf_report3.png)\n",
+    "\n",
+    "## Send Precision-Recall curve to Neptune"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from neptunecontrib.monitoring.reporting import send_precision_recall\n",
+    "\n",
+    "send_prediction_distribution(y_test, y_test_pred)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is now safely logged in Neptune\n",
+    "\n",
+    "![image4](https://gist.githubusercontent.com/jakubczakon/f754769a39ea6b8fa9728ede49b9165c/raw/a1386b3a5edddc0eecb478a81d497336156b5b19/clf_report5.png)\n",
+    "\n",
+    "## Send Precision-Recall curve to Neptune"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from neptunecontrib.monitoring.reporting import send_prediction_distribution\n",
+    "\n",
+    "send_prediction_distribution(y_test, y_test_pred[:, 1])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is now safely logged in Neptune\n",
+    "\n",
+    "![image5](https://gist.githubusercontent.com/jakubczakon/f754769a39ea6b8fa9728ede49b9165c/raw/a1386b3a5edddc0eecb478a81d497336156b5b19/clf_report2.png)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "santander",
+   "language": "python",
+   "name": "santander"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/user_guide/monitoring/reporting.rst

@@ -0,0 +1,6 @@
+Reporting
+===========
+
+.. automodule:: neptunecontrib.monitoring.reporting
+    :members:
+    :show-inheritance: