src/MetricStability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "matplotlib.use('Qt5Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "from src.utils import *\n",
    "from src.InstrumentalVariable import InstrumentalVariable\n",
    "from tqdm.notebook import tnrange"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def run_manifold_tests(X, y, min_p_value=80, max_p_value=95, bootstrap=True, min_l2_reg=0,\n",
    "                       max_l2_reg=50, n_tests=100):\n",
    "    n_samples, n_features, _ = X.shape\n",
    "    experiment_coefs = np.zeros((n_tests, n_features))\n",
    "    for i in tnrange(n_tests):\n",
    "        p_value = np.random.uniform(min_p_value, max_p_value)\n",
    "        if max_l2_reg > 0:\n",
    "            l2_reg = np.random.uniform(min_l2_reg, max_l2_reg)\n",
    "        else:\n",
    "            l2_reg = None\n",
    "        iv_model = InstrumentalVariable(p_value, l2_reg)\n",
    "        feature_size = np.random.randint(8, 20)\n",
    "        feature_inds = np.random.choice(n_features, feature_size, replace=False)\n",
    "\n",
    "        if bootstrap:\n",
    "            bootstrap_inds = np.random.choice(len(X), len(X))\n",
    "            X_train, y_train = X[bootstrap_inds], y[bootstrap_inds]\n",
    "        else:\n",
    "            X_train = X\n",
    "            y_train = y\n",
    "\n",
    "        X_train = X_train[:, feature_inds]\n",
    "\n",
    "        iv_model.fit(X_train, y_train)\n",
    "        np.put(experiment_coefs[i], feature_inds, iv_model.coef_)\n",
    "    return experiment_coefs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def filter_metrics(coefs):\n",
    "    positive_coefs = np.apply_along_axis(lambda feature: len(np.where(feature > 0)[0]), 0, coefs)\n",
    "    negative_coefs = np.apply_along_axis(lambda feature: len(np.where(feature < 0)[0]), 0, coefs)\n",
    "    print(positive_coefs)\n",
    "    print(negative_coefs)\n",
    "    filtered_coef_inds = []\n",
    "    for i, feature_coefs in enumerate(coefs.T):\n",
    "        pos = positive_coefs[i]\n",
    "        neg = negative_coefs[i]\n",
    "        if pos + neg == 0:\n",
    "            continue\n",
    "        if pos == 0 or neg == 0 or min(pos/neg, neg/pos) < 0.2:\n",
    "            filtered_coef_inds.append(i)\n",
    "    return np.array(filtered_coef_inds)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def plot_coefficients(coefs, metric_map=None):\n",
    "\n",
    "    n_tests, n_features = coefs.shape\n",
    "\n",
    "    fig, axes = plt.subplots(nrows=n_features, sharex=True)\n",
    "    fig.suptitle(\"3_metric_stability\")\n",
    "    collections = []\n",
    "    for i, metric_coefs in enumerate(coefs.T):\n",
    "        ax = axes[i]\n",
    "        ax.set_title('Weights for short_term_' + str(metric_map[i]), loc='left')\n",
    "        ax.plot([0, 0], [-1, 1], 'r')\n",
    "        metric_coefs = metric_coefs[metric_coefs != 0]\n",
    "        n_tests = len(metric_coefs)\n",
    "        col = ax.scatter(metric_coefs, np.random.rand(n_tests) * 2 - 1,\n",
    "                         cmap=plt.get_cmap(\"RdBu\"), picker=5, s=50)\n",
    "        collections.append(col)\n",
    "    plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "short_metrics_p, long_metrics_p = read_data(dataset_name='feed_top_ab_tests_pool_big_dataset.csv' ,shift=True)\n",
    "short_metrics = short_metrics_p[:, :, 0]\n",
    "long_metrics = long_metrics_p[:, :, 0]\n",
    "\n",
    "target_metric_p = long_metrics_p[:, 3, :]   # <--- here you can choose target (0, 1, 2, 3)\n",
    "target_metric = target_metric_p[:, 0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#main part of the sandbox, as it allows to change the constraints\n",
    "\n",
    "coefs = run_manifold_tests(short_metrics_p, target_metric,\n",
    "                            min_l2_reg=0, max_l2_reg=0.001,\n",
    "                            min_p_value=50, max_p_value=95, n_tests=1000)\n",
    "\n",
    "clear_metrics = filter_metrics(coefs)\n",
    "filtered_coefs = coefs[:, clear_metrics]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "plot_coefficients(coefs, range(np.shape(coefs)[1]))\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-ec5ed5f6",
   "language": "python",
   "display_name": "PyCharm (VK_Graduation)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}