lasso_study.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# study in lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn import linear_model\n",
    "from sklearn.model_selection import cross_val_score, RepeatedKFold\n",
    "from skopt import BayesSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston = load_boston()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = shuffle(boston.data, boston.target, random_state=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = linear_model.Lasso(alpha=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "/opt/conda/lib/python3.6/site-packages/skopt/optimizer/optimizer.py:399: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7406426206930037 {'alpha': 0.0001}\n",
      "CPU times: user 1min 29s, sys: 2.38 s, total: 1min 31s\n",
      "Wall time: 32.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if True:\n",
    "    # Set to True if we want to run the BayesSearchCV process\n",
    "    params = {'alpha': (0.0001,10)}\n",
    "\n",
    "    optimiser = BayesSearchCV(reg, params, n_jobs=-1, n_iter=50, cv=RepeatedKFold(n_repeats=5))\n",
    "    #opt = BayesSearchCV( SVC(), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1,8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_iter=32 )\n",
    "    #executes bayesian optimization\n",
    "\n",
    "    # note we're fitting on all the data, so we're optimising wrt to the test data (might be silly)\n",
    "    optimiser.fit(X, y)\n",
    "\n",
    "    print(optimiser.score(X, y), optimiser.best_params_)\n",
    "    #pd.DataFrame(optimiser.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=0.1, copy_X=True, fit_intercept=True, max_iter=1000,\n",
       "   normalize=False, positive=False, precompute=False, random_state=None,\n",
       "   selection='cyclic', tol=0.0001, warm_start=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = cross_val_score(reg, \n",
    "                         X, y, \n",
    "                         cv=RepeatedKFold(n_repeats=50), \n",
    "                         scoring=None) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.72249392, 0.72058049, 0.67515424, 0.65345698, 0.77069053,\n",
       "       0.60776792, 0.69522685, 0.80035215, 0.71231319, 0.58369343,\n",
       "       0.6332168 , 0.70986818, 0.7160302 , 0.6842893 , 0.76108948,\n",
       "       0.71947939, 0.58005628, 0.75636689, 0.72864248, 0.70446775,\n",
       "       0.67545046, 0.74407106, 0.76356222, 0.75279732, 0.64152284,\n",
       "       0.79945549, 0.76970771, 0.66028435, 0.75548356, 0.51020518,\n",
       "       0.63762234, 0.7330666 , 0.69732273, 0.70857861, 0.76588775,\n",
       "       0.74330731, 0.55030178, 0.7756624 , 0.69056645, 0.71300812,\n",
       "       0.78163252, 0.77665242, 0.73147031, 0.47522991, 0.67021917,\n",
       "       0.75763865, 0.68146271, 0.61173694, 0.73487249, 0.66867348,\n",
       "       0.75106362, 0.74705305, 0.73108196, 0.66089508, 0.63060888,\n",
       "       0.65811372, 0.71347412, 0.72078468, 0.65993902, 0.79250413,\n",
       "       0.70982495, 0.70884386, 0.7201533 , 0.70600777, 0.63012302,\n",
       "       0.63713318, 0.67884599, 0.8067276 , 0.71041515, 0.72335074,\n",
       "       0.53529192, 0.6029808 , 0.7657159 , 0.78405395, 0.75358565,\n",
       "       0.72569318, 0.66725334, 0.42357432, 0.75864009, 0.7448289 ,\n",
       "       0.68560291, 0.65995981, 0.80894611, 0.71748509, 0.65667824,\n",
       "       0.6377591 , 0.65204262, 0.78865494, 0.71744042, 0.70747166,\n",
       "       0.56130851, 0.6742364 , 0.69163753, 0.82875038, 0.75618092,\n",
       "       0.7154566 , 0.70975643, 0.68920083, 0.67091151, 0.76730271,\n",
       "       0.761814  , 0.71754019, 0.58657202, 0.67326989, 0.73628985,\n",
       "       0.66416415, 0.74766036, 0.57223497, 0.74874472, 0.76225274,\n",
       "       0.60514467, 0.73899114, 0.73533082, 0.75003836, 0.69381602,\n",
       "       0.70538687, 0.74170723, 0.72375932, 0.74800372, 0.62581434,\n",
       "       0.55357441, 0.70438556, 0.78530124, 0.70984342, 0.76891364,\n",
       "       0.72447234, 0.68425969, 0.61677298, 0.78209409, 0.67022421,\n",
       "       0.72026337, 0.68535542, 0.78454311, 0.70083116, 0.6432918 ,\n",
       "       0.66664565, 0.73544631, 0.71339979, 0.74278318, 0.67047779,\n",
       "       0.71340743, 0.61877466, 0.73399568, 0.75004615, 0.69089788,\n",
       "       0.64457247, 0.69780518, 0.7037217 , 0.76635679, 0.72281928,\n",
       "       0.68142994, 0.67566552, 0.68592078, 0.7003511 , 0.73536231,\n",
       "       0.72744668, 0.77100374, 0.64324237, 0.72824198, 0.5732417 ,\n",
       "       0.65625388, 0.66658401, 0.70858808, 0.72230793, 0.7683654 ,\n",
       "       0.6664169 , 0.68651837, 0.76373043, 0.62045004, 0.79800069,\n",
       "       0.76800824, 0.72865244, 0.69367768, 0.71868302, 0.65207811,\n",
       "       0.6869382 , 0.71936181, 0.67086177, 0.72430851, 0.74127763,\n",
       "       0.74797499, 0.70179791, 0.73033042, 0.6488501 , 0.68394036,\n",
       "       0.67178604, 0.70692955, 0.7550735 , 0.61199037, 0.74482135,\n",
       "       0.7202588 , 0.63336399, 0.62941108, 0.75182941, 0.77459981,\n",
       "       0.64563972, 0.76300708, 0.79086515, 0.6081127 , 0.73865325,\n",
       "       0.70440734, 0.70202369, 0.68419365, 0.68850768, 0.75007743,\n",
       "       0.69686   , 0.63626594, 0.7430641 , 0.77847246, 0.70822704,\n",
       "       0.76602808, 0.61344699, 0.67531829, 0.67173495, 0.75948344,\n",
       "       0.57009555, 0.70185975, 0.72347823, 0.76244197, 0.70369769,\n",
       "       0.74812497, 0.70672937, 0.66506508, 0.70758841, 0.69146743,\n",
       "       0.67795536, 0.6955014 , 0.78460406, 0.68749261, 0.65952007,\n",
       "       0.72546342, 0.73990751, 0.6827391 , 0.73087176, 0.66613286,\n",
       "       0.48247908, 0.74933147, 0.74189927, 0.79365677, 0.69979405,\n",
       "       0.71341394, 0.77423269, 0.730862  , 0.63589711, 0.67528537,\n",
       "       0.69054313, 0.77259822, 0.78556033, 0.59501157, 0.66772609])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def make_regression_interaction(n_noise=10, n_informative=10, n_samples=500, interactions=[(10,11), (12,13)]):\n",
    "    size = (500, n_noise + n_informative)\n",
    "    draw = np.random.normal(size=size)\n",
    "    int_mat = np.zeros((n_samples, len(interactions)))\n",
    "    for i, locs in enumerate(interactions):\n",
    "        int_mat[:,i] = draw[:,locs[0]] * draw[:, locs[1]]\n",
    "\n",
    "    X = np.hstack([draw, int_mat])\n",
    "    coefs = np.random.normal(size=n_noise + n_informative + len(interactions))\n",
    "    coefs[:n_noise] = 0\n",
    "    y = np.dot(X, coefs)\n",
    "    return X, y, coefs\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y, true_coefs = make_regression_interaction()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 22)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500,)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.32705405,  0.60971445,  0.7998964 , -0.91425842,  0.01634442,\n",
       "        1.05417015,  0.66081719, -0.28788392,  2.02153814, -0.46540406,\n",
       "       -0.80757082, -1.67809217])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true_coefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = linear_model.Lasso(alpha=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=0.01, copy_X=True, fit_intercept=True, max_iter=1000,\n",
       "   normalize=False, positive=False, precompute=False, random_state=None,\n",
       "   selection='cyclic', tol=0.0001, warm_start=False)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.        ,  0.        ,  0.        , -0.        ,  0.        ,\n",
       "        0.        , -0.        , -0.        , -0.        ,  0.        ,\n",
       "        0.31983329,  0.59830954,  0.78912013, -0.90696491,  0.00535545,\n",
       "        1.04176674,  0.65444073, -0.27681124,  2.00949251, -0.45510983,\n",
       "       -0.79757362, -1.66638592])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}