LinearRegression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import SGDRegressor as sgd\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('../CNN_Keras/SimulationCNN/Kmers6_counts_600bp.csv')\n",
    "train_reads = pd.read_csv('../data/Mouse_DMRs_counts_total.csv',header = None)\n",
    "train_methys = pd.read_csv('../data/Mouse_DMRs_counts_methylated.csv',header = None)\n",
    "train_methy_level = pd.read_csv('../data/Mouse_DMRs_methylation_level.csv',header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(58959, 2080) (58959,) (58959,) (58959,)\n"
     ]
    }
   ],
   "source": [
    "cell_type = 5\n",
    "data = train_data.as_matrix()\n",
    "level = train_methy_level.as_matrix()[:,cell_type]\n",
    "reads = train_reads.as_matrix()[:,cell_type]\n",
    "methy = train_methys.as_matrix()[:,cell_type]\n",
    "print(data.shape,level.shape,reads.shape,methy.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = data[:48000]\n",
    "level_train = level[:48000]\n",
    "reads_train = reads[:48000]\n",
    "methys_train = methy[:48000]\n",
    "data_test = data[48000:]\n",
    "level_test = level[48000:]\n",
    "reads_test = reads[48000:]\n",
    "methy_test = methy[48000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {'loss':['squared_loss','huber'],'penalty':['l2','l1']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Info with no sample weight applied:\n",
      "Start Training: Loss:  squared_loss , penalty:  l2\n",
      "R^2 score: 0.22 (+/- 0.03)\n",
      "Start Training: Loss:  squared_loss , penalty:  l1\n",
      "R^2 score: 0.23 (+/- 0.02)\n",
      "Start Training: Loss:  huber , penalty:  l2\n",
      "R^2 score: 0.21 (+/- 0.02)\n",
      "Start Training: Loss:  huber , penalty:  l1\n",
      "R^2 score: 0.20 (+/- 0.01)\n"
     ]
    }
   ],
   "source": [
    "print('Info with no sample weight applied:')\n",
    "for l in param['loss']:\n",
    "    for p in param['penalty']:\n",
    "        print('Start Training: Loss: ',l,\", penalty: \",p)\n",
    "        clf = sgd(loss = l, n_iter=100,penalty = p)\n",
    "        scores = cross_val_score(clf, data, level, cv=5)\n",
    "        print(\"R^2 score: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Info with sample weight applied:\n",
      "Start Training: Loss:  squared_loss , penalty:  l2\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -203633442007800961325596672.00 (+/- 53612617249645648465100800.00)\n",
      "\t\tTest R^2 score: -195603056422496097663975424.00 (+/- 29784310162538468644749312.00)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.29 (+/- 0.01)\n",
      "\t\tTest R^2 score: 0.22 (+/- 0.03)\n",
      "Start Training: Loss:  squared_loss , penalty:  l1\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -212755296407341937763287040.00 (+/- 53556334791207475403030528.00)\n",
      "\t\tTest R^2 score: -209667190754935327766347776.00 (+/- 49419782694077458821414912.00)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.28 (+/- 0.01)\n",
      "\t\tTest R^2 score: 0.23 (+/- 0.02)\n",
      "Start Training: Loss:  huber , penalty:  l2\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -2.25 (+/- 0.45)\n",
      "\t\tTest R^2 score: -2.28 (+/- 0.22)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.27 (+/- 0.02)\n",
      "\t\tTest R^2 score: 0.20 (+/- 0.02)\n",
      "Start Training: Loss:  huber , penalty:  l1\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -0.24 (+/- 0.14)\n",
      "\t\tTest R^2 score: -0.31 (+/- 0.08)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.25 (+/- 0.02)\n",
      "\t\tTest R^2 score: 0.21 (+/- 0.03)\n"
     ]
    }
   ],
   "source": [
    "print('Info with sample weight applied:')\n",
    "for l in param['loss']:\n",
    "    for p in param['penalty']:\n",
    "        print('Start Training: Loss: ',l,\", penalty: \",p)\n",
    "        clf = sgd(loss = l, n_iter=200,penalty = p,warm_start = False)\n",
    "        kf = KFold(n_splits=5)\n",
    "        train_score = []\n",
    "        test_score = []\n",
    "        Uni_train_score = []\n",
    "        Uni_test_score = []\n",
    "        #Cross Validation\n",
    "        for train_index, test_index in kf.split(data):\n",
    "            X_train, X_test = data[train_index],data[test_index]\n",
    "            Y_train, Y_test = level[train_index],level[test_index]\n",
    "            R_train, R_test = reads[train_index],reads[test_index]\n",
    "            \n",
    "            clf.fit(X_train,Y_train,sample_weight = R_train)\n",
    "            train_score.append(clf.score(X_train,Y_train,R_train))\n",
    "            test_score.append(clf.score(X_test,Y_test,R_test))\n",
    "            \n",
    "            clf.fit(X_train,Y_train)\n",
    "            Uni_train_score.append(clf.score(X_train,Y_train))\n",
    "            Uni_test_score.append(clf.score(X_test,Y_test))\n",
    "        \n",
    "        train_score = np.array(train_score)\n",
    "        test_score = np.array(test_score)\n",
    "        \n",
    "        Uni_train_score = np.array(Uni_train_score)\n",
    "        Uni_test_score = np.array(Uni_test_score)\n",
    "        \n",
    "        print(\"\\t Train with different sample weigt\")\n",
    "        print(\"\\t\\tTrain R^2 score: %0.2f (+/- %0.2f)\" % (train_score.mean(), train_score.std() * 2))\n",
    "        print(\"\\t\\tTest R^2 score: %0.2f (+/- %0.2f)\" % (test_score.mean(), test_score.std() * 2))\n",
    "        \n",
    "        print(\"\\t Train with equal sample weigt\")\n",
    "        print(\"\\t\\tTrain R^2 score: %0.2f (+/- %0.2f)\" % (Uni_train_score.mean(), Uni_train_score.std() * 2))\n",
    "        print(\"\\t\\tTest R^2 score: %0.2f (+/- %0.2f)\" % (Uni_test_score.mean(), Uni_test_score.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epsilon:  0.5  alpha;  0.0001\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -1.68 (+/- 0.34)\n",
      "\t\tTest R^2 score: -1.76 (+/- 0.40)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.28 (+/- 0.01)\n",
      "\t\tTest R^2 score: 0.23 (+/- 0.02)\n",
      "epsilon:  0.5  alpha;  0.001\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -0.93 (+/- 0.65)\n",
      "\t\tTest R^2 score: -1.01 (+/- 0.71)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.23 (+/- 0.01)\n",
      "\t\tTest R^2 score: 0.21 (+/- 0.01)\n",
      "epsilon:  0.5  alpha;  0.01\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -0.31 (+/- 0.14)\n",
      "\t\tTest R^2 score: -0.36 (+/- 0.10)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.03 (+/- 0.00)\n",
      "\t\tTest R^2 score: 0.03 (+/- 0.00)\n",
      "epsilon:  0.1  alpha;  0.0001\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -0.24 (+/- 0.07)\n",
      "\t\tTest R^2 score: -0.31 (+/- 0.07)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.25 (+/- 0.01)\n",
      "\t\tTest R^2 score: 0.21 (+/- 0.02)\n",
      "epsilon:  0.1  alpha;  0.001\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: -0.35 (+/- 0.32)\n",
      "\t\tTest R^2 score: -0.41 (+/- 0.32)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.08 (+/- 0.00)\n",
      "\t\tTest R^2 score: 0.08 (+/- 0.01)\n",
      "epsilon:  0.1  alpha;  0.01\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: 0.05 (+/- 0.04)\n",
      "\t\tTest R^2 score: 0.02 (+/- 0.03)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: -0.09 (+/- 0.00)\n",
      "\t\tTest R^2 score: -0.10 (+/- 0.03)\n",
      "epsilon:  0.01  alpha;  0.0001\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: 0.20 (+/- 0.02)\n",
      "\t\tTest R^2 score: 0.14 (+/- 0.03)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: 0.07 (+/- 0.01)\n",
      "\t\tTest R^2 score: 0.06 (+/- 0.02)\n",
      "epsilon:  0.01  alpha;  0.001\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: 0.20 (+/- 0.03)\n",
      "\t\tTest R^2 score: 0.17 (+/- 0.04)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: -0.11 (+/- 0.00)\n",
      "\t\tTest R^2 score: -0.11 (+/- 0.03)\n",
      "epsilon:  0.01  alpha;  0.01\n",
      "\t Train with different sample weigt\n",
      "\t\tTrain R^2 score: 0.08 (+/- 0.03)\n",
      "\t\tTest R^2 score: 0.08 (+/- 0.02)\n",
      "\t Train with equal sample weigt\n",
      "\t\tTrain R^2 score: -0.11 (+/- 0.00)\n",
      "\t\tTest R^2 score: -0.11 (+/- 0.03)\n"
     ]
    }
   ],
   "source": [
    "for e in [0.5,0.1,0.01]:\n",
    "    for alpha in [0.0001, 0.001,0.01]:\n",
    "        print('epsilon: ', e, \" alpha; \", alpha)\n",
    "        clf = sgd(loss = 'huber', n_iter=200,penalty = 'l1',alpha = alpha, epsilon = e, warm_start = False)\n",
    "        kf = KFold(n_splits=5)\n",
    "        train_score = []\n",
    "        test_score = []\n",
    "        Uni_train_score = []\n",
    "        Uni_test_score = []\n",
    "        #Cross Validation\n",
    "        for train_index, test_index in kf.split(data):\n",
    "            X_train, X_test = data[train_index],data[test_index]\n",
    "            Y_train, Y_test = level[train_index],level[test_index]\n",
    "            R_train, R_test = reads[train_index],reads[test_index]\n",
    "            \n",
    "            clf.fit(X_train,Y_train,sample_weight = R_train)\n",
    "            train_score.append(clf.score(X_train,Y_train,sample_weight = R_train))\n",
    "            test_score.append(clf.score(X_test,Y_test,sample_weight = R_test))\n",
    "            \n",
    "            clf.fit(X_train,Y_train,sample_weight = np.ones(len(X_train)))\n",
    "            Uni_train_score.append(clf.score(X_train,Y_train))\n",
    "            Uni_test_score.append(clf.score(X_test,Y_test))\n",
    "        \n",
    "        train_score = np.array(train_score)\n",
    "        test_score = np.array(test_score)\n",
    "        \n",
    "        Uni_train_score = np.array(Uni_train_score)\n",
    "        Uni_test_score = np.array(Uni_test_score)\n",
    "        \n",
    "        print(\"\\t Train with different sample weigt\")\n",
    "        print(\"\\t\\tTrain R^2 score: %0.2f (+/- %0.2f)\" % (train_score.mean(), train_score.std() * 2))\n",
    "        print(\"\\t\\tTest R^2 score: %0.2f (+/- %0.2f)\" % (test_score.mean(), test_score.std() * 2))\n",
    "        \n",
    "        print(\"\\t Train with equal sample weigt\")\n",
    "        print(\"\\t\\tTrain R^2 score: %0.2f (+/- %0.2f)\" % (Uni_train_score.mean(), Uni_train_score.std() * 2))\n",
    "        print(\"\\t\\tTest R^2 score: %0.2f (+/- %0.2f)\" % (Uni_test_score.mean(), Uni_test_score.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = sgd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}