yedidi_vijay_exam_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#files = \"https://github.com/rpi-techfundamentals/introml_website_fall_2020/raw/master/files/midterm.zip\" \n",
    "#!pip3 install otter-grader && wget $files  && unzip -o midterm.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import sklearn\n",
    "import otter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vZfe_BrN2782"
   },
   "source": [
    "## Midterm\n",
    "\n",
    "**Save your working file in Google drive immediately upon starting this assignment.** \n",
    "\n",
    "Before you turn this in, make sure everything runs as expected. First, restart the kernel (in the menu, select Kernel → Restart) and then run all cells (in the menubar, select Cell → Run All). They should run completely without intervention.\n",
    "\n",
    "**You may find it useful to go through the notebooks from the course materials when doing these exercises.**\n",
    "\n",
    "**If you receive assistance from anyone it will be considered an ethical violation and referred to associate dean.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "id": "DlqW9xLjyRhV"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "      demo  demo1  demo2  demo3  demo4  demo5  demo6  demo7  demo8  demo9  \\\n",
       "0        0      1      1      1      1      1      0      1      1      0   \n",
       "1        0      1      1      0      1      1      0      1      1      0   \n",
       "2        0      0      1      0      1      1      0      0      1      1   \n",
       "3        0      1      1      0      1      1      0      1      1      1   \n",
       "4        1      0      1      1      0      0      0      1      1      1   \n",
       "...    ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "9995     0      0      0      0      0      0      0      0      1      1   \n",
       "9996     1      0      0      1      0      0      0      1      1      0   \n",
       "9997     1      1      0      1      0      0      0      0      1      1   \n",
       "9998     0      0      0      1      0      0      0      0      1      1   \n",
       "9999     1      0      0      1      0      0      0      1      1      1   \n",
       "\n",
       "      ...  dv2  dv3  dv4  dv5  dv6  dv7  dv8  dv9  class  reg_target  \n",
       "0     ...    1    1    1    0    1    0    0    1    0.0  453.521369  \n",
       "1     ...    1    1    1    1    1    0    1    0    0.0  151.919191  \n",
       "2     ...    1    0    1    0    1    0    1    1    0.0 -141.666991  \n",
       "3     ...    0    0    0    1    1    1    0    0    0.0 -227.245474  \n",
       "4     ...    0    1    0    0    0    1    0    0    0.0 -130.752790  \n",
       "...   ...  ...  ...  ...  ...  ...  ...  ...  ...    ...         ...  \n",
       "9995  ...    0    1    0    1    1    0    1    1    9.0   57.045191  \n",
       "9996  ...    1    0    0    0    1    0    1    0    9.0   20.359107  \n",
       "9997  ...    0    0    0    0    1    0    1    1    9.0 -171.269242  \n",
       "9998  ...    0    1    1    1    1    0    0    1    9.0  125.539606  \n",
       "9999  ...    0    1    0    0    0    1    1    0    9.0 -206.742398  \n",
       "\n",
       "[10000 rows x 22 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>demo</th>\n      <th>demo1</th>\n      <th>demo2</th>\n      <th>demo3</th>\n      <th>demo4</th>\n      <th>demo5</th>\n      <th>demo6</th>\n      <th>demo7</th>\n      <th>demo8</th>\n      <th>demo9</th>\n      <th>...</th>\n      <th>dv2</th>\n      <th>dv3</th>\n      <th>dv4</th>\n      <th>dv5</th>\n      <th>dv6</th>\n      <th>dv7</th>\n      <th>dv8</th>\n      <th>dv9</th>\n      <th>class</th>\n      <th>reg_target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>453.521369</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>151.919191</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>-141.666991</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>-227.245474</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>-130.752790</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>9995</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>9.0</td>\n      <td>57.045191</td>\n    </tr>\n    <tr>\n      <th>9996</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>9.0</td>\n      <td>20.359107</td>\n    </tr>\n    <tr>\n      <th>9997</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>9.0</td>\n      <td>-171.269242</td>\n    </tr>\n    <tr>\n      <th>9998</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>9.0</td>\n      <td>125.539606</td>\n    </tr>\n    <tr>\n      <th>9999</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>9.0</td>\n      <td>-206.742398</td>\n    </tr>\n  </tbody>\n</table>\n<p>10000 rows × 22 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 122
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df  = pd.read_csv(\"midterm_part1.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "THYiqAhmbSIU"
   },
   "source": [
    "## Statistical Functions and Calculations - Part 1\n",
    "\n",
    "1a. (3pts) Set the `reg_target_mean` variable value equal to the mean `reg_target` column of the df dataframe. \n",
    "\n",
    "1b. (3pts) Set the `class_num` variable value equal to the number of different values in the `class` column of the df dataframe. (Nan should count as a different value.)\n",
    "\n",
    "1c. (3pts) Set the `class_most_freq` variable value equal to the most frequent class in the `class` column of the df dataframe. Make it an integer.\n",
    "\n",
    "1d. (3pts) Set the `class_na` variable value equal to the number of missing values in the `class` column of the df dataframe.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "id": "IWIMTqVHbSIU"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "-0.7155711461403967\n"
     ]
    }
   ],
   "source": [
    "#Answer (add additional cells as necessary)\n",
    "\n",
    "# 1a: \n",
    "\n",
    "\n",
    "reg_target_mean = df.reg_target.mean() \n",
    "print(reg_target_mean) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "id": "YtKQDEf9bSIY"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "11\n"
     ]
    }
   ],
   "source": [
    "##1b: \n",
    "\n",
    "class_num = df['class'].value_counts(dropna = False).shape[0]\n",
    "print(class_num) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "id": "i_O4i484bSIb"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "## 1c\n",
    " \n",
    "class_most_freq = df['class'].value_counts().idxmax()\n",
    "\n",
    "print(class_most_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "id": "6ayKqcvebSId"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "34\n"
     ]
    }
   ],
   "source": [
    "## 1d. \n",
    "\n",
    "class_na = df['class'].isna().sum() \n",
    "print(class_na) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LGSTaD_fbSIg"
   },
   "source": [
    "## Fill NA Strategies - Part 1 Continued \n",
    "\n",
    "1e-f. (6pts) Create a dataframe `df2` from a copy of `df` which includes only those rows in which the `class` variable is not null.\n",
    "\n",
    "\n",
    "1f-g. (6pts) Create a dataframe `df3` from a copy of `df` which sets all missing values of `class` to `-1` (integer).  \n",
    "\n",
    "After filling the NA values, convert the `class` column to an integer type. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "id": "_3Ac8rT6bSIh"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "1    1012\n",
       "9    1000\n",
       "8    1000\n",
       "7     998\n",
       "0     998\n",
       "2     997\n",
       "6     992\n",
       "4     991\n",
       "3     990\n",
       "5     988\n",
       "Name: class, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 127
    }
   ],
   "source": [
    "#Answer (add additional cells as necessary)\n",
    "#1e - f\n",
    "dfcopy = df\n",
    "df2 = dfcopy \n",
    "df2 = df2.dropna(subset = ['class']).astype(int) \n",
    "\n",
    "df2['class'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "id": "-S0UgkthbSIk"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       " 1    1012\n",
       " 9    1000\n",
       " 8    1000\n",
       " 7     998\n",
       " 0     998\n",
       " 2     997\n",
       " 6     992\n",
       " 4     991\n",
       " 3     990\n",
       " 5     988\n",
       "-1      34\n",
       "Name: class, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 128
    }
   ],
   "source": [
    "\n",
    "# 1 f-g\n",
    "df3 = dfcopy \n",
    "df3['class'] = df3['class'].fillna(value = -1).astype(int) \n",
    "df3['class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1Jjas4BsbSIr"
   },
   "source": [
    "## Train Test Split - Part 2 \n",
    "\n",
    "2a. (3pts) Create a Pandas Series `y_class` equal to an `int` of the `class` variable from the `dfp2` dataframe.\n",
    "\n",
    "2b. (3pts) Create a Pandas Dataframe `X` equal to all other variables (i.e., all variables other than `reg_target` and `class` from the `dfp2` dataframe.  \n",
    "\n",
    "2c-e. (9pts) Use `X` and `y_class` to perform a train test split creating `train_X`,  `test_X`, `train_y`, `test_y` using the following specifications:\n",
    "\n",
    "- Use 75% for the train set, 25% for the test set. \n",
    "- Use the value `99` for the Random State.\n",
    "- Use stratified sampling.\n",
    "\n",
    "2f. Recode `train_y` from a multiclass classification problem to a binary classification problem, such that `train_y_new` = 0 if the value of train_y is 0-4 and `train_y_new` =1 if train_y = 5-9.\n",
    "\n",
    "2f. Recode `test_y` from a multiclass classification problem to a binary classification problem, such that `test_y_new` = 0 if the value of test_y is 0-4 and `test_y_new` =1 if test_y = 5-9.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "id": "KSR2a8H7bSIr"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "      demo  demo1  demo2  demo3  demo4  demo5  demo6  demo7  demo8  demo9  \\\n",
       "0        0      1      1      1      1      1      0      1      1      0   \n",
       "1        0      1      1      0      1      1      0      1      1      0   \n",
       "2        0      0      1      0      1      1      0      0      1      1   \n",
       "3        0      1      1      0      1      1      0      1      1      1   \n",
       "4        1      0      1      1      0      0      0      1      1      1   \n",
       "...    ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "9961     0      0      0      0      0      0      0      0      1      1   \n",
       "9962     1      0      0      1      0      0      0      1      1      0   \n",
       "9963     1      1      0      1      0      0      0      0      1      1   \n",
       "9964     0      0      0      1      0      0      0      0      1      1   \n",
       "9965     1      0      0      1      0      0      0      1      1      1   \n",
       "\n",
       "      ...  dv2  dv3  dv4  dv5  dv6  dv7  dv8  dv9  class  reg_target  \n",
       "0     ...    1    1    1    0    1    0    0    1    0.0  453.521369  \n",
       "1     ...    1    1    1    1    1    0    1    0    0.0  151.919191  \n",
       "2     ...    1    0    1    0    1    0    1    1    0.0 -141.666991  \n",
       "3     ...    0    0    0    1    1    1    0    0    0.0 -227.245474  \n",
       "4     ...    0    1    0    0    0    1    0    0    0.0 -130.752790  \n",
       "...   ...  ...  ...  ...  ...  ...  ...  ...  ...    ...         ...  \n",
       "9961  ...    0    1    0    1    1    0    1    1    9.0   57.045191  \n",
       "9962  ...    1    0    0    0    1    0    1    0    9.0   20.359107  \n",
       "9963  ...    0    0    0    0    1    0    1    1    9.0 -171.269242  \n",
       "9964  ...    0    1    1    1    1    0    0    1    9.0  125.539606  \n",
       "9965  ...    0    1    0    0    0    1    1    0    9.0 -206.742398  \n",
       "\n",
       "[9966 rows x 22 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>demo</th>\n      <th>demo1</th>\n      <th>demo2</th>\n      <th>demo3</th>\n      <th>demo4</th>\n      <th>demo5</th>\n      <th>demo6</th>\n      <th>demo7</th>\n      <th>demo8</th>\n      <th>demo9</th>\n      <th>...</th>\n      <th>dv2</th>\n      <th>dv3</th>\n      <th>dv4</th>\n      <th>dv5</th>\n      <th>dv6</th>\n      <th>dv7</th>\n      <th>dv8</th>\n      <th>dv9</th>\n      <th>class</th>\n      <th>reg_target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>453.521369</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>151.919191</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>-141.666991</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>-227.245474</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>-130.752790</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>9961</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>9.0</td>\n      <td>57.045191</td>\n    </tr>\n    <tr>\n      <th>9962</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>9.0</td>\n      <td>20.359107</td>\n    </tr>\n    <tr>\n      <th>9963</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>9.0</td>\n      <td>-171.269242</td>\n    </tr>\n    <tr>\n      <th>9964</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>9.0</td>\n      <td>125.539606</td>\n    </tr>\n    <tr>\n      <th>9965</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>9.0</td>\n      <td>-206.742398</td>\n    </tr>\n  </tbody>\n</table>\n<p>9966 rows × 22 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 129
    }
   ],
   "source": [
    "#Load Part 2 Data\n",
    "dfp2  = pd.read_csv(\"midterm_part2.csv\")\n",
    "dfp2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "id": "_YFjf9lEbSIt"
   },
   "outputs": [],
   "source": [
    "#Answer (add additional cells as necessary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "id": "KirEXORbbSIx"
   },
   "outputs": [],
   "source": [
    "\n",
    "#Part A: y class equals to an int of the class variable of the dfp2 dataframe\n",
    "y_class = dfp2['class'].astype(int) \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "id": "MIt7TG0GbSI5"
   },
   "outputs": [],
   "source": [
    "# Part B: x equals all variables in dataframe except reg_target\n",
    "X = dfp2.drop( columns = ['class', 'reg_target']) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Part C-E \n",
    "# Create a test/train split, where train is 75% \n",
    "# and test is 25%. \n",
    "## 99 for random state \n",
    "## stratified sampling \n",
    "from sklearn.model_selection import train_test_split \n",
    "\n",
    "train_X, test_X, train_y, test_y = train_test_split(X, y_class, \n",
    "                                                    train_size = 0.75, \n",
    "                                                    test_size = 0.25, \n",
    "                                                    random_state = 99, \n",
    "                                                    stratify = y_class)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "5135    1\n",
       "6198    1\n",
       "5261    1\n",
       "1679    0\n",
       "5309    1\n",
       "       ..\n",
       "2331    0\n",
       "7112    1\n",
       "6673    1\n",
       "1265    0\n",
       "7364    1\n",
       "Name: class, Length: 7474, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 134
    }
   ],
   "source": [
    "## 2F \n",
    "## Recode train_y from a multiclass classification to a binary classificaition\n",
    "## such that train_y_new = 0 if test_y = 0-4, and test_y_new = 1 if test_y = 5-9 \n",
    "\n",
    "train_y_new = ((train_y <= 9 ) & ( train_y >= 5) ).astype(int) \n",
    "train_y_new  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "8794    1\n",
       "8588    1\n",
       "5513    1\n",
       "7672    1\n",
       "2664    0\n",
       "       ..\n",
       "8218    1\n",
       "5416    1\n",
       "8917    1\n",
       "2558    0\n",
       "9641    1\n",
       "Name: class, Length: 2492, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 135
    }
   ],
   "source": [
    "## 2f. Recode test_y from a multiclass classification problem to a binary classification problem, \n",
    "# such that test_y_new = 0 if the value of test_y is 0-4 and test_y_new =1 if test_y = 5-9\n",
    "\n",
    "test_y_new = ((test_y <= 9) & (test_y >= 5) ).astype(int) \n",
    "test_y_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1rX8YKAq0vfJ"
   },
   "source": [
    "## Classification Models - Part 3\n",
    "\n",
    "Start with the `dfp3_train` and `df_class_test` dataframes.  Don't use the output of the train test split from part2. \n",
    "\n",
    "Create the values for `dfp3_X_train`, `dfp3_y_train`, `dfp3_X_test`, `dfp3_y_test` from the given training and test set `dfp3_train` and `dfp3_test`. \n",
    "\n",
    "Use a [naive bayes](https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html#sklearn.naive_bayes.GaussianNB) classifier for both models below. \n",
    "\n",
    "In the first model (model1), use all of the variables that start with `demo` or `dv` predict the class variable. Create the following variables:\n",
    "\n",
    "3a. (6pts) `model1_train_accuracy`  Accuracy for model 1 in the train set.\n",
    "\n",
    "3b. (6pts) `model1_test_accuracy`   Accuracy for model 1 in the test set.\n",
    "\n",
    "3c. (6pts) `model1_test_ROC`        Area Under the Receiver Operating Characteristic Curve (ROC AUC) for model 1 \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "id": "ENLw2wcFbSI8"
   },
   "outputs": [],
   "source": [
    "#Load Part 3 Data\n",
    "dfp3_train  = pd.read_csv('midterm_train_part3.csv')\n",
    "dfp3_test   = pd.read_csv('midterm_test_part3.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   demo  demo1  demo2  demo3  demo4  demo5  demo6  demo7  demo8  demo9  ...  \\\n",
       "0     1      0      0      0      0      0      1      0      0      0  ...   \n",
       "1     0      0      1      0      1      0      0      1      0      1  ...   \n",
       "2     1      1      0      1      1      0      0      1      0      1  ...   \n",
       "3     0      1      0      0      1      1      0      0      1      0  ...   \n",
       "4     0      0      1      1      0      1      0      0      1      0  ...   \n",
       "\n",
       "   dv1  dv2  dv3  dv4  dv5  dv6  dv7  dv8  dv9  class  \n",
       "0    0    1    0    0    0    0    0    0    1      1  \n",
       "1    0    0    0    1    1    1    1    1    0      1  \n",
       "2    0    1    1    1    1    1    1    0    1      1  \n",
       "3    1    0    1    0    1    1    1    0    1      1  \n",
       "4    1    0    0    1    0    0    1    1    1      0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>demo</th>\n      <th>demo1</th>\n      <th>demo2</th>\n      <th>demo3</th>\n      <th>demo4</th>\n      <th>demo5</th>\n      <th>demo6</th>\n      <th>demo7</th>\n      <th>demo8</th>\n      <th>demo9</th>\n      <th>...</th>\n      <th>dv1</th>\n      <th>dv2</th>\n      <th>dv3</th>\n      <th>dv4</th>\n      <th>dv5</th>\n      <th>dv6</th>\n      <th>dv7</th>\n      <th>dv8</th>\n      <th>dv9</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 21 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 137
    }
   ],
   "source": [
    "dfp3_test.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   demo  demo1  demo2  demo3  demo4  demo5  demo6  demo7  demo8  demo9  ...  \\\n",
       "0     1      0      1      1      0      1      1      0      1      1  ...   \n",
       "1     0      0      1      1      0      0      0      0      1      0  ...   \n",
       "2     1      0      1      0      0      1      1      1      1      1  ...   \n",
       "3     1      1      0      1      1      1      1      0      0      0  ...   \n",
       "4     1      0      1      1      0      1      1      1      1      1  ...   \n",
       "\n",
       "   dv1  dv2  dv3  dv4  dv5  dv6  dv7  dv8  dv9  class  \n",
       "0    1    0    1    1    0    0    0    0    1      1  \n",
       "1    1    1    1    1    0    0    0    0    1      1  \n",
       "2    1    1    1    0    0    1    1    0    0      1  \n",
       "3    1    1    0    1    1    1    1    1    0      0  \n",
       "4    0    0    1    0    0    1    1    1    1      1  \n",
       "\n",
       "[5 rows x 21 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>demo</th>\n      <th>demo1</th>\n      <th>demo2</th>\n      <th>demo3</th>\n      <th>demo4</th>\n      <th>demo5</th>\n      <th>demo6</th>\n      <th>demo7</th>\n      <th>demo8</th>\n      <th>demo9</th>\n      <th>...</th>\n      <th>dv1</th>\n      <th>dv2</th>\n      <th>dv3</th>\n      <th>dv4</th>\n      <th>dv5</th>\n      <th>dv6</th>\n      <th>dv7</th>\n      <th>dv8</th>\n      <th>dv9</th>\n      <th>class</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 21 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 138
    }
   ],
   "source": [
    "dfp3_train.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "id": "5XqRflTlbSI-"
   },
   "outputs": [],
   "source": [
    "#uncomment andUselect out the X and Y variables as approprate.\n",
    "\n",
    "dfp3_X_train= dfp3_train.drop(columns = ['class']) \n",
    "dfp3_y_train= dfp3_train['class'].astype(int) \n",
    "dfp3_X_test= dfp3_test.drop(columns = ['class']) \n",
    "dfp3_y_test=dfp3_test['class'].astype(int) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "id": "me0iXREvbSJA"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(7474, 20)\n(7474,)\n"
     ]
    }
   ],
   "source": [
    "#Answer (add additional cells as necessary)\n",
    "from sklearn.linear_model import LogisticRegression \n",
    "classifier = LogisticRegression() \n",
    "print(dfp3_X_train.shape) \n",
    "print(dfp3_y_train.shape) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "id": "9IjsNavjbSJH"
   },
   "outputs": [],
   "source": [
    "classifier.fit(dfp3_X_train, dfp3_y_train) \n",
    "prediction = classifier.predict(dfp3_X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.6335295691731335\n"
     ]
    }
   ],
   "source": [
    "## Part A: Find the accuracy for model 1 in the train set\n",
    "\n",
    "model1_train_accuracy = classifier.score(dfp3_X_train, dfp3_y_train)\n",
    "print(model1_train_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.6215890850722311\n"
     ]
    }
   ],
   "source": [
    "## Part B: Find the accuracy for model 1 for the test set\n",
    "model1_test_accuracy = classifier.score(dfp3_X_test, dfp3_y_test) \n",
    "print(model1_test_accuracy) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.6215775692988474\n"
     ]
    }
   ],
   "source": [
    "## Part C: find the Area under the receiver operating characteristic curve (ROC AUC) \n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "model1_test_ROC = roc_auc_score(dfp3_y_test, prediction) \n",
    "print(model1_test_ROC) \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G2iTL3asbSJK"
   },
   "source": [
    "## Regression Models -  Models - Part 4\n",
    "\n",
    "For part 4 use `dfp4_train` and `dfp4_test` with `reg_target` as the dependent variable and all other variables as the independent variables. \n",
    "\n",
    "Create the values for `dfp4_X_train`, `dfp4_y_train`, `dfp4_X_test`, `dfp4_y_test` from the given training and test set `dfp4_train` and `dfp4_test`.  \n",
    "\n",
    "For the regression model we want to compare the performance of a linear regression (model1) with a RandomForest Regression (model2).\n",
    "\n",
    "In the first model (model1), use all of the variables except the `reg_target` and use linear regression. Create the following variables:\n",
    "\n",
    "4a. (3pts) `model1_train_r2`  R-squared for model 1 in the train set.\n",
    "\n",
    "4b. (3pts) `model1_test_r2`   R-squared for model 1 in the test set.\n",
    "\n",
    "4c. (3pts) `model1_test_msele`  Mean squared logarithmic error for model 1 in the test set. \n",
    "\n",
    "\n",
    "\n",
    "In the second model (model2), use all of the variables except the `reg_target` and use RandomForest regression. Create the following variables:\n",
    "\n",
    "4d. (3pts) `model2_train_r2`  R-squared for model 2 in the train set.\n",
    "\n",
    "4e. (3pts) `model2_test_r2`   R-squared for model 2 in the test set.\n",
    "\n",
    "4f. (3pts) `model2_test_msele`  Mean squared logarithmic error for model 2 in the test set. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "id": "kEDJILOfbSJK"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "      demo  demo1  demo2  demo3  demo4  demo5  demo6  demo7  demo8  demo9  \\\n",
       "0        1      0      0      0      0      0      1      1      0      0   \n",
       "1        0      0      1      1      0      0      0      1      1      1   \n",
       "2        1      0      1      0      0      0      0      0      1      1   \n",
       "3        1      1      1      0      0      1      1      0      1      1   \n",
       "4        0      1      0      0      1      1      0      1      1      1   \n",
       "...    ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "7469     0      1      0      1      1      1      0      0      0      1   \n",
       "7470     1      1      0      0      1      1      1      0      0      1   \n",
       "7471     1      1      0      1      0      0      0      1      0      0   \n",
       "7472     0      1      0      0      0      1      0      0      1      1   \n",
       "7473     1      0      1      0      0      0      1      0      1      1   \n",
       "\n",
       "      ...  dv1  dv2  dv3  dv4  dv5  dv6  dv7  dv8  dv9  reg_target  \n",
       "0     ...    0    1    1    1    0    0    0    0    1   91.883010  \n",
       "1     ...    0    0    1    0    1    1    0    0    1   58.996601  \n",
       "2     ...    1    0    1    1    1    1    0    0    1  166.333351  \n",
       "3     ...    0    0    1    0    0    0    0    0    0    4.450023  \n",
       "4     ...    1    0    0    1    0    0    0    1    1   47.069012  \n",
       "...   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...         ...  \n",
       "7469  ...    1    1    1    1    1    1    0    1    1   94.764442  \n",
       "7470  ...    1    0    1    1    1    0    0    0    1  -30.130155  \n",
       "7471  ...    1    1    1    1    1    1    0    1    1   96.604269  \n",
       "7472  ...    0    1    0    1    0    0    1    0    1    7.491651  \n",
       "7473  ...    0    0    1    1    0    1    1    0    1  115.824757  \n",
       "\n",
       "[7474 rows x 21 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>demo</th>\n      <th>demo1</th>\n      <th>demo2</th>\n      <th>demo3</th>\n      <th>demo4</th>\n      <th>demo5</th>\n      <th>demo6</th>\n      <th>demo7</th>\n      <th>demo8</th>\n      <th>demo9</th>\n      <th>...</th>\n      <th>dv1</th>\n      <th>dv2</th>\n      <th>dv3</th>\n      <th>dv4</th>\n      <th>dv5</th>\n      <th>dv6</th>\n      <th>dv7</th>\n      <th>dv8</th>\n      <th>dv9</th>\n      <th>reg_target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>91.883010</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>58.996601</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>166.333351</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>4.450023</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>47.069012</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>7469</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>94.764442</td>\n    </tr>\n    <tr>\n      <th>7470</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-30.130155</td>\n    </tr>\n    <tr>\n      <th>7471</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>96.604269</td>\n    </tr>\n    <tr>\n      <th>7472</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>7.491651</td>\n    </tr>\n    <tr>\n      <th>7473</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>115.824757</td>\n    </tr>\n  </tbody>\n</table>\n<p>7474 rows × 21 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 145
    }
   ],
   "source": [
    "#Load Part 3 Data\n",
    "dfp4_train  = pd.read_csv('midterm_train_part4.csv')\n",
    "dfp4_test   = pd.read_csv('midterm_test_part4.csv')\n",
    "dfp4_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "id": "8cSTe7nFb5Wu"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "      demo  demo1  demo2  demo3  demo4  demo5  demo6  demo7  demo8  demo9  \\\n",
       "0        1      1      0      1      0      0      1      0      0      0   \n",
       "1        1      1      0      0      0      0      0      1      0      0   \n",
       "2        0      0      0      1      0      0      0      1      1      1   \n",
       "3        0      0      1      1      0      0      0      1      1      1   \n",
       "4        1      1      0      0      1      1      0      0      1      1   \n",
       "...    ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "2487     0      0      1      0      1      0      1      1      0      0   \n",
       "2488     1      1      0      1      1      1      1      1      0      0   \n",
       "2489     1      1      0      0      1      0      0      0      0      0   \n",
       "2490     1      1      0      1      1      1      1      0      0      0   \n",
       "2491     1      0      1      1      0      0      0      0      1      1   \n",
       "\n",
       "      ...  dv1  dv2  dv3  dv4  dv5  dv6  dv7  dv8  dv9  reg_target  \n",
       "0     ...    0    1    1    1    1    1    1    0    0  141.044222  \n",
       "1     ...    0    1    0    0    1    0    1    1    0  127.018843  \n",
       "2     ...    0    0    1    0    1    1    0    0    0    5.828156  \n",
       "3     ...    0    0    0    1    0    1    1    0    0   22.186969  \n",
       "4     ...    1    0    1    1    1    0    1    1    0  126.927394  \n",
       "...   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...         ...  \n",
       "2487  ...    1    0    1    0    0    1    1    1    0  -60.596384  \n",
       "2488  ...    0    0    0    1    1    1    0    1    0  -69.268449  \n",
       "2489  ...    1    1    0    1    1    0    1    0    0   -6.871772  \n",
       "2490  ...    1    1    0    1    0    0    0    0    0   81.985439  \n",
       "2491  ...    0    0    0    0    0    1    1    1    1 -218.094282  \n",
       "\n",
       "[2492 rows x 21 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>demo</th>\n      <th>demo1</th>\n      <th>demo2</th>\n      <th>demo3</th>\n      <th>demo4</th>\n      <th>demo5</th>\n      <th>demo6</th>\n      <th>demo7</th>\n      <th>demo8</th>\n      <th>demo9</th>\n      <th>...</th>\n      <th>dv1</th>\n      <th>dv2</th>\n      <th>dv3</th>\n      <th>dv4</th>\n      <th>dv5</th>\n      <th>dv6</th>\n      <th>dv7</th>\n      <th>dv8</th>\n      <th>dv9</th>\n      <th>reg_target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>141.044222</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>127.018843</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>5.828156</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>22.186969</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>126.927394</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2487</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>-60.596384</td>\n    </tr>\n    <tr>\n      <th>2488</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>-69.268449</td>\n    </tr>\n    <tr>\n      <th>2489</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>-6.871772</td>\n    </tr>\n    <tr>\n      <th>2490</th>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>81.985439</td>\n    </tr>\n    <tr>\n      <th>2491</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-218.094282</td>\n    </tr>\n  </tbody>\n</table>\n<p>2492 rows × 21 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 146
    }
   ],
   "source": [
    "dfp4_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfp4_X_train = dfp2.drop( columns = ['class', 'reg_target']) \n",
    "y_class = dfp2['class'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "id": "2PYKO6Z5bSJM"
   },
   "outputs": [],
   "source": [
    "#Uncomment and Select out the X and Y variables as approprate. Don't reuse train_X, test_X, train_y, test_y.\n",
    "\n",
    "dfp4_X_train= dfp4_train.drop(columns = ['reg_target']) \n",
    "dfp4_y_train= dfp4_train['reg_target'].astype(int) \n",
    "dfp4_X_test= dfp4_test.drop(columns = ['reg_target']) \n",
    "dfp4_y_test=dfp4_test['reg_target'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "id": "_EhVTOhqbSJP"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "metadata": {},
     "execution_count": 149
    }
   ],
   "source": [
    "#Answer (add additional cells as necessary)\n",
    "\n",
    "## Model 1 = Linear Regression \n",
    "from sklearn.linear_model import LinearRegression \n",
    "lm = LinearRegression() \n",
    "lm.fit(dfp4_X_train, dfp4_y_train) \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.4519495623123778\n"
     ]
    }
   ],
   "source": [
    "# Part A: Model R^2 for model 1 in the train set \n",
    "model1_train_r2 = lm.score(dfp4_X_train, dfp4_y_train) \n",
    "print(model1_train_r2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "id": "D5cf2jVQbSJS"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.46648347875557716\n"
     ]
    }
   ],
   "source": [
    "#Part B: Model R^2 for model 1 in the test set \n",
    "model1_test_r2 = lm.score(dfp4_X_test, dfp4_y_test) \n",
    "print(model1_test_r2)   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "id": "b9rud5CTbSJW"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "8260.58848154461\n"
     ]
    }
   ],
   "source": [
    "##Part C: Model mean squared error for model 1 in the test set \n",
    "dfp4_predicted_linear = lm.predict(dfp4_X_test)\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "model1_test_msele = mean_squared_error(dfp4_y_test, dfp4_predicted_linear)\n",
    "print(model1_test_msele) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "RandomForestRegressor()"
      ]
     },
     "metadata": {},
     "execution_count": 153
    }
   ],
   "source": [
    "## Model 2: Random Forest Model\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "forest_reg = RandomForestRegressor() \n",
    "forest_reg.fit(dfp4_X_train, dfp4_y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "id": "5pqt7IobbSJZ"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.9283704360200933\n"
     ]
    }
   ],
   "source": [
    "## Part D: Find R^2 for model 2 in the train set \n",
    "model2_train_r2 = forest_reg.score(dfp4_X_train, dfp4_y_train) \n",
    "print(model2_train_r2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.532420073563072\n"
     ]
    }
   ],
   "source": [
    "## Part E: Find R^2 for model 2 in the test set \n",
    "model2_test_r2 = forest_reg.score(dfp4_X_test, dfp4_y_test) \n",
    "print(model2_test_r2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "8260.58848154461\n"
     ]
    }
   ],
   "source": [
    "## Part F: Find the Mean Squared error for model 2 in the test set\n",
    "dfp4_predicted_random_forest = forest_reg.predict(dfp4_X_test) \n",
    "model2_test_msele = mean_squared_error(dfp4_y_test, dfp4_predicted_random_forest) \n",
    "print(model1_test_msele) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "81MFVxX9bSJb"
   },
   "source": [
    "## Validate Variables\n",
    "There aren't tests, but this last check wil help you find if you have defined all expected variables (and don't have any accidental misspellings)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "id": "8yVYvy4ZbSJb"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The value for class_most_freq is defined. \nThe value for reg_target_mean is defined. \nThe value for class_num is defined. \nThe value for class_na is defined. \nThe value for df2 is defined. \nThe value for df3 is defined. \nThe value for y_class is defined. \nThe value for train_X is defined. \nThe value for test_X is defined. \nThe value for train_y is defined. \nThe value for test_y is defined. \nThe value for dfp3_X_train is defined. \nThe value for dfp3_y_train is defined. \nThe value for dfp3_X_test is defined. \nThe value for dfp3_y_test is defined. \nThe value for dfp4_X_train is defined. \nThe value for dfp4_y_train is defined. \nThe value for dfp4_X_test is defined. \nThe value for dfp4_y_test is defined. \nThe value for model1_train_accuracy is defined. \nThe value for model1_test_accuracy is defined. \nThe value for model1_test_ROC is defined. \nThe value for model1_train_r2 is defined. \nThe value for model1_test_r2 is defined. \nThe value for model1_test_msele is defined. \nThe value for model2_train_r2 is defined. \nThe value for model2_test_r2 is defined. \nThe value for model2_test_msele is defined. \n"
     ]
    }
   ],
   "source": [
    "expected_answers=['class_most_freq','reg_target_mean','class_num','class_na', 'df2','df3', \\\n",
    "                 'y_class', 'train_X', 'test_X', 'train_y', 'test_y', \\\n",
    "                'dfp3_X_train', 'dfp3_y_train', 'dfp3_X_test', 'dfp3_y_test', \\\n",
    "                  'dfp4_X_train', 'dfp4_y_train', 'dfp4_X_test', 'dfp4_y_test', \\\n",
    "                 'model1_train_accuracy','model1_test_accuracy','model1_test_ROC',  \\\n",
    "                 'model1_train_r2','model1_test_r2','model1_test_msele',\\\n",
    "                 'model2_train_r2','model2_test_r2','model2_test_msele']\n",
    "\n",
    "for x in expected_answers:\n",
    "        if x in globals():\n",
    "            print('The value for', x, 'is defined. ' )\n",
    "        else:\n",
    "            print('WARNING: The value for', x, 'is not defined. Check it out.') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "q01a:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01b:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01c:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01d:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01e:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01f:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01g:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q01h:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02a:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02b:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02c:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02d:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02e:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02f:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q02g:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q03a:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q03b:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q03c:\n",
       "\n",
       "    \n",
       "    0 of 1 tests passed\n",
       "    \n",
       "    \n",
       "    Tests failed:\n",
       "    \n",
       "           tests/q03c.py\n",
       "\n",
       "Test result:\n",
       "Trying:\n",
       "    round(model1_test_ROC,1)==0.7\n",
       "Expecting:\n",
       "    True\n",
       "**********************************************************************\n",
       "Line 2, in tests/q03c.py 0\n",
       "Failed example:\n",
       "    round(model1_test_ROC,1)==0.7\n",
       "Expected:\n",
       "    True\n",
       "Got:\n",
       "    False\n",
       "\n",
       "    \n",
       "    \n",
       "    \n",
       "    \n",
       "\n",
       "q04a:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q04b:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q04c:\n",
       "\n",
       "    \n",
       "    0 of 1 tests passed\n",
       "    \n",
       "    \n",
       "    Tests failed:\n",
       "    \n",
       "           tests/q04c.py\n",
       "\n",
       "Test result:\n",
       "Trying:\n",
       "    round(model1_test_mse,0)==8310.0\n",
       "Expecting:\n",
       "    True\n",
       "**********************************************************************\n",
       "Line 2, in tests/q04c.py 0\n",
       "Failed example:\n",
       "    round(model1_test_mse,0)==8310.0\n",
       "Exception raised:\n",
       "    Traceback (most recent call last):\n",
       "      File \"/Users/vijayyedidi/opt/anaconda3/lib/python3.8/doctest.py\", line 1336, in __run\n",
       "        exec(compile(example.source, filename, \"single\",\n",
       "      File \"<doctest tests/q04c.py 0[0]>\", line 1, in <module>\n",
       "        round(model1_test_mse,0)==8310.0\n",
       "    NameError: name 'model1_test_mse' is not defined\n",
       "\n",
       "    \n",
       "    \n",
       "    \n",
       "    \n",
       "\n",
       "q04d:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q04e:\n",
       "\n",
       "    All tests passed!\n",
       "    \n",
       "\n",
       "q04f:\n",
       "\n",
       "    \n",
       "    0 of 1 tests passed\n",
       "    \n",
       "    \n",
       "    Tests failed:\n",
       "    \n",
       "           tests/q04f.py\n",
       "\n",
       "Test result:\n",
       "Trying:\n",
       "    model2_test_mse > 7000 and model2_test_mse < 9000\n",
       "Expecting:\n",
       "    True\n",
       "**********************************************************************\n",
       "Line 2, in tests/q04f.py 0\n",
       "Failed example:\n",
       "    model2_test_mse > 7000 and model2_test_mse < 9000\n",
       "Exception raised:\n",
       "    Traceback (most recent call last):\n",
       "      File \"/Users/vijayyedidi/opt/anaconda3/lib/python3.8/doctest.py\", line 1336, in __run\n",
       "        exec(compile(example.source, filename, \"single\",\n",
       "      File \"<doctest tests/q04f.py 0[0]>\", line 1, in <module>\n",
       "        model2_test_mse > 7000 and model2_test_mse < 9000\n",
       "    NameError: name 'model2_test_mse' is not defined\n",
       "\n",
       "    \n",
       "    \n",
       "    \n",
       "    \n"
      ],
      "text/html": "<p><strong>q01a:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01b:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01c:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01d:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01e:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01f:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01g:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q01h:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02a:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02b:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02c:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02d:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02e:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02f:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q02g:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q03a:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q03b:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q03c:</strong></p>\n\n    \n    \n        <p>0 of 1 tests passed</p>\n        \n        \n        <p> <strong>Tests failed: </strong>\n            <ul>\n            \n                <li> \n    <p><strong style='color: red;'>tests/q03c.py</strong></p>\n    <p><strong>Test code:</strong><pre><div class=\"highlight\" style=\"background: #f8f8f8\"><pre style=\"line-height: 125%\"><span></span><span style=\"color: #000080; font-weight: bold\">&gt;&gt;&gt; </span><span style=\"color: #008000\">round</span>(model1_test_ROC,<span style=\"color: #666666\">1</span>)<span style=\"color: #666666\">==0.7</span>\n<span style=\"color: #888888\">True</span>\n</pre></div>\n</pre></p>\n    <p><strong>Test result:</strong><pre>Trying:\n    round(model1_test_ROC,1)==0.7\nExpecting:\n    True\n**********************************************************************\nLine 2, in tests/q03c.py 0\nFailed example:\n    round(model1_test_ROC,1)==0.7\nExpected:\n    True\nGot:\n    False\n</pre></p>\n     </li>\n            \n            </ul>\n        \n    \n    \n\n<p><strong>q04a:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q04b:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q04c:</strong></p>\n\n    \n    \n        <p>0 of 1 tests passed</p>\n        \n        \n        <p> <strong>Tests failed: </strong>\n            <ul>\n            \n                <li> \n    <p><strong style='color: red;'>tests/q04c.py</strong></p>\n    <p><strong>Test code:</strong><pre><div class=\"highlight\" style=\"background: #f8f8f8\"><pre style=\"line-height: 125%\"><span></span><span style=\"color: #000080; font-weight: bold\">&gt;&gt;&gt; </span><span style=\"color: #008000\">round</span>(model1_test_mse,<span style=\"color: #666666\">0</span>)<span style=\"color: #666666\">==8310.0</span>\n<span style=\"color: #888888\">True</span>\n</pre></div>\n</pre></p>\n    <p><strong>Test result:</strong><pre>Trying:\n    round(model1_test_mse,0)==8310.0\nExpecting:\n    True\n**********************************************************************\nLine 2, in tests/q04c.py 0\nFailed example:\n    round(model1_test_mse,0)==8310.0\nException raised:\n    Traceback (most recent call last):\n      File \"/Users/vijayyedidi/opt/anaconda3/lib/python3.8/doctest.py\", line 1336, in __run\n        exec(compile(example.source, filename, \"single\",\n      File \"<doctest tests/q04c.py 0[0]>\", line 1, in <module>\n        round(model1_test_mse,0)==8310.0\n    NameError: name 'model1_test_mse' is not defined\n</pre></p>\n     </li>\n            \n            </ul>\n        \n    \n    \n\n<p><strong>q04d:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q04e:</strong></p>\n\n    \n    \n        <p>All tests passed!</p>\n    \n    \n\n<p><strong>q04f:</strong></p>\n\n    \n    \n        <p>0 of 1 tests passed</p>\n        \n        \n        <p> <strong>Tests failed: </strong>\n            <ul>\n            \n                <li> \n    <p><strong style='color: red;'>tests/q04f.py</strong></p>\n    <p><strong>Test code:</strong><pre><div class=\"highlight\" style=\"background: #f8f8f8\"><pre style=\"line-height: 125%\"><span></span><span style=\"color: #000080; font-weight: bold\">&gt;&gt;&gt; </span>model2_test_mse <span style=\"color: #666666\">&gt;</span> <span style=\"color: #666666\">7000</span> <span style=\"color: #AA22FF; font-weight: bold\">and</span> model2_test_mse <span style=\"color: #666666\">&lt;</span> <span style=\"color: #666666\">9000</span>\n<span style=\"color: #888888\">True</span>\n</pre></div>\n</pre></p>\n    <p><strong>Test result:</strong><pre>Trying:\n    model2_test_mse > 7000 and model2_test_mse < 9000\nExpecting:\n    True\n**********************************************************************\nLine 2, in tests/q04f.py 0\nFailed example:\n    model2_test_mse > 7000 and model2_test_mse < 9000\nException raised:\n    Traceback (most recent call last):\n      File \"/Users/vijayyedidi/opt/anaconda3/lib/python3.8/doctest.py\", line 1336, in __run\n        exec(compile(example.source, filename, \"single\",\n      File \"<doctest tests/q04f.py 0[0]>\", line 1, in <module>\n        model2_test_mse > 7000 and model2_test_mse < 9000\n    NameError: name 'model2_test_mse' is not defined\n</pre></p>\n     </li>\n            \n            </ul>\n        \n    \n    \n\n"
     },
     "metadata": {},
     "execution_count": 158
    }
   ],
   "source": [
    "grader = otter.Notebook()\n",
    "grader.check_all()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of midterm_student.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "name": "Python 3.8.3 64-bit ('base': conda)",
   "display_name": "Python 3.8.3 64-bit ('base': conda)",
   "metadata": {
    "interpreter": {
     "hash": "212983865df447a587cd02ed42187f39760d467fb0227a7a0f46f8f8baf099d2"
    }
   }
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}