JupyterLabTest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b1292af4-ac6c-41aa-a776-d8870aa5f7c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>satisfaction_level</th>\n",
       "      <th>last_evaluation</th>\n",
       "      <th>number_project</th>\n",
       "      <th>average_montly_hours</th>\n",
       "      <th>time_spend_company</th>\n",
       "      <th>Work_accident</th>\n",
       "      <th>left</th>\n",
       "      <th>promotion_last_5years</th>\n",
       "      <th>sales</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.38</td>\n",
       "      <td>0.53</td>\n",
       "      <td>2</td>\n",
       "      <td>157</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>5</td>\n",
       "      <td>262</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.88</td>\n",
       "      <td>7</td>\n",
       "      <td>272</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.72</td>\n",
       "      <td>0.87</td>\n",
       "      <td>5</td>\n",
       "      <td>223</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.37</td>\n",
       "      <td>0.52</td>\n",
       "      <td>2</td>\n",
       "      <td>159</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   satisfaction_level  last_evaluation  number_project  average_montly_hours  \\\n",
       "0                0.38             0.53               2                   157   \n",
       "1                0.80             0.86               5                   262   \n",
       "2                0.11             0.88               7                   272   \n",
       "3                0.72             0.87               5                   223   \n",
       "4                0.37             0.52               2                   159   \n",
       "\n",
       "   time_spend_company  Work_accident  left  promotion_last_5years  sales  \\\n",
       "0                   3              0     1                      0  sales   \n",
       "1                   6              0     1                      0  sales   \n",
       "2                   4              0     1                      0  sales   \n",
       "3                   5              0     1                      0  sales   \n",
       "4                   3              0     1                      0  sales   \n",
       "\n",
       "   salary  \n",
       "0     low  \n",
       "1  medium  \n",
       "2  medium  \n",
       "3     low  \n",
       "4     low  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "df = pd.read_csv('HR_comma_sep.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5fb94a17",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>satisfaction_level</th>\n",
       "      <th>last_evaluation</th>\n",
       "      <th>number_project</th>\n",
       "      <th>average_montly_hours</th>\n",
       "      <th>time_spend_company</th>\n",
       "      <th>Work_accident</th>\n",
       "      <th>left</th>\n",
       "      <th>promotion_last_5years</th>\n",
       "      <th>sales_RandD</th>\n",
       "      <th>sales_accounting</th>\n",
       "      <th>sales_hr</th>\n",
       "      <th>sales_management</th>\n",
       "      <th>sales_marketing</th>\n",
       "      <th>sales_product_mng</th>\n",
       "      <th>sales_sales</th>\n",
       "      <th>sales_support</th>\n",
       "      <th>sales_technical</th>\n",
       "      <th>salary_low</th>\n",
       "      <th>salary_medium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.38</td>\n",
       "      <td>0.53</td>\n",
       "      <td>2</td>\n",
       "      <td>157</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>5</td>\n",
       "      <td>262</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.88</td>\n",
       "      <td>7</td>\n",
       "      <td>272</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.72</td>\n",
       "      <td>0.87</td>\n",
       "      <td>5</td>\n",
       "      <td>223</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.37</td>\n",
       "      <td>0.52</td>\n",
       "      <td>2</td>\n",
       "      <td>159</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   satisfaction_level  last_evaluation  number_project  average_montly_hours  \\\n",
       "0                0.38             0.53               2                   157   \n",
       "1                0.80             0.86               5                   262   \n",
       "2                0.11             0.88               7                   272   \n",
       "3                0.72             0.87               5                   223   \n",
       "4                0.37             0.52               2                   159   \n",
       "\n",
       "   time_spend_company  Work_accident  left  promotion_last_5years  \\\n",
       "0                   3              0     1                      0   \n",
       "1                   6              0     1                      0   \n",
       "2                   4              0     1                      0   \n",
       "3                   5              0     1                      0   \n",
       "4                   3              0     1                      0   \n",
       "\n",
       "   sales_RandD  sales_accounting  sales_hr  sales_management  sales_marketing  \\\n",
       "0            0                 0         0                 0                0   \n",
       "1            0                 0         0                 0                0   \n",
       "2            0                 0         0                 0                0   \n",
       "3            0                 0         0                 0                0   \n",
       "4            0                 0         0                 0                0   \n",
       "\n",
       "   sales_product_mng  sales_sales  sales_support  sales_technical  salary_low  \\\n",
       "0                  0            1              0                0           1   \n",
       "1                  0            1              0                0           0   \n",
       "2                  0            1              0                0           0   \n",
       "3                  0            1              0                0           1   \n",
       "4                  0            1              0                0           1   \n",
       "\n",
       "   salary_medium  \n",
       "0              0  \n",
       "1              1  \n",
       "2              1  \n",
       "3              0  \n",
       "4              0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#adjusting for dummy-variable trap: two or more variables that are highly correlated leads to poor model performance (why?)  drope one variable\n",
    "feats = ['sales','salary'] #key-error: had ()tuple and not an list array[]\n",
    "df_final = pd.get_dummies(df,columns=feats,drop_first=True)\n",
    "df_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "438330e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#seperate training and testing datasets using scikit-learn package\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# the column with category 'left' is your predicted output (y) so leave this out of the dataset\n",
    "X = df_final.drop(['left'],axis=1).values  #pandas.DataFrame.drop and .values is converting your csv \n",
    "y = df_final['left'].values #.values converts to numpy arrays, keras expects data as arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "55348826",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split data in 70% training - 30% testing (can also experiment with 80-20)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3)\n",
    "#capital X for df_final.drop and y for df_final[output]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5467801d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaling/normalizing the data for efficient computing: scikit-learn Standardscaler\n",
    "# normal distribution scaling: mean of 0 and SD (standard deviation) of 1\n",
    "# important because you are comparing features with different measurement \n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test) #why not fit-transform here too?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b5ad9a8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras #to build deep learning model with tensorflow as backend\n",
    "from keras.models import Sequential  #to initialize artifical neural network\n",
    "from keras.layers import Dense # to add layers to the model\n",
    "\n",
    "# 3-layer-model: input player (no computation) - hidden layer - output layer (result y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "154a263e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#initialize linear stack of layers, its a classification problem \n",
    "classifier = Sequential() \n",
    "\n",
    "# .add first layer to your initialized neural network\n",
    "classifier.add(Dense(9, kernel_initializer = 'uniform',activation = 'relu', input_dim=18))\n",
    "# first parameter (9) is the number of nodes (one strategy is avg of input and output )\n",
    "# second is the weights being initialized/seeded close to zero (but not zero)\n",
    "# third (most important) is the activation function\n",
    "# -- relu is chosen (ouput is either 0 or =input: max(x,0)) because fits well with the dataset, linear function wouldnt fit\n",
    "## activation functions(relu, sigmoid, softmax etc): https://keras.io/api/layers/activations/\n",
    "# last parameter: input_dim, represents the number of features/dimensions of your dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0e54d385",
   "metadata": {},
   "outputs": [],
   "source": [
    "# .add output layer \n",
    "classifier.add(Dense(1, kernel_initializer = 'uniform', activation = 'sigmoid'))\n",
    "# you exepct one output, P(employee leaving), thus 1 node\n",
    "# because you want the probability P(employee leaving): use sigmoid activation function\n",
    "# -- if you were dealing with more then 2 categories, use variant of sigmoid: softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "32f5967f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimizations step: reduce errors faster during training\n",
    "# apply 'gradient descent': how randomly assigned weights are adjusted\n",
    "# by reducing the 'cost function': find local minimum (error is at its least)\n",
    "classifier.compile(optimizer='adam',loss = 'binary_crossentropy', metrics=['accuracy'])\n",
    "# adam = a popular gradient descent optimization strategy\n",
    "# loss = a function used in the gradient descent, since binary problem use binary crossentropy\n",
    "# metric = the parameter you will evaluate the model with: accuracy of prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e1127c8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1050/1050 [==============================] - 3s 2ms/step - loss: 0.5620 - accuracy: 0.7585\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x21453d05e70>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now ready to fit composed classifier (2 layers + optimization) to your dataset\n",
    "classifier.fit(X_train, y_train, batch_size=10, epochs=1)\n",
    "# first parameter: training set\n",
    "# second parameter: column you make predictions on\n",
    "# batch_size = # of samples that will go through the network each round\n",
    "# epochs = number of times the dataset will be passed via the network, more takes longer but better results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "84e007ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141/141 [==============================] - 0s 2ms/step\n"
     ]
    }
   ],
   "source": [
    "# run predictions on test set\n",
    "y_pred = classifier.predict(X_test)\n",
    "y_pred = (y_pred>0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "65e5e39b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[False]\n",
      " [False]\n",
      " [False]\n",
      " ...\n",
      " [False]\n",
      " [False]\n",
      " [False]]\n"
     ]
    }
   ],
   "source": [
    "print(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "052544d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3464,    0],\n",
       "       [1036,    0]], dtype=int64)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate prediction accuracy using confusion/error matrix\n",
    "# shows # true and false positives, true and false negatives\n",
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(y_test,y_pred)\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ccc1fc5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7697777777777778\n"
     ]
    }
   ],
   "source": [
    "# 3464 + 0  correct predictions and 0 + 1036 wrong predictions\n",
    "# accuracy of\n",
    "print( (3464/(3464+1036)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "b3d23b1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 29ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[False]])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction for single employee data p(employee leaving)=\n",
    "new_pred = classifier.predict(sc.transform(np.array([[0.26,0.7 ,3., 238., 6., 0.,0.,0.,0., 0.,0.,0.,0.,0.,1.,0., 0.,1.]])))\n",
    "new_pred = (new_pred>0.5)\n",
    "new_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "ee11fb09",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tinke\\AppData\\Local\\Temp\\ipykernel_21780\\3256088042.py:18: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
      "  classifier = KerasClassifier(build_fn=make_classifier, batch_size=10, nb_epoch=1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8291264295578002"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training the model multiple times gets results with high variance, to\n",
    "# solve for this problem: K-fold-cross-validation\n",
    "# K=10 means train on first 9 folds then test on the last fold\n",
    "# iterate over all folds, accuracy is avg of all accuracies in each iteration\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "#import scikeras\n",
    "#from scikeras.wrappers import KerasClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "def make_classifier(): #basically the model you designed earlier \n",
    "        classifier = Sequential()\n",
    "        classifier.add(Dense(9, kernel_initializer='uniform', activation='relu',input_dim=18))\n",
    "        classifier.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))\n",
    "        classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "        return classifier\n",
    "\n",
    "#now pass to the kerasclassifier, where build_fn = your model as defined in make_classifier\n",
    "classifier = KerasClassifier(build_fn=make_classifier, batch_size=10, nb_epoch=1)\n",
    "\n",
    "# now apply cross validation for avg accuracies, time to fit\n",
    "accuracies = cross_val_score(estimator = classifier,X = X_train,y = y_train,cv = 10,n_jobs = -1)\n",
    "# cv = number of folds\n",
    "# n_jobs = # of cpu (-1 = all)\n",
    "\n",
    "mean = accuracies.mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "45b2b92e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0026780018634431713"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance = accuracies.var()\n",
    "variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "feb64ffc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#low variance between accuracies means model is performing well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "64b4e9c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combat overfitting by using dropout layer: randomly deactivates x (=rate) neurons each iteration\n",
    "from keras.layers import Dropout\n",
    "\n",
    "classifier = Sequential()\n",
    "classifier.add(Dense(9, kernel_initializer = \"uniform\", activation = \"relu\", input_dim=18))\n",
    "classifier.add(Dropout(rate = 0.1))\n",
    "classifier.add(Dense(1, kernel_initializer = \"uniform\", activation = \"sigmoid\"))\n",
    "classifier.compile(optimizer= \"adam\",loss = \"binary_crossentropy\",metrics = [\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "6ea6637d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tinke\\AppData\\Local\\Temp\\ipykernel_21780\\596553029.py:11: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
      "  classifier = KerasClassifier(build_fn = make_classifier)\n"
     ]
    }
   ],
   "source": [
    "# Gridsearch: experiment with different model parameters\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "def make_classifier(optimizer):\n",
    "    classifier = Sequential()\n",
    "    classifier.add(Dense(9, kernel_initializer = \"uniform\", activation = \"relu\", input_dim=18))\n",
    "    classifier.add(Dense(1, kernel_initializer = \"uniform\", activation = \"sigmoid\"))\n",
    "    classifier.compile(optimizer= optimizer,loss = \"binary_crossentropy\",metrics = [\"accuracy\"])\n",
    "    return classifier\n",
    "\n",
    "classifier = KerasClassifier(build_fn = make_classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "861e3248",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.5842 - accuracy: 0.7632\n",
      "Epoch 2/2\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.4014 - accuracy: 0.7967\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.5918 - accuracy: 0.7522\n",
      "Epoch 2/2\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.3973 - accuracy: 0.8213\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.5979 - accuracy: 0.7628\n",
      "Epoch 2/2\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.4315 - accuracy: 0.7969\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.5921 - accuracy: 0.7579\n",
      "Epoch 2/2\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.4361 - accuracy: 0.7672\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.6031 - accuracy: 0.7529\n",
      "Epoch 2/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.4078 - accuracy: 0.8213\n",
      "Epoch 3/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.3126 - accuracy: 0.8720\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.6011 - accuracy: 0.7533\n",
      "Epoch 2/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.3908 - accuracy: 0.8436\n",
      "Epoch 3/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.2943 - accuracy: 0.8926\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.5941 - accuracy: 0.7565\n",
      "Epoch 2/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.4279 - accuracy: 0.7861\n",
      "Epoch 3/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.3656 - accuracy: 0.7994\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "263/263 [==============================] - 1s 2ms/step - loss: 0.6014 - accuracy: 0.7547\n",
      "Epoch 2/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.4327 - accuracy: 0.7865\n",
      "Epoch 3/3\n",
      "263/263 [==============================] - 0s 2ms/step - loss: 0.3563 - accuracy: 0.8512\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "150/150 [==============================] - 1s 2ms/step - loss: 0.6386 - accuracy: 0.7525\n",
      "Epoch 2/2\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.4725 - accuracy: 0.7596\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "150/150 [==============================] - 1s 1ms/step - loss: 0.6496 - accuracy: 0.7429\n",
      "Epoch 2/2\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.5029 - accuracy: 0.7956\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "150/150 [==============================] - 1s 2ms/step - loss: 0.6455 - accuracy: 0.7491\n",
      "Epoch 2/2\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.5120 - accuracy: 0.7582\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/2\n",
      "150/150 [==============================] - 1s 1ms/step - loss: 0.6255 - accuracy: 0.7589\n",
      "Epoch 2/2\n",
      "150/150 [==============================] - 0s 1ms/step - loss: 0.4889 - accuracy: 0.7589\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "150/150 [==============================] - 1s 2ms/step - loss: 0.6484 - accuracy: 0.7451\n",
      "Epoch 2/3\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.4979 - accuracy: 0.7668\n",
      "Epoch 3/3\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.3990 - accuracy: 0.8331\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "150/150 [==============================] - 1s 1ms/step - loss: 0.6304 - accuracy: 0.7571\n",
      "Epoch 2/3\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.4744 - accuracy: 0.7777\n",
      "Epoch 3/3\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.4014 - accuracy: 0.7949\n",
      "165/165 [==============================] - 0s 2ms/step\n",
      "Epoch 1/3\n",
      "150/150 [==============================] - 1s 2ms/step - loss: 0.6412 - accuracy: 0.7539\n",
      "Epoch 2/3\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.5169 - accuracy: 0.7857\n",
      "Epoch 3/3\n",
      "150/150 [==============================] - 0s 2ms/step - loss: 0.4316 - accuracy: 0.7958\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "150/150 [==============================] - 1s 1ms/step - loss: 0.6496 - accuracy: 0.7528\n",
      "Epoch 2/3\n",
      "150/150 [==============================] - 0s 1ms/step - loss: 0.5357 - accuracy: 0.7878\n",
      "Epoch 3/3\n",
      "150/150 [==============================] - 0s 1ms/step - loss: 0.4312 - accuracy: 0.8318\n",
      "165/165 [==============================] - 0s 1ms/step\n",
      "Epoch 1/3\n",
      "525/525 [==============================] - 1s 1ms/step - loss: 0.4873 - accuracy: 0.7850\n",
      "Epoch 2/3\n",
      "525/525 [==============================] - 1s 2ms/step - loss: 0.2920 - accuracy: 0.8860\n",
      "Epoch 3/3\n",
      "525/525 [==============================] - 1s 1ms/step - loss: 0.2217 - accuracy: 0.9292\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 20, 'epochs': 3, 'optimizer': 'adam'}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#parameters to tune\n",
    "params = {\n",
    "    'batch_size':[20,35],\n",
    "    'epochs':[2,3],\n",
    "    'optimizer':['adam','rmsprop']\n",
    "}\n",
    "grid_search = GridSearchCV(estimator=classifier,\n",
    "                           param_grid=params,\n",
    "                           scoring=\"accuracy\",\n",
    "                           cv=2)\n",
    "#cv = # of folds you test on\n",
    "grid_search = grid_search.fit(X_train,y_train)\n",
    "\n",
    "best_param = grid_search.best_params_\n",
    "best_accuracy = grid_search.best_score_\n",
    "best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "6be2b7aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9009434722260022"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9122f648",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}