Prediction of wine quality II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "confident-resistance",
   "metadata": {},
   "source": [
    "### Done as assignments for the microcredential in data science course. Copyright of the University of Houston HPE Data Science Institute."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distant-prague",
   "metadata": {},
   "source": [
    "### Train and tune the MLPClassifier (via cross-validation) using at least three different combinations of architecture choices (e.g., number of layers, # of neurons per layer, activation function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "quarterly-carol",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "dataset = pd.read_csv('winequality-white.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "sophisticated-protection",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "central-burden",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "X_name = [ 'pH', 'residual sugar', 'alcohol','chlorides','citric acid','density','fixed acidity','volatile acidity','free sulfur dioxide','total sulfur dioxide','sulphates'] \n",
    "X = dataset[X_name]  \n",
    "y_name = 'quality'\n",
    "y = dataset[y_name]\n",
    "from sklearn import model_selection\n",
    "\n",
    "test_pct = 0.20\n",
    "seed = 7          \n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=test_pct, random_state=seed)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "registered-secondary",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 12.77745932\n",
      "Iteration 2, loss = 1.56635230\n",
      "Iteration 3, loss = 1.32958311\n",
      "Iteration 4, loss = 1.29540173\n",
      "Iteration 5, loss = 1.29111377\n",
      "Iteration 6, loss = 1.28921733\n",
      "Iteration 7, loss = 1.28800320\n",
      "Iteration 8, loss = 1.28732176\n",
      "Iteration 9, loss = 1.28683566\n",
      "Iteration 10, loss = 1.28687340\n",
      "Iteration 1, loss = 12.76780987\n",
      "Iteration 2, loss = 1.63110048\n",
      "Iteration 3, loss = 1.36862926\n",
      "Iteration 4, loss = 1.31338810\n",
      "Iteration 5, loss = 1.30367410\n",
      "Iteration 6, loss = 1.29973356\n",
      "Iteration 7, loss = 1.29781610\n",
      "Iteration 8, loss = 1.29615698\n",
      "Iteration 9, loss = 1.29551779\n",
      "Iteration 10, loss = 1.29473346\n",
      "Iteration 1, loss = 15.13161274\n",
      "Iteration 2, loss = 1.62703347\n",
      "Iteration 3, loss = 1.36958971\n",
      "Iteration 4, loss = 1.29924558\n",
      "Iteration 5, loss = 1.29242682\n",
      "Iteration 6, loss = 1.29029582\n",
      "Iteration 7, loss = 1.28937051\n",
      "Iteration 8, loss = 1.28885211\n",
      "Iteration 9, loss = 1.28863709\n",
      "Iteration 10, loss = 1.28872640\n",
      "Iteration 1, loss = 13.02687592\n",
      "Iteration 2, loss = 1.54212359\n",
      "Iteration 3, loss = 1.33314067\n",
      "Iteration 4, loss = 1.29794260\n",
      "Iteration 5, loss = 1.29277227\n",
      "Iteration 6, loss = 1.29239982\n",
      "Iteration 7, loss = 1.29168808\n",
      "Iteration 8, loss = 1.29083847\n",
      "Iteration 9, loss = 1.29092717\n",
      "Iteration 10, loss = 1.29067439\n",
      "Iteration 1, loss = 13.33808979\n",
      "Iteration 2, loss = 1.52850759\n",
      "Iteration 3, loss = 1.31480676\n",
      "Iteration 4, loss = 1.28723264\n",
      "Iteration 5, loss = 1.28485223\n",
      "Iteration 6, loss = 1.28385874\n",
      "Iteration 7, loss = 1.28354488\n",
      "Iteration 8, loss = 1.28368542\n",
      "Iteration 9, loss = 1.28311202\n",
      "Iteration 10, loss = 1.28300285\n",
      "Iteration 1, loss = 11.26460565\n",
      "Iteration 2, loss = 1.47371606\n",
      "Iteration 3, loss = 1.31062950\n",
      "Iteration 4, loss = 1.29630351\n",
      "Iteration 5, loss = 1.29276321\n",
      "Iteration 6, loss = 1.29097879\n",
      "Iteration 7, loss = 1.29001556\n",
      "Iteration 8, loss = 1.28940681\n",
      "Iteration 9, loss = 1.28976531\n",
      "Iteration 10, loss = 1.28919854\n",
      "Training set score: 0.444615\n",
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00         6\n",
      "           4       0.00      0.00      0.00        39\n",
      "           5       0.00      0.00      0.00       280\n",
      "           6       0.47      1.00      0.64       456\n",
      "           7       0.00      0.00      0.00       162\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.47       980\n",
      "   macro avg       0.07      0.14      0.09       980\n",
      "weighted avg       0.22      0.47      0.30       980\n",
      "\n",
      "Done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# First combination - relu as activation function and one hidden layer with 50 neurons\n",
    "# Design the classifier neural network\n",
    "model1 = MLPClassifier(hidden_layer_sizes=(50, ), \n",
    "                    activation = 'relu',  # ReLU is the default option\n",
    "                    # solver='sgd',  # default is Adam\n",
    "                    alpha=1e-4,  # regulariztion parameter, set to default=0.0001 (increase up to 1.0 for stronger regularization)\n",
    "                    learning_rate_init=.1 ,  # initial step-size for updating the weights, default is 0.001\n",
    "                    max_iter=10,  # number of epochs, default=200\n",
    "                    random_state=42,\n",
    "                    verbose=100, \n",
    "                    )\n",
    "scoring = 'accuracy'\n",
    "from sklearn.model_selection import KFold\n",
    "k4folds = 5\n",
    "\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", category=ConvergenceWarning, module=\"sklearn\")\n",
    "    kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "    cv_results = model_selection.cross_val_score(model1, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "    model1.fit(X_train, y_train)\n",
    "\n",
    "print(\"Training set score: %f\" % model1.score(X_train, y_train))\n",
    "y_predicted = model1.predict(X_test)   # use the trained classifier to predict on the test set\n",
    "\n",
    "print('\\n clasification report:\\n', classification_report(y_test, y_predicted))  # compare predictions with ground truth\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "presidential-satellite",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 8.82795478\n",
      "Iteration 2, loss = 1.40334913\n",
      "Iteration 3, loss = 1.31386856\n",
      "Iteration 4, loss = 1.29175688\n",
      "Iteration 5, loss = 1.28484202\n",
      "Iteration 6, loss = 1.28123264\n",
      "Iteration 7, loss = 1.26740329\n",
      "Iteration 8, loss = 1.25743891\n",
      "Iteration 9, loss = 1.25315052\n",
      "Iteration 10, loss = 1.25220460\n",
      "Iteration 1, loss = 15.28041417\n",
      "Iteration 2, loss = 1.61130694\n",
      "Iteration 3, loss = 1.36169679\n",
      "Iteration 4, loss = 1.30645469\n",
      "Iteration 5, loss = 1.29917096\n",
      "Iteration 6, loss = 1.29714099\n",
      "Iteration 7, loss = 1.29584971\n",
      "Iteration 8, loss = 1.29496916\n",
      "Iteration 9, loss = 1.29497842\n",
      "Iteration 10, loss = 1.29435878\n",
      "Iteration 1, loss = 8.93966092\n",
      "Iteration 2, loss = 1.60164288\n",
      "Iteration 3, loss = 1.30820019\n",
      "Iteration 4, loss = 1.29622515\n",
      "Iteration 5, loss = 1.28689753\n",
      "Iteration 6, loss = 1.28686033\n",
      "Iteration 7, loss = 1.28608162\n",
      "Iteration 8, loss = 1.28389985\n",
      "Iteration 9, loss = 1.28161881\n",
      "Iteration 10, loss = 1.28375809\n",
      "Iteration 1, loss = 8.72004619\n",
      "Iteration 2, loss = 1.30340249\n",
      "Iteration 3, loss = 1.29741915\n",
      "Iteration 4, loss = 1.29353044\n",
      "Iteration 5, loss = 1.29659322\n",
      "Iteration 6, loss = 1.29915799\n",
      "Iteration 7, loss = 1.30271992\n",
      "Iteration 8, loss = 1.30144743\n",
      "Iteration 9, loss = 1.29308780\n",
      "Iteration 10, loss = 1.29942779\n",
      "Iteration 1, loss = 11.05866729\n",
      "Iteration 2, loss = 2.09196511\n",
      "Iteration 3, loss = 1.32265800\n",
      "Iteration 4, loss = 1.28880521\n",
      "Iteration 5, loss = 1.28599699\n",
      "Iteration 6, loss = 1.27327105\n",
      "Iteration 7, loss = 1.25156187\n",
      "Iteration 8, loss = 1.24803568\n",
      "Iteration 9, loss = 1.25185445\n",
      "Iteration 10, loss = 1.24378940\n",
      "Iteration 1, loss = 7.19495248\n",
      "Iteration 2, loss = 1.31318838\n",
      "Iteration 3, loss = 1.30132439\n",
      "Iteration 4, loss = 1.29569851\n",
      "Iteration 5, loss = 1.29253972\n",
      "Iteration 6, loss = 1.28935749\n",
      "Iteration 7, loss = 1.29012507\n",
      "Iteration 8, loss = 1.29010887\n",
      "Iteration 9, loss = 1.29052847\n",
      "Iteration 10, loss = 1.29132076\n",
      "Training set score: 0.444615\n",
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00         6\n",
      "           4       0.00      0.00      0.00        39\n",
      "           5       0.00      0.00      0.00       280\n",
      "           6       0.47      1.00      0.64       456\n",
      "           7       0.00      0.00      0.00       162\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.47       980\n",
      "   macro avg       0.07      0.14      0.09       980\n",
      "weighted avg       0.22      0.47      0.30       980\n",
      "\n",
      "Done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# Second combination - relu as activation function and two hidden layer with 50 neurons and 100 neurons\n",
    "# Design the classifier neural network\n",
    "model2 = MLPClassifier(hidden_layer_sizes=(50, 100, ), \n",
    "                    activation = 'relu',  # ReLU is the default option\n",
    "                    # solver='sgd',  # default is Adam\n",
    "                    alpha=1e-4,  # regulariztion parameter, set to default=0.0001 (increase up to 1.0 for stronger regularization)\n",
    "                    learning_rate_init=.1 ,  # initial step-size for updating the weights, default is 0.001\n",
    "                    max_iter=10,  # number of epochs, default=200\n",
    "                    random_state=42,\n",
    "                    verbose=100, \n",
    "                    )\n",
    "scoring = 'accuracy'\n",
    "from sklearn.model_selection import KFold\n",
    "k4folds = 5\n",
    "\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", category=ConvergenceWarning, module=\"sklearn\")\n",
    "    kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "    cv_results = model_selection.cross_val_score(model2, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "    model2.fit(X_train, y_train)\n",
    "\n",
    "print(\"Training set score: %f\" % model2.score(X_train, y_train))\n",
    "y_predicted = model2.predict(X_test)   # use the trained classifier to predict on the test set\n",
    "\n",
    "print('\\n clasification report:\\n', classification_report(y_test, y_predicted))  # compare predictions with ground truth\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "saving-bradford",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 1.72798418\n",
      "Iteration 2, loss = 1.30713678\n",
      "Iteration 3, loss = 1.29665975\n",
      "Iteration 4, loss = 1.30047860\n",
      "Iteration 5, loss = 1.29747556\n",
      "Iteration 6, loss = 1.29629814\n",
      "Iteration 7, loss = 1.29583118\n",
      "Iteration 8, loss = 1.31184922\n",
      "Iteration 9, loss = 1.29743480\n",
      "Iteration 10, loss = 1.29395435\n",
      "Iteration 1, loss = 1.90612734\n",
      "Iteration 2, loss = 1.31404759\n",
      "Iteration 3, loss = 1.29934070\n",
      "Iteration 4, loss = 1.29802014\n",
      "Iteration 5, loss = 1.29588708\n",
      "Iteration 6, loss = 1.29621981\n",
      "Iteration 7, loss = 1.29587339\n",
      "Iteration 8, loss = 1.29593253\n",
      "Iteration 9, loss = 1.29735829\n",
      "Iteration 10, loss = 1.29999985\n",
      "Iteration 1, loss = 1.89950859\n",
      "Iteration 2, loss = 1.31290498\n",
      "Iteration 3, loss = 1.29364097\n",
      "Iteration 4, loss = 1.29396050\n",
      "Iteration 5, loss = 1.29112162\n",
      "Iteration 6, loss = 1.29560368\n",
      "Iteration 7, loss = 1.29311657\n",
      "Iteration 8, loss = 1.29571787\n",
      "Iteration 9, loss = 1.29502590\n",
      "Iteration 10, loss = 1.29801790\n",
      "Iteration 1, loss = 1.71984380\n",
      "Iteration 2, loss = 1.30238852\n",
      "Iteration 3, loss = 1.29978301\n",
      "Iteration 4, loss = 1.29768518\n",
      "Iteration 5, loss = 1.29557429\n",
      "Iteration 6, loss = 1.29850094\n",
      "Iteration 7, loss = 1.28939955\n",
      "Iteration 8, loss = 1.28996862\n",
      "Iteration 9, loss = 1.31807128\n",
      "Iteration 10, loss = 1.29824666\n",
      "Iteration 1, loss = 1.85754686\n",
      "Iteration 2, loss = 1.29901816\n",
      "Iteration 3, loss = 1.28934412\n",
      "Iteration 4, loss = 1.29223316\n",
      "Iteration 5, loss = 1.29283537\n",
      "Iteration 6, loss = 1.29487771\n",
      "Iteration 7, loss = 1.29123009\n",
      "Iteration 8, loss = 1.29219784\n",
      "Iteration 9, loss = 1.29518239\n",
      "Iteration 10, loss = 1.28930059\n",
      "Iteration 1, loss = 1.71648093\n",
      "Iteration 2, loss = 1.30583797\n",
      "Iteration 3, loss = 1.29333936\n",
      "Iteration 4, loss = 1.29431049\n",
      "Iteration 5, loss = 1.29817348\n",
      "Iteration 6, loss = 1.29515081\n",
      "Iteration 7, loss = 1.29491208\n",
      "Iteration 8, loss = 1.29522883\n",
      "Iteration 9, loss = 1.29412108\n",
      "Iteration 10, loss = 1.29742390\n",
      "Training set score: 0.444615\n",
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00         6\n",
      "           4       0.00      0.00      0.00        39\n",
      "           5       0.00      0.00      0.00       280\n",
      "           6       0.47      1.00      0.64       456\n",
      "           7       0.00      0.00      0.00       162\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.47       980\n",
      "   macro avg       0.07      0.14      0.09       980\n",
      "weighted avg       0.22      0.47      0.30       980\n",
      "\n",
      "Done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# third combination - logistic as activation function and two hidden layer with 50 neurons and 100 neurons\n",
    "# Design the classifier neural network\n",
    "model3 = MLPClassifier(hidden_layer_sizes=(50,100, ), \n",
    "                    activation = 'logistic',  # ReLU is the default option\n",
    "                    # solver='sgd',  # default is Adam\n",
    "                    alpha=1e-4,  # regulariztion parameter, set to default=0.0001 (increase up to 1.0 for stronger regularization)\n",
    "                    learning_rate_init=.1 ,  # initial step-size for updating the weights, default is 0.001\n",
    "                    max_iter=10,  # number of epochs, default=200\n",
    "                    random_state=42,\n",
    "                    verbose=100, \n",
    "                    )\n",
    "scoring = 'accuracy'\n",
    "from sklearn.model_selection import KFold\n",
    "k4folds = 5\n",
    "\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", category=ConvergenceWarning, module=\"sklearn\")\n",
    "    kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "    cv_results = model_selection.cross_val_score(model3, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "    model3.fit(X_train, y_train)\n",
    "\n",
    "print(\"Training set score: %f\" % model3.score(X_train, y_train))\n",
    "y_predicted = model3.predict(X_test)   # use the trained classifier to predict on the test set\n",
    "\n",
    "print('\\n clasification report:\\n', classification_report(y_test, y_predicted))  # compare predictions with ground truth\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-proof",
   "metadata": {},
   "source": [
    "#### The best combination is the third combination with logistic as activation function and two hidden layer with 50 neurons and 100 neurons as there is low loss and high accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "editorial-villa",
   "metadata": {},
   "source": [
    "### Study and describe the performance impact of varying at least three different combinations of optimizer parameter values (e.g., solver, epoch, learning rate) for one of the architectures in Step 1. Test the performance of the best."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "enhanced-microwave",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 1.72798418\n",
      "Iteration 2, loss = 1.30713678\n",
      "Iteration 3, loss = 1.29665975\n",
      "Iteration 4, loss = 1.30047860\n",
      "Iteration 5, loss = 1.29747556\n",
      "Iteration 6, loss = 1.29629814\n",
      "Iteration 7, loss = 1.29583118\n",
      "Iteration 8, loss = 1.31184922\n",
      "Iteration 9, loss = 1.29743480\n",
      "Iteration 10, loss = 1.29395435\n",
      "Iteration 1, loss = 1.90612734\n",
      "Iteration 2, loss = 1.31404759\n",
      "Iteration 3, loss = 1.29934070\n",
      "Iteration 4, loss = 1.29802014\n",
      "Iteration 5, loss = 1.29588708\n",
      "Iteration 6, loss = 1.29621981\n",
      "Iteration 7, loss = 1.29587339\n",
      "Iteration 8, loss = 1.29593253\n",
      "Iteration 9, loss = 1.29735829\n",
      "Iteration 10, loss = 1.29999985\n",
      "Iteration 1, loss = 1.89950859\n",
      "Iteration 2, loss = 1.31290498\n",
      "Iteration 3, loss = 1.29364097\n",
      "Iteration 4, loss = 1.29396050\n",
      "Iteration 5, loss = 1.29112162\n",
      "Iteration 6, loss = 1.29560368\n",
      "Iteration 7, loss = 1.29311657\n",
      "Iteration 8, loss = 1.29571787\n",
      "Iteration 9, loss = 1.29502590\n",
      "Iteration 10, loss = 1.29801790\n",
      "Iteration 1, loss = 1.71984380\n",
      "Iteration 2, loss = 1.30238852\n",
      "Iteration 3, loss = 1.29978301\n",
      "Iteration 4, loss = 1.29768518\n",
      "Iteration 5, loss = 1.29557429\n",
      "Iteration 6, loss = 1.29850094\n",
      "Iteration 7, loss = 1.28939955\n",
      "Iteration 8, loss = 1.28996862\n",
      "Iteration 9, loss = 1.31807128\n",
      "Iteration 10, loss = 1.29824666\n",
      "Iteration 1, loss = 1.85754686\n",
      "Iteration 2, loss = 1.29901816\n",
      "Iteration 3, loss = 1.28934412\n",
      "Iteration 4, loss = 1.29223316\n",
      "Iteration 5, loss = 1.29283537\n",
      "Iteration 6, loss = 1.29487771\n",
      "Iteration 7, loss = 1.29123009\n",
      "Iteration 8, loss = 1.29219784\n",
      "Iteration 9, loss = 1.29518239\n",
      "Iteration 10, loss = 1.28930059\n",
      "Iteration 1, loss = 1.93600572\n",
      "Iteration 2, loss = 1.53316520\n",
      "Iteration 3, loss = 1.43790999\n",
      "Iteration 4, loss = 1.41236195\n",
      "Iteration 5, loss = 1.40094239\n",
      "Iteration 6, loss = 1.39510657\n",
      "Iteration 7, loss = 1.39097767\n",
      "Iteration 8, loss = 1.38827007\n",
      "Iteration 9, loss = 1.38590893\n",
      "Iteration 10, loss = 1.38400258\n",
      "Training set score: 0.444615\n",
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00         6\n",
      "           4       0.00      0.00      0.00        39\n",
      "           5       0.00      0.00      0.00       280\n",
      "           6       0.47      1.00      0.64       456\n",
      "           7       0.00      0.00      0.00       162\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.47       980\n",
      "   macro avg       0.07      0.14      0.09       980\n",
      "weighted avg       0.22      0.47      0.30       980\n",
      "\n",
      "Done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "#combination 1\n",
    "#logistic as activation function and two hidden layer with 50 neurons and 100 neurons\n",
    "#First combination of optimizer parameter - solver - sgd, learning rate 0.001, alpha = 1.0\n",
    "\n",
    "model4 = MLPClassifier(hidden_layer_sizes=(50, 100,), \n",
    "                    activation = 'logistic',  # ReLU is the default option\n",
    "                    solver='sgd',  # default is Adam\n",
    "                    alpha=1.0,  # regulariztion parameter, set to default=0.0001 (increase up to 1.0 for stronger regularization)\n",
    "                    learning_rate_init=.001 ,  # initial step-size for updating the weights, default is 0.001\n",
    "                    max_iter=10,  # number of epochs, default=200\n",
    "                    random_state=42,\n",
    "                    verbose=100, \n",
    "                    )\n",
    "scoring = 'accuracy'\n",
    "from sklearn.model_selection import KFold\n",
    "k4folds = 5\n",
    "\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", category=ConvergenceWarning, module=\"sklearn\")\n",
    "    kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "    cv_results = model_selection.cross_val_score(model3, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "    model4.fit(X_train, y_train)\n",
    "\n",
    "print(\"Training set score: %f\" % model4.score(X_train, y_train))\n",
    "y_predicted = model4.predict(X_test)   # use the trained classifier to predict on the test set\n",
    "\n",
    "print('\\n clasification report:\\n', classification_report(y_test, y_predicted))  # compare predictions with ground truth\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "legitimate-buyer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 1.72798418\n",
      "Iteration 2, loss = 1.30713678\n",
      "Iteration 3, loss = 1.29665975\n",
      "Iteration 4, loss = 1.30047860\n",
      "Iteration 5, loss = 1.29747556\n",
      "Iteration 6, loss = 1.29629814\n",
      "Iteration 7, loss = 1.29583118\n",
      "Iteration 8, loss = 1.31184922\n",
      "Iteration 9, loss = 1.29743480\n",
      "Iteration 10, loss = 1.29395435\n",
      "Iteration 1, loss = 1.90612734\n",
      "Iteration 2, loss = 1.31404759\n",
      "Iteration 3, loss = 1.29934070\n",
      "Iteration 4, loss = 1.29802014\n",
      "Iteration 5, loss = 1.29588708\n",
      "Iteration 6, loss = 1.29621981\n",
      "Iteration 7, loss = 1.29587339\n",
      "Iteration 8, loss = 1.29593253\n",
      "Iteration 9, loss = 1.29735829\n",
      "Iteration 10, loss = 1.29999985\n",
      "Iteration 1, loss = 1.89950859\n",
      "Iteration 2, loss = 1.31290498\n",
      "Iteration 3, loss = 1.29364097\n",
      "Iteration 4, loss = 1.29396050\n",
      "Iteration 5, loss = 1.29112162\n",
      "Iteration 6, loss = 1.29560368\n",
      "Iteration 7, loss = 1.29311657\n",
      "Iteration 8, loss = 1.29571787\n",
      "Iteration 9, loss = 1.29502590\n",
      "Iteration 10, loss = 1.29801790\n",
      "Iteration 1, loss = 1.71984380\n",
      "Iteration 2, loss = 1.30238852\n",
      "Iteration 3, loss = 1.29978301\n",
      "Iteration 4, loss = 1.29768518\n",
      "Iteration 5, loss = 1.29557429\n",
      "Iteration 6, loss = 1.29850094\n",
      "Iteration 7, loss = 1.28939955\n",
      "Iteration 8, loss = 1.28996862\n",
      "Iteration 9, loss = 1.31807128\n",
      "Iteration 10, loss = 1.29824666\n",
      "Iteration 1, loss = 1.85754686\n",
      "Iteration 2, loss = 1.29901816\n",
      "Iteration 3, loss = 1.28934412\n",
      "Iteration 4, loss = 1.29223316\n",
      "Iteration 5, loss = 1.29283537\n",
      "Iteration 6, loss = 1.29487771\n",
      "Iteration 7, loss = 1.29123009\n",
      "Iteration 8, loss = 1.29219784\n",
      "Iteration 9, loss = 1.29518239\n",
      "Iteration 10, loss = 1.28930059\n",
      "Iteration 1, loss = 919359844759349401968300425557835776.00000000\n",
      "Iteration 2, loss = 5141987129152192146364238514463318702729452954187158529245749797516738560.00000000\n",
      "Iteration 3, loss = 28759176058301446978518153074373274920065456323777373470116341723297361901753195662643675194266786725753782272.00000000\n",
      "Iteration 4, loss = 160850306851847220406104643944769509239928051733444870466471623314644529493851560083560268585530577782737134582175652483237053013499580909172555776.00000000\n",
      "Iteration 5, loss = 899637081461696420007123740145730023974479577897806767064503978220974106197857946759585579064975537492444136193029695376687564424156190284492104710317200744034999394017507536462151680.00000000\n",
      "Iteration 6, loss = 5031677552759509115343752882205689086385213949177431564316673339659304929876969457184604687612377448903823892086942998124403086937732813638545582216939290831890621512476034579966067108451138373461969381242872062975934464.00000000\n",
      "Iteration 7, loss = 28142213695558848451833336333981415544508096614072272027225695511331481825340579338383598914458060751881284976384556997826480534887904666091315585210847626326186978540349137004144405468966976719062930980242492947028889961486915649669575700980371977794486272.00000000\n",
      "Iteration 8, loss = 157399631312255618777209974171684771324733092708741111372786110938149775195321253848888105612389888415005092140193177361264148637860333044115046773017565044333865865092346085090706985699917855902228925321935815423696899438056393821327510861947957156157850029648185562395830460603103991140712448.00000000\n",
      "Iteration 9, loss = inf\n",
      "Iteration 10, loss = inf\n",
      "Training set score: 0.001021\n",
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00         6\n",
      "           4       0.00      0.00      0.00        39\n",
      "           5       0.00      0.00      0.00       280\n",
      "           6       0.00      0.00      0.00       456\n",
      "           7       0.00      0.00      0.00       162\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      1.00      0.00         1\n",
      "\n",
      "    accuracy                           0.00       980\n",
      "   macro avg       0.00      0.14      0.00       980\n",
      "weighted avg       0.00      0.00      0.00       980\n",
      "\n",
      "Done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "#combination 2\n",
    "#logistic as activation function and two hidden layer with 50 neurons and 100 neurons\n",
    "#First combination of optimizer parameter - solver - sgd, learning rate 100, alpha = 10.0\n",
    "\n",
    "model4 = MLPClassifier(hidden_layer_sizes=(50, 100,), \n",
    "                    activation = 'logistic',  # ReLU is the default option\n",
    "                    solver='sgd',  # default is Adam\n",
    "                    alpha=10.0,  # regulariztion parameter, set to default=0.0001 (increase up to 1.0 for stronger regularization)\n",
    "                    learning_rate_init=100 ,  # initial step-size for updating the weights, default is 0.001\n",
    "                    max_iter=10,  # number of epochs, default=200\n",
    "                    random_state=2,\n",
    "                    verbose=100, \n",
    "                    )\n",
    "scoring = 'accuracy'\n",
    "from sklearn.model_selection import KFold\n",
    "k4folds = 5\n",
    "\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", category=ConvergenceWarning, module=\"sklearn\")\n",
    "    kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "    cv_results = model_selection.cross_val_score(model3, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "    model4.fit(X_train, y_train)\n",
    "\n",
    "print(\"Training set score: %f\" % model4.score(X_train, y_train))\n",
    "y_predicted = model4.predict(X_test)   # use the trained classifier to predict on the test set\n",
    "\n",
    "print('\\n clasification report:\\n', classification_report(y_test, y_predicted))  # compare predictions with ground truth\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ahead-incentive",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 1.72798418\n",
      "Iteration 2, loss = 1.30713678\n",
      "Iteration 3, loss = 1.29665975\n",
      "Iteration 4, loss = 1.30047860\n",
      "Iteration 5, loss = 1.29747556\n",
      "Iteration 6, loss = 1.29629814\n",
      "Iteration 7, loss = 1.29583118\n",
      "Iteration 8, loss = 1.31184922\n",
      "Iteration 9, loss = 1.29743480\n",
      "Iteration 10, loss = 1.29395435\n",
      "Iteration 1, loss = 1.90612734\n",
      "Iteration 2, loss = 1.31404759\n",
      "Iteration 3, loss = 1.29934070\n",
      "Iteration 4, loss = 1.29802014\n",
      "Iteration 5, loss = 1.29588708\n",
      "Iteration 6, loss = 1.29621981\n",
      "Iteration 7, loss = 1.29587339\n",
      "Iteration 8, loss = 1.29593253\n",
      "Iteration 9, loss = 1.29735829\n",
      "Iteration 10, loss = 1.29999985\n",
      "Iteration 1, loss = 1.89950859\n",
      "Iteration 2, loss = 1.31290498\n",
      "Iteration 3, loss = 1.29364097\n",
      "Iteration 4, loss = 1.29396050\n",
      "Iteration 5, loss = 1.29112162\n",
      "Iteration 6, loss = 1.29560368\n",
      "Iteration 7, loss = 1.29311657\n",
      "Iteration 8, loss = 1.29571787\n",
      "Iteration 9, loss = 1.29502590\n",
      "Iteration 10, loss = 1.29801790\n",
      "Iteration 1, loss = 1.71984380\n",
      "Iteration 2, loss = 1.30238852\n",
      "Iteration 3, loss = 1.29978301\n",
      "Iteration 4, loss = 1.29768518\n",
      "Iteration 5, loss = 1.29557429\n",
      "Iteration 6, loss = 1.29850094\n",
      "Iteration 7, loss = 1.28939955\n",
      "Iteration 8, loss = 1.28996862\n",
      "Iteration 9, loss = 1.31807128\n",
      "Iteration 10, loss = 1.29824666\n",
      "Iteration 1, loss = 1.85754686\n",
      "Iteration 2, loss = 1.29901816\n",
      "Iteration 3, loss = 1.28934412\n",
      "Iteration 4, loss = 1.29223316\n",
      "Iteration 5, loss = 1.29283537\n",
      "Iteration 6, loss = 1.29487771\n",
      "Iteration 7, loss = 1.29123009\n",
      "Iteration 8, loss = 1.29219784\n",
      "Iteration 9, loss = 1.29518239\n",
      "Iteration 10, loss = 1.28930059\n",
      "Iteration 1, loss = 3902463767253084262258002418743503519167465619976933785433872353209220071424.00000000\n",
      "Iteration 2, loss = 79573914169403214976341882173625050137656844779596758467434137597225523301603253594809033138577769706495836113281606769847556878057593360082108566474850304.00000000\n",
      "Iteration 3, loss = 1622566715256552439223606150485330692073084184215081377274675651931362033022451858249217754180786720804925206740003084097341470205276855893043609338601588511615353384108477765142570485554915976538385729898301868449778676504665289392128.00000000\n",
      "Iteration 4, loss = inf\n",
      "Iteration 5, loss = inf\n",
      "Iteration 6, loss = inf\n",
      "Iteration 7, loss = inf\n",
      "Iteration 8, loss = nan\n",
      "Iteration 9, loss = nan\n",
      "Iteration 10, loss = nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\utils\\extmath.py:152: RuntimeWarning: overflow encountered in matmul\n",
      "  ret = a @ b\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\neural_network\\_stochastic_optimizers.py:191: RuntimeWarning: overflow encountered in subtract\n",
      "  self.momentum * velocity - self.learning_rate * grad\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\neural_network\\_base.py:66: RuntimeWarning: invalid value encountered in subtract\n",
      "  tmp = X - X.max(axis=1)[:, np.newaxis]\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:186: RuntimeWarning: overflow encountered in multiply\n",
      "  coef_grads[layer] += self.alpha * self.coefs_[layer]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-e277674a76cf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[0mkfold\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_selection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mKFold\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_splits\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mk4folds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[0mcv_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_selection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcross_val_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkfold\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mscoring\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mmodel4\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Training set score: %f\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mmodel4\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    760\u001b[0m             \u001b[0mReturns\u001b[0m \u001b[0ma\u001b[0m \u001b[0mtrained\u001b[0m \u001b[0mMLP\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    761\u001b[0m         \"\"\"\n\u001b[1;32m--> 762\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mincremental\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    763\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    764\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_check_solver\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, incremental)\u001b[0m\n\u001b[0;32m    446\u001b[0m         \u001b[0mweights\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mchain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcoefs_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintercepts_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    447\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mw\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 448\u001b[1;33m             raise ValueError(\n\u001b[0m\u001b[0;32m    449\u001b[0m                 \u001b[1;34m\"Solver produced non-finite parameter weights. The input data may\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    450\u001b[0m                 \u001b[1;34m\" contain large values and need to be preprocessed.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed."
     ]
    }
   ],
   "source": [
    "#combination 3\n",
    "#logistic as activation function and two hidden layer with 50 neurons and 100 neurons\n",
    "#First combination of optimizer parameter - solver - sgd, learning rate 1000, alpha = 10.0\n",
    "\n",
    "model4 = MLPClassifier(hidden_layer_sizes=(50, 100,), \n",
    "                    activation = 'logistic',  # ReLU is the default option\n",
    "                    solver='sgd',  # default is Adam\n",
    "                    alpha=10.0,  # regulariztion parameter, set to default=0.0001 (increase up to 1.0 for stronger regularization)\n",
    "                    learning_rate_init=1000 ,  # initial step-size for updating the weights, default is 0.001\n",
    "                    max_iter=10,  # number of epochs, default=200\n",
    "                    random_state=2,\n",
    "                    verbose=100, \n",
    "                    )\n",
    "scoring = 'accuracy'\n",
    "from sklearn.model_selection import KFold\n",
    "k4folds = 5\n",
    "\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", category=ConvergenceWarning, module=\"sklearn\")\n",
    "    kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "    cv_results = model_selection.cross_val_score(model3, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "    model4.fit(X_train, y_train)\n",
    "\n",
    "print(\"Training set score: %f\" % model4.score(X_train, y_train))\n",
    "y_predicted = model4.predict(X_test)   # use the trained classifier to predict on the test set\n",
    "\n",
    "print('\\n clasification report:\\n', classification_report(y_test, y_predicted))  # compare predictions with ground truth\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "broadband-delivery",
   "metadata": {},
   "source": [
    "### Test the performance of the best MLPClassifier from Steps 1 and 2, using three scoring methods of your choice. Discuss in detail your results."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "diverse-syracuse",
   "metadata": {},
   "source": [
    "#### The scoring methods and results are already shown and discussed in steps 1 and 2. However, to summarize:\n",
    "#### 1. From step 1, the best combination is the third combination with logistic as activation function and two hidden layer with 50 neurons and 100 neurons as there is low loss and high accuracy 0.44.\n",
    "#### 2. From step 2, I used the logistic activation function and varied the values of alpha and learning rate. The besty combination is first combination of optimizer parameter - solver - sgd, learning rate 0.001, alpha = 1.0. The loss does not blow up and accuracy is also 0.44."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "threatened-civilian",
   "metadata": {},
   "source": [
    "### Train and tune a different classifier that is not a neural network; compare the MLPClassifier test results from Step 3 to that other classifier. Discuss in detail your results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "liable-victory",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ++ NOW WORKING ON ALGORITHM Scaled KNN ++\n",
      "Splitting data into 5 folds\n",
      "Training model on each split\n",
      "algorithm Scaled KNN accuracy results: mean = 0.530122 (std = 0.010059)\n",
      "done \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Pandas for data handling\n",
    "import pandas # https://pandas.pydata.org/\n",
    "\n",
    "# NumPy for numerical computing\n",
    "import numpy as np# https://numpy.org/\n",
    "\n",
    "# MatPlotLib for visualization\n",
    "import matplotlib.pyplot as pl  # https://matplotlib.org/\n",
    "import seaborn as sns\n",
    "\n",
    "# assessment\n",
    "from sklearn import model_selection # for model comparisons\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "\n",
    "# algorithms\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "# combining\n",
    "from sklearn.pipeline import make_pipeline\n",
    "# data preprocessing / feature selection\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "# Chose the Algorithms\n",
    "\n",
    "seed = 42 # setting the seed allows for repeatability\n",
    "k4folds = 5\n",
    "model = make_pipeline( MinMaxScaler(),KNeighborsClassifier(n_neighbors=5, weights='uniform', algorithm='auto', p=2) )\n",
    "modelname = 'Scaled KNN'\n",
    "scoring = 'accuracy'\n",
    "print(\" ++ NOW WORKING ON ALGORITHM %s ++\" % modelname)\n",
    "print(\"Splitting data into %s folds\" % k4folds)\n",
    "kfold = model_selection.KFold(n_splits=k4folds, random_state=seed, shuffle=True)\n",
    "print(\"Training model on each split\")\n",
    "cv_results = model_selection.cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "msg = \"algorithm %s %s results: mean = %f (std = %f)\" % (modelname, scoring, cv_results.mean(), cv_results.std())\n",
    "print(msg)\n",
    "\n",
    "    \n",
    "print('done \\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "proof-crack",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ++++ Detailed classification report for the selected model ++++ \n",
      "Algorithm: Scaled KNN \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAEGCAYAAACHNTs8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkg0lEQVR4nO3deXQUZdrG4d/TSQgEQoAEwuYMioooyg6y+YUtrCKjDg6KjogyKioMrqigoyOCIoIjqAGNCIIiwogsgiL7EhIIO0RWFcKSxLAFBZJ+vz+6w0SKsISuKmye65wcOp1U35Wc4k5VddX7ijEGpZQqyOP2CiilLj1aDEopCy0GpZSFFoNSykKLQSllEer2ChQmtFiVy/LtEo+Ia9lefYfqspN7Ys8ZNzjdY1BKWWgxKKUstBiUUhZaDEopCy0GpZSFFoNSykKLQSllocWglLLQYlBKWWgxKKUstBiUUhZBVwzt4uPYuGERWzYt4Zmn+ziaPSbhLdJ3r2VN6jxHc8PDw1m6ZAYpyXNZkzqPQQOfdDTfrZ/b7Wxwd3uzMzuoisHj8fDOyNfofGsPbqzdkrvu6krNmtc4lv/JJ5Pp1Pkex/LyHT9+nPh23WjQMJ4GDdsRHx9Ho0b1HMt36+d2O9vN7c3u7KAqhkYN67J9+y527vyJkydPMnnyV3S5tZ1j+YuXJPFL9kHH8grKyTkGQFhYKGFhoTg5lqebP7eb2W5ub3Zn21YMItJIRBr6H18vIv1FpKNdeQCVq1Tk593ppz7fvWcvlStXtDPykuHxeEheOYc9u9cyb95ikpNT3V6loOfm9mZ3ti3FICIvAe8A74nI68C7QEngORF54SzL9RaRFBFJ8Xpz7Fi1oOX1emnYqB1XXtWQBg3qcMP1NdxeJfUHZtdALXcCdYBwYB9Q1RhzWESGAUnAa2dayBiTACRA0QZqSd+zjyuqVj71edUqlUhP33fBK/9HdujQYRYuXEZ8uzg2bkpze3WCmpvbm93Zdh1K5Bpj8owxx4DtxpjDAMaYXwGvTZkkp6zh6quvpFq1KwgLC6Nbt9v4esZcu+IuGTEx5YiKKg1A8eLFad26BWlp21xeq+Dn5vZmd7ZdxXBCRCL8j+vnPykiUdhYDHl5efTt9yKzZk5kw7oFTJnyNZs2/WBXnMWE8aNYsmg6Na6tzq4dKfS8/2+O5FaqGMu3cyezKuVbli+bwbx5i5k1y7m379z6ud3OdnN7sztb7Dh7LSLhxpjjZ3g+BqhkjFl/rtfQMR+dp2M+Xn4KG/PRlnMMZyoF//OZQKYdmUqpwAmq6xiUUoGhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallIUWg1LKQotBKWVh192Vf2ihnhDXsqNLRLqWffTkb65l/3ryjBfLOkIvBbfSPQallIUWg1LKQotBKWWhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallIUWg1LKQotBKWURdMXg5rTkaWlLSUmZS1LSbJYunWFrVuUqFflieiLzl0/n+2Vf0esfPQAoUyaKSVPHsCRlFpOmjjk1EU2gvTt6CNt2rmT5ytmnnuv6lw6sSJ5N9uGt1K17oy25pwsPD2fpkhmkJM9lTeo8Bg180pHcfG5ub3Zm2zKvRCAUZV4Jj8fD5o2Lad+xO7t372XF8ln0uPdRNm/eemHZRbyJKi1tKU2bdiYrK7tIy8P530RVITaGCrHl2bBuMyVLRfDN/C94oMcTdLu7KwezDzFqxFj69HuQqDKlGfzy8PN6zQu5iapps4bkHD3G+2OG0aRRBwCurVEdr9fLiHf+zcDnh5Caes7pQ065mJuoSpaMICfnGKGhoSyYP43+T77EypWrz3v5ot5EFajtzc3swuaVCKo9BjenJXfagf2ZbFi3GYCco8fY+sMOKlaqQLsOLfli0n8B+GLSf2nfsZUt+cuWJpN92vTzP6RtZ9vWnbbknU1OzjEAwsJCCQsLxak/dm5ub3ZnO1YMIvKJ3RluTksOYIxhxowJLFs2k1697nYst+oVlal1U01SV60jpkI0B/b75vQ5sD+TmArRjq2HWzweD8kr57Bn91rmzVtMcnKqI7lubm92Z9syHoOITD/9KaCliJQBMMZ0KWS53kBvAAmJwuMpacfq2aZVqztIT99P+fLRzJz5KWlp21iyZKWtmRElIxjzyQheGjCEo0dyLF+/VA8VA8nr9dKwUTuiokrzxeSx3HB9DZ3p+yLZtcdQFTgMDAfe8n8cKfD4jIwxCcaYBsaYBkUpBTenJQdIT98PQEZGFtOnz6FBgzq25oWGhjJm3AimfTGT2TO+AyDzQBYVYmMA33mIrIxfbF2HS8mhQ4dZuHAZ8e3iHMlzc3uzO9uuYmgArAJeAA4ZYxYAvxpjFhpjFtqU6eq05BERJShVquSpx61bt2DjRnv/ar31n1fY9sMOEkaPO/Xc3G/m89fuXQH4a/euzJk939Z1cFtMTLlT77wUL16c1q1bkJa2zZFsN7c3u7PtmtTWC7wtIl/4/91vV1ZBBacGD/F4+Hjc545NSx4bW57PP08AfH/JP//8v3z7rW0dSMOb63Hn325j08Y05i76EoAhr45g1NtjeT9xON173M7un9N5uKc9b999mDiC5i0aEx1dlk1pS3j9tZFkZx/ijWGDiIkpx+Qvx7J+3SZu79rTlvx8lSrG8uGHbxMSEoLHI0yZMoNZs+bZmpnPze3N7mxH3q4UkU5AM2PM8+e7TFHergwUHfPReTrmozsKe7vSkcFgjTEzgZlOZCmlLl5QXceglAoMLQallIUWg1LKQotBKWWhxaCUstBiUEpZaDEopSy0GJRSFloMSikLR658/KMxuHeJbNK1sa5lv7PfvewxmcmuZR858atr2Zcq3WNQSlloMSilLLQYlFIWWgxKKQstBqWUhRaDUspCi0EpZaHFoJSy0GJQSlloMSilLLQYlFIWQVcMbk5LDr55FJNWzGba1MSAv3bZF5+m0uwviZ344annyv17IBXGJ1BhfAIVp02kwviE3y0TEluByvNnUuqebgFdl2Y929N/zhv0n/smzR/wzXZdIqokD45/nmfmD+fB8c9TorQ9Uwz+Z/Tr/LAziWUrZ5167pV/P0vS6jksWTGD8ZNGUzrKmWH43dze7MwOqmLweDy8M/I1Ot/agxtrt+Suu7pSs+Y1jq7D44/1YotNMyHlzJhDZr/nfvfcLy++yoF7e3Pg3t78On8Rvy5Y/LuvR/V7hN+WB3b+zNhrq9L4b634z20vMqLDs9RsVZfoP8fS8pHb2LZsA2+07M+2ZRuIe/SMU5RetEmfTuXOrg/87rn53y+lacOONL+5M9u37qT/kw/bkl2Qm9ub3dlBVQxuTksOUKVKRTp0aEVi4iRbXv/EmnV4Dx8u9Osl2sTx69zvT31e/JZm5KXvI3fHroCuR4Wrq/DTmm2c/O0E3jwvO5I2U6t9I25oW59VUxYBsGrKImq1bRDQ3HzLliaTnX3wd8/N/34JeXl5ACQnr6FyFftnnXZze7M725FiEJHmItJfROLtzHFzWnKAYW++zIDnB+P1eh3LzFeszk14f8km9+c9AEiJ4kTe9zcOjx13jiUv3P60n7my4XVElClFWPFiXNeyDmUqRVOqfBRHMg4CcCTjIKXKRwU8+3z0uPevfDd3ke05bm5vdmfbUgwisrLA44eAd4FI4CURee4sy/UWkRQRSfF6rVO6X8o6dmhNRkYWqanrXcmPiG/FsQJ7C6Ufup+jk6Zgfg38tHMHtqez4P3pPDh+AL3GPUf6ph/PWIZOTH94uieffoTcvFwmf/6V49nBxK6BWsIKPO4NtDXGZIjIMGAFMORMCxljEoAEKNrclW5OS96kaQM6dWpLu/YtKR4eTunSkSQmjqRnz772h4d4KNGyOQf+/r/j6mI3XEeJlrcQ9dg/8ESWAq8Xc/wEOVP+G5DI5MkLSJ68AID2T9/Fob2/cDTjEJHly3Ak4yCR5cuQk1n4YY8dut9zO/HtW9G1872O5Lm5vdmdbdehhEdEyopINL6JczMAjDE5QK5Nma5OSz5w4FCqX92IGjWacu99fViwYKkzpQCEN6xP7q6fyTuQeeq5jH/0Y99f7mbfX+7m6GdfcnjcxICVAkDJaN/U82UqR1OrfUNSpy9l03erqH/nLQDUv/MWNn67KmB559K6zS088c/e3H3XP/jVhr2kM3Fze7M72649hihgFSCAEZFKxpi9IlLK/5wt3JyW3AnlXn2R8Hq18ZSJouLXn3M44WOOfT2biLYtf3cY4YT73vsnEWVLkZebx38HJvLb4WPMf28694zqS6NucWTvyWRCn5G2ZI9NfJtmLRoTHV2WDWlLGPLaSP755MOEhxdj2vSPAUhJXkP/voNsyc/n5vZmd7Y4eRwoIhFArDFm57m+tyiHEoES4nHvzZrttZ19e7UgHfPx8pN7Ys8Z/1A7OhisMeYYcM5SUEq5K6iuY1BKBYYWg1LKQotBKWWhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallIWjVz7+UeS5MJ5Cvo/2VXIt+7VVr7qWvbruI65lL87Y5Fq2m9va2egeg1LKQotBKWWhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallMU5i0FErhWReSKywf/5TSLyov2rppRyy/nsMYwBBgAnAYwx64C/2blSSil3nU8xRBhjTp8V1ba5IZRS7jufeyUyRaQ6YABE5E5gr61rdRHaxccxfPgrhHg8fJQ4iTfeHOVY9piEt+jUsQ0HMjKpU7e17XmNerajXveWIELqpPkkffQNsdf/mU6vPUBoeBjevDxmvZhI+todF511/PgJ/t7naU6cPElebh5tWzbnsQfvZXf6Pp5+aQgHDx3m+hrXMGTQU4SFhXHixAkGvPoWm9K2UiaqNMNeGUCVSoEZnn7CsnH8mvMreXle8vLy6NPpce5/6j6axjfB6zUczDrIm/2HkbX/l4DknY3H42H5spmkp+/jL7f3tD2vIDu39fPZY+gDfABcJyJ7gH6Ae3e8nIWb05IDfPLJZDp1vseRrPLXVqVe95aM7TKID9oP4JrWdSn751jaDOjOopFTSej4PAuGT6HNgO4ByStWLIyP3hnC1HGjmTJuFEuTVrF2w2befu8j7r2rK7Mnf0TpyFJ8OWMOAFNnzKV0ZClmT/Z9ffjojwKyHvme7PYMD7d/lD6dHgdg8vtT6B3/CA+3f5QV3yXRo2+PgOYV5vHHerElbZsjWQXZva2fsxiMMTuMMW2A8sB1xpjmxphdAVuDAHJzWnKAxUuS+OW06dntEnN1Zfas2U7ubycweV5+TNpMzfYNwRiKlSoBQHhkBEcOBGZ9RISICN/r5ubmkpubi4iQtGot8XEtALitYxu+X7QcgO8XL+e2jm0AiI9rQdKqNbZOcnvs6LFTj0tEFMe/g2urKlUq0qFDKxITJ9medTq7t/VzHkqIyKDTPgfAGPPKWZZpDGw2xhwWkRLAc0A9YBMw2Bhz6GJWujBnmhq8UcO6dkS5LuOH3bR6uhslypTi5G8nuKZlHdLX7WDOK+O555NnafvC3YhHSLz9XwHLzMvLo9sDT/DTnnS6396ZK6pUIrJUSUJDQwCILR/DgYwsAA5kZFGxQgwAoaEhlCoZwcFDhylbJuqi18MYGPrpYIyBmZ/OZObE2QD0fOZ+2t7RhpwjOTzV7ZmLzjmXYW++zIDnBxMZWdL2rNPZva2fz6FEToGPPKADUO0cy3wE5Ff4SHxzWQ71P5dY2EIi0ltEUkQkxevNOY9Vu3xlbktn6ftfc8+E57jnk2fZt/FHvHle6vdow5xXJzCyyRPMfWUCt77xUMAyQ0JC+HLcKOZNG8/6TT+w88efA/baF6LfHf15pONjPH/fC3T5exdubFwLgMQ3Pubuxj34ftr33HZ/F1vXoWOH1mRkZJGaut7WHLecz6HEWwU+XgPigKvO9brGmPx3LhoYY/oZY5YYY/51tmWNMQnGmAbGmAYez4W3sJvTkrthzecLGdv5RcZ1e5XfDuXwy8591L6jBVtm++aB3DQziSq1qwc8t3RkKRrVu4k1G7Zw5GgOubl5AOzPyKRC+WgAKpSPZp9/9u3c3DyO5hyjTFTpgORn7fPtlRzMOsTSb5ZyXZ3rfvf1edO+p0XH5gHJKkyTpg3o1KktaWnLGP/JKOLimpGYaM8kvmdi97ZelCsfI4Cq5/ieDSKSf4p2rYg0AN/FUvivh7CDm9OSuyHCPxV96crRXNe+Ieu/WsaRA9n8+eaaAFzZ7AaydgVmY/kl+yCHjxwF4Lfjx1menMpV1a6gUb2bmLtgMQBfzfqOVi2aANCy+c18Nes7AOYuWEzj+rVPHYZejOIlwilRssSpx/Vvqc+utF1Uqfa//yRN45vw8zZ792YGDhxK9asbUaNGU+69rw8LFiylZ8++tmYWZPe2fj7nGNbzvzM5IfhOQhZ6fsHvQWCk/wrJTGC5iPwM/Oz/mi3cnJYcYML4UfzfLU2IiSnHrh0p/OuVYSR+/Jlted3e70uJspHkncxl9qCPOX74GDOeHUu7l+/DE+Ih7/hJZj43NiBZGVnZvPDvYeR5vRivoV2rFsQ1a0z1an/i6ZeG8J+ET6h5bXVu7xwPwO2d2zHg1Tfp0O0BokpH8ua/ngvIepQtX5aXx7wE+A5tvv9qPskLUnjpg4FUrV4V4/Wyf/cBRjz/TkDyLlV2b+tyrjPFIvLnAp/mAvsLHCaca9nSwJX4Cmi3MWb/+a5YaLEq9p9WvgQNqhTnWvYAF8d87KBjProi98SeM+7GnXWPQURCgDnGmOvO9n2FMcYcBtYWZVmllHvOeo7BGJMHpInInxxaH6XUJeB8LokuC2wUkZX43rIEwBhj7/tBSinXnE8xFAc6F/hc8F2ToJQKUudTDKHGmIUFn/BfzaiUClKFFoOIPAI8ClwlIusKfCkSWGr3iiml3HO2PYaJwGzgdXz3OuQ7Yoyx/35WpZRrCi0G/41Oh4DA3LerlPrD0MFglVIWWgxKKYtzXhLtlsv1kuiyJUq5lt2iTA3XsstLuGvZiXuXu5btdfn/X2GXROseg1LKQotBKWWhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallIUWg1LKQotBKWWhxaCUsgi6YmgXH8fGDYvYsmkJzzzdJ6izR747mE3blrFo+de/e/7B3j1YljybxStmMOiVp23Jjihdkqffe5Z35o3mnXmjuLbe/+6z6PJQV6b+OJ3IspEBz429qjIvznrz1MeI9eNo/UBHqtb8M89OfY1B37xFn7HPUryU/YOMhYeHs3TJDFKS57ImdR6DBj5pe2ZBdm5v5zO02x9G/tTg7Tt2Z/fuvaxYPouvZ8xl8+atQZn92cSpfDhmAu++/78hOJu1aEz7Tq2Ja9aFEydOEhNTzpbsXi89ROrC1bz5yFBCw0IpVsJ3E1R0pRhqt6hDxu4DtuTu35HOvzv6yk48HoYmfUDqnJX8Y/STTBk8nq1Jm2j615bE9+7C9OGf27IO+Y4fP058u27k5BwjNDSUBfOn8c2c+axcudrWXLB/ewuqPQa7pwa/1LKXL0shO/v3E4f37NWdd95O4MQJ30yAmZmBH2wrIjKC6xvfwHeffQtA7slcjh32DSD+wKBejH/9Y1unvM93XbNaZPy4j1/2ZBJ7ZWW2Jvkmjtm8ZB11O9xsez5ATo5v7uawsFDCwkId+bnB/u3NlmIQkSdE5Ao7XvtszjQ1eOXKFYM+u6Dq1atxc5MGfDNvMl/NHE+dejcGPKPCFbEczjrEY8P6MmzWCB4d+hjhJcJp2LYxWfuy2LV5V8Azz6Thrc1Inu4bfjR968/Ujm8IQP2OTShXKdqRdfB4PCSvnMOe3WuZN28xycmpjuTavb3ZtcfwKpAkIotF5FERKX8+C4lIbxFJEZEUrzfn3Asoi5DQEMqWjaJ96268PPANxn48IvAZISFcVas6cybM5qmO/fjt2G/c9c/u3NHnTj4bPjHgeWdch7BQardpwKpZvrEUxj0zmrge7Xj+66EUL1Wc3JPnNYviRfN6vTRs1I4rr2pIgwZ1uOF698a0CCS7imEHvhmxXwXqA5tE5BsR+buIFHpGyhiTYIxpYIxp4PGUvOBQu6cGv1SzC9qbvp8ZX/t28VNXr8fr9RIdXTagGVn7Msnam8nWNb5JVJfPWsZVtaoTe0Usw2eP5P0lY4iuFMOwmSMoU75MQLPz1Yqrw08bdnIk03cotX97OiPv+zeDb32WldOXkvHjeU+TGhCHDh1m4cJlxLeLcyTP7u3NrmIwxhivMWauMaYXUBkYDbTHVxq2sHtq8Es1u6BZM7+jeYvGAFxVvRrFwsLIysoOaMbBjINk7s2k8lVVALipWW12bNhOz/r38XDzh3i4+UNk7c3kqU79OJhxMKDZ+Rp2aU7y10tOfR4ZXRoAEaHjY3ew6FP7f/cxMeWIivLlFi9enNatW5CWts32XLB/e7PrXYnfDRdljDkJTAemi0iETZm2Tw1+qWV/8OFbNGveiHLRZVm7aSFvvP4fJo7/kpGjBrNo+decPHmSxx4JzPTzpxv7UgL9RvYnNCyM/T/t492nRtqScybFSoRTs/lNTHg+4dRzDbs0J+5e38m31DkrWfbFfNvXo1LFWD788G1CQkLweIQpU2Ywa9Y823PB/u3NljEfReRaY8xFraWO+eg8HfPReZfVmI8XWwpKKXcF1XUMSqnA0GJQSlloMSilLLQYlFIWWgxKKQstBqWUhRaDUspCi0EpZaHFoJSysOWS6EC4XC+JDvG419WRxewfDq0wJ715rmWfyHPmFu0zOeliNjh8SbRS6o9Ni0EpZaHFoJSy0GJQSlloMSilLLQYlFIWWgxKKQstBqWUhRaDUspCi0EpZaHFoJSyCLpicHoq+kslG3zzKCatmM20qYm2Z40cNZjN25ezeMWMU889M+Bx1m9ZzPwlXzF/yVe0if8/W7LfHT2EbTtXsnzl7FPPdf1LB1Ykzyb78Fbq1g38fJ2FiYoqzcSJ77FmzTxSU+fRuHE9x7Lt3N6CqhjypwbvfGsPbqzdkrvu6krNmtcEfXa+xx/rxRaHZkL67NOp3HV7L8vz749KpGXz22jZ/Da+m7vQluyJn37JHV17/u65TZt+oMfdj7J06UpbMgszbNhLzJ27kDp1WtOoUXu2bHHm92/39hZUxeDGVPSXQjZAlSoV6dChFYmJkxzJW74shezsQ45knW7Z0mSysw/+7rkf0razbetOR9ejdOlImjdvzMcffwbAyZMnOXTosCPZdm9vthSDiBQTkftEpI3/87tF5F0R6SMiYXZkgrtT0buZDTDszZcZ8PxgvF6vY5ln0qt3DxYum87IUYOJKlPa1XWxW7VqV5CZmUVCwjCWL5/F6NFDiYhw5tZ1u7c3u/YYEoFOQF8RGQ/8FUgCGgJjC1tIRHqLSIqIpHi9OTatWvDp2KE1GRlZpKaud3U9EsdOpEHtNsQ1u439+zJ45TV75s28VISGhlCnTi3GjJlAkyYdOXbsGE899ajbqxUQdk1qe6Mx5iYRCQX2AJWNMXkiMgFYW9hCxpgEIAGKNlCLm1PRu5ndpGkDOnVqS7v2LSkeHk7p0pEkJo6kZ8++juTny8jIOvV4/LjJTJz8gaP5TtuzZx979uwlOXkNANOmzeLJJ50pBru3N7v2GDwiUgyIBCKAKP/z4YBthxJuTkXvZvbAgUOpfnUjatRoyr339WHBgqWOlwJAbGz5U4873dqWLZu3Or4OTtq/P4Pdu/dyzTVXARAX14wtW5z5me3e3uzaY/gQ2AKEAC8AX4jIDuBm4DObMl2Ziv5SyHZDwkfDada8EeWiy7Ju8yKGDn6HZi0aU+vG6zDG8PNPe3iy7yBbsj9MHEHzFo2Jji7LprQlvP7aSLKzD/HGsEHExJRj8pdjWb9uE7ef9s6FHfr3f4nExJEUKxbGrl0/0bv3U7Zngv3bm21jPopIZQBjTLqIlAHaAD8ZY87r/SQd89F5Ouaj8y7VMR/t2mPAGJNe4PFBYIpdWUqpwAqq6xiUUoGhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallIUWg1LKQotBKWVh25WPqmjyXBxP4eBvequ78tE9BqWUhRaDUspCi0EpZaHFoJSy0GJQSlloMSilLLQYlFIWWgxKKQstBqWUhRaDUspCi0EpZRF0xeDmVPSa7Xz2mIS3SN+9ljWp8xzNzResv3fb5pW4WEWZV8Lj8bB542Lad+zO7t17WbF8Fj3ufZTNDsyIpNnOZwO0aN6Yo0dzSEwcSZ26rR3JzBcMv/fC5pUIqj0GN6ei12znswEWL0nil+yDjuUVFMy/d9uKQUSuEpGnRGSkiAwXkYdFxNZ50d2cil6znc92WzD/3m0pBhF5AngfKA40xDeZ7RXAChGJO8tyvUUkRURSvF4dG0Apt9g1UMtDQB1jTJ6IDAdmGWPiROQD4Cug7pkWMsYkAAlQtHMMbk5Fr9nOZ7stmH/vdp5jyC+dcKAUgDHmJyDMrkA3p6LXbOez3RbMv3e79hjGAskikgS0AIYCiEh54BebMl2dil6znc8GmDB+FP93SxNiYsqxa0cK/3plGIkff+ZIdjD/3m17u1JEbgBqAhuMMVsudPmiHEoopS5MYW9XBtV1DEqpC3NZXMeglAoMLQallIUWg1LKQotBKWWhxaCUstBiUEpZaDEopSy0GJRSFloMSikLLQallMUle0n0xRKR3v7buDVbszX7AgXzHkNvzdZszS6aYC4GpVQRaTEopSyCuRhcOebTbM0OhuygPfmolCq6YN5jUEoVkRaDUsoiqIpBRIqLyEoRWSsiG0XkXy6sQ4iIpIrIDIdzd4nIehFZIyIpDmeXEZEpIrJFRDaLSBMHs2v4f+b8j8Mi0s/B/H/6t7UNIjJJRIo7mN3Xn7sx0D9zUJ1jEBEBShpjjopIGLAE6GuMWeHgOvQHGgCljTGdHczdBTQwxmQ6lVkgexyw2BgzVkSKARHGmIMurEcIsAdobIz50YG8Kvi2seuNMb+KyGR8c6h87EB2LeAzoBFwAvgGeNgYsy0Qrx9UewzG56j/0zD/h2PNJyJVgU74hs+/LIhIFHAL8CGAMeaEG6Xg1xrY7kQpFBAKlBCRUCACSD/H9wdKTSDJGHPMGJMLLARuD9SLB1UxwKld+TXAAeBbY0ySg/EjgGcAr4OZ+QwwV0RWiYiTV+JdCWQAif5DqLEiUtLB/IL+BkxyKswYswcYBvwE7AUOGWOcmm1nA9BCRKJFJALoiG8ayIAIumIwxuQZY+oAVYFG/l0u24lIZ+CAMWaVE3ln0NwYUw/oAPQRkVscyg0F6gHvGWPqAjnAcw5ln+I/hOkCfOFgZlngNnzlWBkoKSI9nMg2xmzGN5HTXHyHEWuAvEC9ftAVQz7/7ux8oL1Dkc2ALv5j/c+AViIywaHs/L9eGGMOANPwHXs6YTewu8Ce2RR8ReG0DsBqY8x+BzPbADuNMRnGmJPAVKCpU+HGmA+NMfWNMbcA2UDApqIKqmIQkfIiUsb/uATQFrjgWbCKwhgzwBhT1RhTDd8u7ffGGEf+eohISRGJzH8MxOPb1bSdMWYf8LOI1PA/1RrY5ET2abrj4GGE30/AzSIS4T/x3RrY7FS4iFTw//snfOcXJgbqte2au9ItlYBx/rPTHmCyMcbRtw1dEgtM822bhAITjTHfOJj/OPCpf3d+B9DTwez8MmwL/MPJXGNMkohMAVYDuUAqzl4e/aWIRAMngT6BPOkbVG9XKqUCI6gOJZRSgaHFoJSy0GJQSlloMSilLLQYlFIWWgyqyEQkLv8uUhHpIiKFXvHovwPz0SJkvCwiT13MeqoLp8WgLPzXgVwQY8x0Y8yQs3xLGeCCi0G5Q4vhMiMi1fzjJnzqHzthiv/KvV0iMlREVgN/FZF4EVkuIqtF5AsRKeVfvr1/+dUUuJtPRO4XkXf9j2NFZJp/XIy1ItIUGAJU94+Z8Kb/+54WkWQRWVdw7AwReUFEfhCRJUANlOOC7cpHdX5qAL2MMUtF5CP+95c8yxhTT0Ri8F3338YYkyMizwL9ReQNYAzQCtgGfF7I678DLDTG/MW/91EK341Vtfw3uCEi8cA1+O7pEGC6/8avHHyXlNfBt32uBty6Me2ypcVwefrZGLPU/3gC8IT/cf5/9JuB64Gl/susiwHLgevw3TS0FcB/k9iZbvFuBdwHvrtdgUP+OxELivd/pPo/L4WvKCKBacaYY/6M6UX/MVVRaTFcnk6/Dj7/8xz/v4JvLIvuBb9JROoEcB0EeN0Y88FpGf0CmKGKSM8xXJ7+VGBcxrvxDU9W0AqgmYhcDafu3rwW352q1USkuv/7unNm84BH/MuG+Ed5OoJvbyDfHOCBAucuqvjvFlwEdBWREv47Rm+9mB9UFY0Ww+UpDd9gLpuBssB7Bb9ojMkA7gcmicg6/IcRxpjf8B06zPSffDxQyOv3BVqKyHp85weuN8Zk4Ts02SAib/pHOpoILPd/3xQg0hizGt8hzVpgNpAcyB9cnR+9u/IyIyLVgBnGGEdGtlJ/TLrHoJSy0D0GpZSF7jEopSy0GJRSFloMSikLLQallIUWg1LK4v8Bd5NcdabDnZgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00         6\n",
      "           4       0.45      0.13      0.20        39\n",
      "           5       0.55      0.62      0.58       280\n",
      "           6       0.62      0.66      0.64       456\n",
      "           7       0.49      0.49      0.49       162\n",
      "           8       0.46      0.17      0.24        36\n",
      "           9       0.00      0.00      0.00         1\n",
      "\n",
      "    accuracy                           0.58       980\n",
      "   macro avg       0.37      0.29      0.31       980\n",
      "weighted avg       0.56      0.58      0.56       980\n",
      "\n",
      "Cohen Kappa Score: 0.34880288540636717\n",
      "\n",
      "\n",
      "done \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\abhij\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "selected_model = make_pipeline( MinMaxScaler(),KNeighborsClassifier() ) \n",
    "selected_model.fit(X_train, y_train)\n",
    "predictions = selected_model.predict(X_test)\n",
    "print(\" ++++ Detailed classification report for the selected model ++++ \" )\n",
    "print(\"Algorithm: %s \" % modelname)\n",
    "#\n",
    "predictions = selected_model.predict(X_test)\n",
    "cm = confusion_matrix(y_test, predictions)\n",
    "df_cm = pandas.DataFrame(cm, columns=np.unique(y_test), index = np.unique(y_test))\n",
    "sns.heatmap(df_cm, square=True, annot=True, fmt='d', cbar=False )\n",
    "pl.xlabel('predicted')\n",
    "pl.ylabel('true')\n",
    "pl.show()\n",
    "#\n",
    "print('\\n clasification report:\\n', classification_report(y_test, predictions))\n",
    "print('Cohen Kappa Score:', cohen_kappa_score(y_test, predictions))\n",
    "print('\\n')        \n",
    "print('done \\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unlike-leonard",
   "metadata": {},
   "source": [
    "#### I have used the scaled KNN classifier instead of the neural network. We have seen, after appyling two models (MLP classifier and scaled KNN classifier) on tha data, we got better result for scaled KNN classifier. The score for the scaled KNN is 0.58 which is higher than the other. So we may incur that for a tabular data, we should not use network architecture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sealed-second",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}