Nepal-Earthquake-predictor-XGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Welcome to my Richter's Predictor Nepal Earthquake Damage Predictor XGBoost Model!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default code from Kaggle Notebook:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "   \n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename)) \n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Displaying some important libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import tensorflow as tf\n",
    "# print(\"Tensorflow:\", tf.__version__)\n",
    "\n",
    "# import kerastuner as kt\n",
    "# print(\"kerastuner:\", kt.__version__)\n",
    "\n",
    "# import keras_tuner as kt2\n",
    "# print(\"keras_tuner:\", kt2.__version__)\n",
    "\n",
    "# import platform\n",
    "# print(\"Python:\", platform.python_version())\n",
    "\n",
    "# import numpy as np\n",
    "# print(\"numpy:\", np.__version__)\n",
    "\n",
    "# import pandas as pd\n",
    "# print(\"pandas:\", pd.__version__)\n",
    "\n",
    "# import sklearn\n",
    "# print(\"sklearn version:\", sklearn.__version__)\n",
    "\n",
    "# import sklearn\n",
    "# print(\"sklearn path:\", sklearn.__path__)\n",
    "\n",
    "# import matplotlib\n",
    "# print(\"matplotlib:\", matplotlib.__version__)\n",
    "\n",
    "# import seaborn as sns\n",
    "# print(\"seaborn:\", sns.__version__)\n",
    "\n",
    "# # WARNING:tensorflow:From c:\\Users\\Micha\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
    "\n",
    "# # Tensorflow: 2.15.0\n",
    "# # C:\\Users\\Micha\\AppData\\Local\\Temp\\ipykernel_6936\\1753711907.py:4: DeprecationWarning: `import kerastuner` is deprecated, please use `import keras_tuner`.\n",
    "# #   import kerastuner as kt\n",
    "# # kerastuner: 1.0.5\n",
    "# # keras_tuner: 1.3.5\n",
    "# # Python: 3.10.11\n",
    "# # numpy: 1.24.3\n",
    "# # pandas: 2.1.4\n",
    "# # sklearn version: 1.2.2\n",
    "# # sklearn path: ['c:\\\\Users\\\\Micha\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python310\\\\lib\\\\site-packages\\\\sklearn']\n",
    "# # matplotlib: 3.8.2\n",
    "# # seaborn: 0.13.0\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Global random seed to make sure we can replicate any model that we create (no randomness)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-28 22:11:41.906175: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-01-28 22:11:41.935259: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-01-28 22:11:41.935290: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-01-28 22:11:41.936700: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-01-28 22:11:41.943162: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-01-28 22:11:42.519726: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "\n",
    "\n",
    "np.random.seed(42)\n",
    "random.seed(42)\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "os.environ['TF_DETERMINISTIC_OPS'] = '1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the training and testing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train_values are the features (X), and train_labels is the target/label (Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = pd.read_csv(\"train_values.csv\")\n",
    "train_Y = pd.read_csv(\"train_labels.csv\")\n",
    "\n",
    "test_values = pd.read_csv(\"test_values.csv\")\n",
    "\n",
    "# print(\"train labels:\\n\", train_Y.head())\n",
    "\n",
    "# print(\"train values:\\n\", train_X.head())\n",
    "      \n",
    "# print(\"test_values:\\n\", test_values.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I want to find out which features to use since there are so many. Here are some common data science techniques:\n",
    "\n",
    "1. **Correlation Matrix with Heatmap**: Correlation states how the features are related to each other or the target variable. You can use a heatmap to visualize the correlation matrix.\n",
    "\n",
    "2. **Univariate Selection**: Statistical tests can be used to select those features that have the strongest relationship with the output variable. The scikit-learn library provides the `SelectKBest` class that can be used with a suite of different statistical tests to select a specific number of features.\n",
    "\n",
    "3. **Recursive Feature Elimination (RFE)**: RFE is a popular feature selection method that fits a model and removes the weakest feature (or features) until the specified number of features is reached.\n",
    "\n",
    "4. **Feature Importance**: You can get the feature importance of each feature of your dataset by using the feature importance property of the model. For example, Decision Trees models in the scikit-learn library offer an importance property that can be accessed directly.\n",
    "\n",
    "For categorical features, you can convert them into numerical values using techniques like One-Hot Encoding or Label Encoding before applying these feature selection techniques. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First, I will try RFE (Recursive Feature Elimination)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.feature_selection import RFE\n",
    "# from lightgbm import LGBMRegressor #Lightgbm is a great gradient boosting model for large amount of data\n",
    "\n",
    "# # Assuming X is your feature set and y is the target value\n",
    "# X = train_X.drop('building_id', axis=1)\n",
    "# X = pd.get_dummies(X)\n",
    "\n",
    "# y = train_Y.drop('building_id', axis=1)\n",
    "# y = np.ravel(y) # converting dataframe to a one-dimensional array using the ravel function from numpy\n",
    "\n",
    "# estimator = LGBMRegressor(verbose = 0, random_state = 42)  # It's best to find the best model for you\n",
    "# selector = RFE(estimator, step=1)\n",
    "# selector = selector.fit(X, y)\n",
    "\n",
    "# # Assuming 'X' is your DataFrame with the feature data\n",
    "# feature_names = X.columns\n",
    "\n",
    "# # Map the feature names to the support array, which tells you which features were selected\n",
    "# support_dict = dict(zip(feature_names, selector.support_))\n",
    "\n",
    "# # Get the selected features\n",
    "# selected_features = [feature for feature, support in support_dict.items() if support]\n",
    "\n",
    "# # Print the selected features\n",
    "# print(\"Selected features:\\n\", selected_features)\n",
    "\n",
    "# # ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
    "# # 'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag', 'has_superstructure_cement_mortar_stone',\n",
    "# # 'has_superstructure_mud_mortar_brick', 'has_superstructure_cement_mortar_brick', 'has_superstructure_timber', 'has_superstructure_bamboo', \n",
    "# # 'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_superstructure_other', 'count_families', 'has_secondary_use', \n",
    "# # 'land_surface_condition_n', 'land_surface_condition_o', 'foundation_type_h', 'foundation_type_r', 'foundation_type_u', 'roof_type_n', \n",
    "# # 'roof_type_q', 'roof_type_x', 'ground_floor_type_f', 'ground_floor_type_v', 'ground_floor_type_x', 'other_floor_type_q', 'position_s',\n",
    "# # 'plan_configuration_u']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# # The feature ranking, such that ranking_[i] corresponds to the ranking position of the i-th feature. \n",
    "# # Selected features are assigned rank 1.\n",
    "# # Map the feature names to the ranking array\n",
    "# ranking_dict = dict(zip(feature_names, selector.ranking_))\n",
    "# print(ranking_dict)\n",
    "\n",
    "\n",
    "# # [ 1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  2 18 10 31\n",
    "# #  27 33 28 25 30  6  1  1  3  1  8  1  1  5  1  1  1  1 34  1  1 16  9  1\n",
    "# #  23  7  4 14  1 29 15 20 17 35 32 24 21 13 26  1 11 22 12 19]\n",
    "\n",
    "\n",
    "# # {'geo_level_1_id': 1, 'geo_level_2_id': 1, 'geo_level_3_id': 1, 'count_floors_pre_eq': 1, 'age': 1, 'area_percentage': 1, \n",
    "# #  'height_percentage': 1, 'has_superstructure_adobe_mud': 1, 'has_superstructure_mud_mortar_stone': 1, 'has_superstructure_stone_flag': 1, \n",
    "# #  'has_superstructure_cement_mortar_stone': 1, 'has_superstructure_mud_mortar_brick': 1, 'has_superstructure_cement_mortar_brick': 1, \n",
    "# #  'has_superstructure_timber': 1, 'has_superstructure_bamboo': 1, 'has_superstructure_rc_non_engineered': 1, 'has_superstructure_rc_engineered': 1, \n",
    "# #  'has_superstructure_other': 1, 'count_families': 1, 'has_secondary_use': 1, 'has_secondary_use_agriculture': 2, 'has_secondary_use_hotel': 18, \n",
    "# #  'has_secondary_use_rental': 10, 'has_secondary_use_institution': 31, 'has_secondary_use_school': 27, 'has_secondary_use_industry': 33, \n",
    "# #  'has_secondary_use_health_post': 28, 'has_secondary_use_gov_office': 25, 'has_secondary_use_use_police': 30, 'has_secondary_use_other': 6, \n",
    "# #  'land_surface_condition_n': 1, 'land_surface_condition_o': 1, 'land_surface_condition_t': 3, 'foundation_type_h': 1, 'foundation_type_i': 8, \n",
    "# #  'foundation_type_r': 1, 'foundation_type_u': 1, 'foundation_type_w': 5, 'roof_type_n': 1, 'roof_type_q': 1, 'roof_type_x': 1, \n",
    "# #  'ground_floor_type_f': 1, 'ground_floor_type_m': 34, 'ground_floor_type_v': 1, 'ground_floor_type_x': 1, 'ground_floor_type_z': 16, \n",
    "# #  'other_floor_type_j': 9, 'other_floor_type_q': 1, 'other_floor_type_s': 23, 'other_floor_type_x': 7, 'position_j': 4, 'position_o': 14, \n",
    "# #  'position_s': 1, 'position_t': 29, 'plan_configuration_a': 15, 'plan_configuration_c': 20, 'plan_configuration_d': 17, 'plan_configuration_f': 35, \n",
    "# #  'plan_configuration_m': 32, 'plan_configuration_n': 24, 'plan_configuration_o': 21, 'plan_configuration_q': 13, 'plan_configuration_s': 26, \n",
    "# #  'plan_configuration_u': 1, 'legal_ownership_status_a': 11, 'legal_ownership_status_r': 22, 'legal_ownership_status_v': 12, \n",
    "# #  'legal_ownership_status_w': 19}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\n",
      "New features:\n",
      " ['land_surface_condition_n', 'land_surface_condition_o', 'foundation_type_h', 'foundation_type_r', 'foundation_type_u', 'roof_type_n', 'roof_type_q', 'roof_type_x', 'ground_floor_type_f', 'ground_floor_type_v', 'ground_floor_type_x', 'other_floor_type_q', 'position_s', 'plan_configuration_u']\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "features = ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
    "'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag', 'has_superstructure_cement_mortar_stone',\n",
    "'has_superstructure_mud_mortar_brick', 'has_superstructure_cement_mortar_brick', 'has_superstructure_timber', 'has_superstructure_bamboo', \n",
    "'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_superstructure_other', 'count_families', 'has_secondary_use', \n",
    "'land_surface_condition_n', 'land_surface_condition_o', 'foundation_type_h', 'foundation_type_r', 'foundation_type_u', 'roof_type_n', \n",
    "'roof_type_q', 'roof_type_x', 'ground_floor_type_f', 'ground_floor_type_v', 'ground_floor_type_x', 'other_floor_type_q', 'position_s',\n",
    "'plan_configuration_u']\n",
    "\n",
    "print(len(features))\n",
    "\n",
    "\n",
    "# Find out which features are created through one-hot-encoding\n",
    "import pandas as pd\n",
    "\n",
    "# Load the original data\n",
    "original_data = train_X # full original training data\n",
    "\n",
    "# Get the original feature names\n",
    "original_features = original_data.columns\n",
    "\n",
    "# Check which features are not in the original data\n",
    "new_features = [feature for feature in features if feature not in original_features]\n",
    "\n",
    "# Print the new features\n",
    "print(\"New features:\\n\", new_features)\n",
    "\n",
    "\n",
    "#Manually remove the one-hot-encoding that pd.get_dummies() used on categorial \n",
    "features_before_dummies = ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'count_floors_pre_eq', 'age', 'area_percentage', 'height_percentage',\n",
    "'has_superstructure_adobe_mud', 'has_superstructure_mud_mortar_stone', 'has_superstructure_stone_flag', 'has_superstructure_cement_mortar_stone',\n",
    "'has_superstructure_mud_mortar_brick', 'has_superstructure_cement_mortar_brick', 'has_superstructure_timber', 'has_superstructure_bamboo', \n",
    "'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_superstructure_other', 'count_families', 'has_secondary_use', \n",
    "'land_surface_condition', 'foundation_type', 'roof_type', 'ground_floor_type','other_floor_type', 'position','plan_configuration']\n",
    "\n",
    "print(len(features_before_dummies))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Analysis from ChatGPT-4 (second way to find best features):\n",
    "\n",
    "The categorical variables in the features dataset have been successfully encoded. Now, let's look at the correlation of these features with the `damage_grade`:\n",
    "\n",
    "#### Correlation with `damage_grade`\n",
    "The correlation values range between -1 and 1. A value closer to 1 indicates a strong positive correlation, meaning that as the feature increases, the `damage_grade` tends to increase. Conversely, a value closer to -1 indicates a strong negative correlation, where an increase in the feature leads to a decrease in `damage_grade`. Values around 0 imply weak or no linear correlation.\n",
    "\n",
    "#### Top Positively Correlated Features:\n",
    "- `has_superstructure_mud_mortar_stone`\n",
    "- `count_floors_pre_eq`\n",
    "- Other features like `legal_ownership_status`, `has_superstructure_stone_flag`, etc., also show positive correlation but to a lesser extent.\n",
    "\n",
    "#### Top Negatively Correlated Features:\n",
    "- `has_superstructure_cement_mortar_brick`\n",
    "- `ground_floor_type`\n",
    "- `has_superstructure_rc_engineered`\n",
    "- Other features like `roof_type`, `has_superstructure_rc_non_engineered`, etc., also show negative correlation.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are the selected features based on the correlation threshold (of 0.05):\n",
    "\n",
    "1. `has_superstructure_mud_mortar_stone`\n",
    "2. `count_floors_pre_eq`\n",
    "3. `legal_ownership_status`\n",
    "4. `has_superstructure_stone_flag`\n",
    "5. `count_families`\n",
    "6. `has_superstructure_adobe_mud`\n",
    "7. `position`\n",
    "8. `has_superstructure_cement_mortar_stone`\n",
    "9. `has_superstructure_bamboo`\n",
    "10. `has_superstructure_timber`\n",
    "11. `geo_level_1_id`\n",
    "12. `has_secondary_use`\n",
    "13. `has_secondary_use_rental`\n",
    "14. `has_secondary_use_hotel`\n",
    "15. `foundation_type`\n",
    "16. `area_percentage`\n",
    "17. `has_superstructure_rc_non_engineered`\n",
    "18. `roof_type`\n",
    "19. `has_superstructure_rc_engineered`\n",
    "20. `ground_floor_type`\n",
    "21. `has_superstructure_cement_mortar_brick`\n",
    "\n",
    "These features were chosen because they have a correlation with the target variable `damage_grade` greater than the specified threshold of 0.05 (in absolute value). You can use these features for building your predictive model. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "features2 = [\n",
    "    \"has_superstructure_mud_mortar_stone\",\n",
    "    \"count_floors_pre_eq\",\n",
    "    \"legal_ownership_status\",\n",
    "    \"has_superstructure_stone_flag\",\n",
    "    \"count_families\",\n",
    "    \"has_superstructure_adobe_mud\",\n",
    "    \"position\",\n",
    "    \"has_superstructure_cement_mortar_stone\",\n",
    "    \"has_superstructure_bamboo\",\n",
    "    \"has_superstructure_timber\",\n",
    "    \"geo_level_1_id\",\n",
    "    \"has_secondary_use\",\n",
    "    \"has_secondary_use_rental\",\n",
    "    \"has_secondary_use_hotel\",\n",
    "    \"foundation_type\",\n",
    "    \"area_percentage\",\n",
    "    \"has_superstructure_rc_non_engineered\",\n",
    "    \"roof_type\",\n",
    "    \"has_superstructure_rc_engineered\",\n",
    "    \"ground_floor_type\",\n",
    "    \"has_superstructure_cement_mortar_brick\"\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Third, I will try SelectKBest to find best features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "\n",
    "# # Create the SelectKBest with the f_classif function. You can set the parameter \"k\" equal to a number if you want to limit the amount of features\n",
    "# selector = SelectKBest(f_classif, k = 20) # Getting the 20 best features\n",
    "\n",
    "# # Assuming X is your feature set and y is the target value\n",
    "# X = train_X.drop('building_id', axis=1)\n",
    "# X = pd.get_dummies(X)\n",
    "\n",
    "# y = train_Y.drop('building_id', axis=1)\n",
    "# y = np.ravel(y) # converting dataframe to a one-dimensional array using the ravel function from numpy\n",
    "\n",
    "\n",
    "# # Fit the selector to the data\n",
    "# selector.fit(X, y)\n",
    "\n",
    "# # Get the boolean mask of the selected features\n",
    "# mask = selector.get_support()\n",
    "\n",
    "# # Get the names of the selected features\n",
    "# selected_features = X.columns[mask]\n",
    "\n",
    "# # Convert the Index object to a list\n",
    "# features3 = selected_features.tolist()\n",
    "\n",
    "# print(features3)\n",
    "\n",
    "# # ['geo_level_1_id', 'count_floors_pre_eq', 'area_percentage', 'has_superstructure_mud_mortar_stone', 'has_superstructure_cement_mortar_brick', \n",
    "# #  'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_secondary_use_hotel', 'has_secondary_use_rental', \n",
    "# #  'foundation_type_i', 'foundation_type_r', 'foundation_type_u', 'foundation_type_w', 'roof_type_n', 'roof_type_x', 'ground_floor_type_f', \n",
    "# #  'ground_floor_type_v', 'other_floor_type_j', 'other_floor_type_q', 'other_floor_type_s']\n",
    "\n",
    "\n",
    "# # Get the scores\n",
    "# scores = selector.scores_\n",
    "\n",
    "# # Create a DataFrame with the scores\n",
    "# features_scores = pd.DataFrame({'Feature': X.columns, 'Score': scores})\n",
    "\n",
    "# # Sort the DataFrame by score in descending order\n",
    "# features_scores = features_scores.sort_values(by='Score', ascending=False)\n",
    "\n",
    "# # print the best 20 features\n",
    "# print(features_scores[0:19])\n",
    "\n",
    "# #                                    Feature         Score\n",
    "# # 35                       foundation_type_r  23787.275036\n",
    "# # 43                     ground_floor_type_v  20782.933584\n",
    "# # 40                             roof_type_x  16891.038184\n",
    "# # 8      has_superstructure_mud_mortar_stone  16490.386507\n",
    "# # 34                       foundation_type_i  16385.772905\n",
    "# # 12  has_superstructure_cement_mortar_brick  11120.193268\n",
    "# # 48                      other_floor_type_s  10507.484572\n",
    "# # 41                     ground_floor_type_f  10151.525359\n",
    "# # 16        has_superstructure_rc_engineered   7757.593854\n",
    "# # 47                      other_floor_type_q   7378.599061\n",
    "# # 15    has_superstructure_rc_non_engineered   4721.916051\n",
    "# # 37                       foundation_type_w   4568.674306\n",
    "# # 46                      other_floor_type_j   4533.708398\n",
    "# # 36                       foundation_type_u   2972.409108\n",
    "# # 0                           geo_level_1_id   2657.791274\n",
    "# # 3                      count_floors_pre_eq   2544.836052\n",
    "# # 5                          area_percentage   2529.046730\n",
    "# # 38                             roof_type_n   1776.396178\n",
    "# # 21                 has_secondary_use_hotel   1537.672773\n",
    "\n",
    "# # ...\n",
    "\n",
    "# # 22                has_secondary_use_rental   1342.099336\n",
    "# # 64                legal_ownership_status_a   1166.606551\n",
    "# # 19                       has_secondary_use    841.802928\n",
    "# # 39                             roof_type_q    761.885856\n",
    "# # 7             has_superstructure_adobe_mud    739.412821\n",
    "# # 13               has_superstructure_timber    659.199014\n",
    "# # 9            has_superstructure_stone_flag    576.438023\n",
    "# # 14               has_superstructure_bamboo    538.551492\n",
    "# # 66                legal_ownership_status_v    536.308634\n",
    "# # 11     has_superstructure_mud_mortar_brick    531.784659\n",
    "# # 63                    plan_configuration_u    515.087147\n",
    "# # 10  has_superstructure_cement_mortar_stone    478.844199\n",
    "# # 18                          count_families    476.562914\n",
    "# # 56                    plan_configuration_d    378.234531\n",
    "# # 53                              position_t    373.594539\n",
    "# # 6                        height_percentage    370.173817\n",
    "# # 20           has_secondary_use_agriculture    289.462856\n",
    "# # 1                           geo_level_2_id    264.447807\n",
    "# # 49                      other_floor_type_x    244.432657\n",
    "# # 4                                      age    219.626253\n",
    "# # 33                       foundation_type_h    209.425818\n",
    "# # 32                land_surface_condition_t    201.698101\n",
    "# # 30                land_surface_condition_n    182.152148\n",
    "# # 61                    plan_configuration_q    165.370472\n",
    "# # 23           has_secondary_use_institution    146.731486\n",
    "# # 17                has_superstructure_other    142.014204\n",
    "# # 50                              position_j    136.783490\n",
    "# # 67                legal_ownership_status_w    116.039402\n",
    "# # 52                              position_s    110.038831\n",
    "# # 55                    plan_configuration_c     75.423022\n",
    "# # 51                              position_o     75.104323\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "features3 = ['geo_level_1_id', 'count_floors_pre_eq', 'area_percentage', 'has_superstructure_mud_mortar_stone', 'has_superstructure_cement_mortar_brick', \n",
    " 'has_superstructure_rc_non_engineered', 'has_superstructure_rc_engineered', 'has_secondary_use_hotel', 'has_secondary_use_rental', \n",
    " 'foundation_type_i', 'foundation_type_r', 'foundation_type_u', 'foundation_type_w', 'roof_type_n', 'roof_type_x', 'ground_floor_type_f', \n",
    " 'ground_floor_type_v', 'other_floor_type_j', 'other_floor_type_q', 'other_floor_type_s']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check to see if there are any missing values in the data. If so, we have to do imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values in train_X: 0\n",
      "Number of missing values in train_Y: 0\n",
      "Number of missing values in test_values: 0\n"
     ]
    }
   ],
   "source": [
    "missing_train_X = train_X.isnull().sum().sum()\n",
    "print(\"Number of missing values in train_X:\", missing_train_X)\n",
    "\n",
    "missing_train_Y = train_Y.isnull().sum().sum()\n",
    "print(\"Number of missing values in train_Y:\", missing_train_Y)\n",
    "\n",
    "missing_test_values = test_values.isnull().sum().sum()\n",
    "print(\"Number of missing values in test_values:\", missing_test_values )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we have 0 missing values in each dataframe, we don't have to do imputation!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now, we have 3 different list of features (features, features2, and features3), which I found using RFE. Data Analysis ChatGPT-4, and SelectKBest respectively\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now, I have to turn one-hot-encode the data using pd.get_dummies, and I'll be creating 3 seperate train_X, one for each list of possible best features. And also on the test_data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. testX1 for the RFE features\n",
    "# Since the features from RFE are the one-hot-encoded features, we have to apply features after doing pd.get_dummies()\n",
    "\n",
    "trainX1 = pd.get_dummies(train_X)\n",
    "trainX1 = trainX1[features]\n",
    "\n",
    "\n",
    "# 2. testX2 for the Data Analysis ChatGPT-4\n",
    "# Since the features from Data Analysis are from the original feature set, we have to apply the features before doing pd.get_dummies()\n",
    "\n",
    "trainX2 = train_X[features2]\n",
    "trainX2 = pd.get_dummies(trainX2)\n",
    "\n",
    "\n",
    "\n",
    "# 3. testX3 for the SelectKBest\n",
    "# Since the features from RFE are the one-hot-encoded features, we have to apply features after doing pd.get_dummies()\n",
    "\n",
    "trainX3 = pd.get_dummies(train_X)\n",
    "trainX3 = trainX3[features3]\n",
    "\n",
    "\n",
    "\n",
    "# 4. Do pd.get_dummies() on test data. I will create a seperate test_data for each feature selection, since each test_data needs to have a certain set of features\n",
    "\n",
    "test_data1 = pd.get_dummies(test_values)\n",
    "test_data1 = test_data1[features + ['building_id'] ]\n",
    "\n",
    "test_data2 = test_values[features2 + ['building_id'] ]\n",
    "test_data2 = pd.get_dummies(test_data2)\n",
    "\n",
    "test_data3 = pd.get_dummies(test_values)\n",
    "test_data3 = test_data3[features3 + ['building_id'] ]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### It's important to do pd.get_dummies() before doing the train_valid_test split. Now we can do the split\n",
    "I have to do train_valid_test split three times, one for each different train_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 1. train_valid_test split for train_X1\n",
    "train_X1, test_X1, train_Y1, test_Y1 = train_test_split(trainX1, train_Y, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "\n",
    "valid_X1, test_X1, valid_Y1, test_Y1 = train_test_split(test_X1, test_Y1, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)\n",
    "\n",
    "\n",
    "# 2. train_valid_test split for train_X2\n",
    "train_X2, test_X2, train_Y2, test_Y2 = train_test_split(trainX2, train_Y, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "\n",
    "valid_X2, test_X2, valid_Y2, test_Y2 = train_test_split(test_X2, test_Y2, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)\n",
    "\n",
    "\n",
    "# 3. train_valid_test split for train_X3\n",
    "train_X3, test_X3, train_Y3, test_Y3 = train_test_split(trainX3, train_Y, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "\n",
    "valid_X3, test_X3, valid_Y3, test_Y3 = train_test_split(test_X3, test_Y3, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now, we can normalize the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Creating the Normalizing Scaler**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many different Normalization Scalers:\n",
    "\n",
    "1. **MinMaxScaler**: This scaler scales and translates each feature individually such that it is in the given range on the training set, e.g., between zero and one.\n",
    "\n",
    "\n",
    "2. **StandardScaler**: This scaler standardizes features by removing the mean and scaling to unit variance.\n",
    "\n",
    "\n",
    "3. **RobustScaler**: This scaler scales features using statistics that are robust to outliers. It uses the Interquartile Range (IQR) to scale the data, making it a better choice for when the data has outliers.\n",
    "\n",
    "\n",
    "4. **Normalizer**: This scaler scales individual samples to have unit norm. This scaler works on the rows, not the columns!\n",
    "\n",
    "\n",
    "5. **MaxAbsScaler**: This scaler scales and translates each feature individually such that the maximal absolute value of each feature in the training set will be 1.0. It does not shift/center the data, and thus does not destroy any sparsity.\n",
    "\n",
    "\n",
    "Remember, the choice of scaler can depend on your specific dataset and the machine learning algorithm that you're using."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I've actually decided to not do normalization since almost all the columns are categorial columns, and the non-categorial columns are mostly normalized already in the dataset, so there's no need"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I've actually decided to not do normalization since almost all the columns are categorial columns, and the non-categorial columns are mostly normalized already in the dataset, so there's no need"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Later on, I decided I want to try an XGBoost model with all the features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values in train_X: 0\n",
      "Number of missing values in train_Y: 0\n",
      "Number of missing values in test_values: 0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "train_X = train_X # full training features\n",
    "train_Y = train_Y # full training labels\n",
    "test_values = test_values # full test values\n",
    "\n",
    "# See if there are any missing values. There are none, luckily.\n",
    "missing_train_X = train_X.isnull().sum().sum()\n",
    "print(\"Number of missing values in train_X:\", missing_train_X)\n",
    "\n",
    "missing_train_Y = train_Y.isnull().sum().sum()\n",
    "print(\"Number of missing values in train_Y:\", missing_train_Y)\n",
    "\n",
    "missing_test_values = test_values.isnull().sum().sum()\n",
    "print(\"Number of missing values in test_values:\", missing_test_values )\n",
    "\n",
    "# One hot encoding\n",
    "train_X4 = pd.get_dummies(train_X).drop('building_id', axis=1)\n",
    "train_Y4 = pd.get_dummies(train_Y)\n",
    "test_data4 = pd.get_dummies(test_values)\n",
    "\n",
    "\n",
    "\n",
    "# Make sure to do train_test_validaation split before normalization, to avoid data leakage\n",
    "train_X4, test_X4, train_Y4, test_Y4 = train_test_split(train_X4, train_Y4, test_size=0.3, random_state = 42) # split into training (70%) and a test set (30%)\n",
    "valid_X4, test_X4, valid_Y4, test_Y4 = train_test_split(test_X4, test_Y4, test_size = 0.5, random_state = 42) # split test set into a validation (15%) and test set (15%)\n",
    "\n",
    "# Save column names\n",
    "column_names = train_X4.columns\n",
    "\n",
    "# Normalize data\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(train_X4)\n",
    "\n",
    "train_X4 = scaler.transform(train_X4) # FYI, this turns the data from a dataframe to a numpy array\n",
    "valid_X4 = scaler.transform(valid_X4)  # FYI, this turns the data from a dataframe to a numpy array\n",
    "test_X4 = scaler.transform(test_X4)  # FYI, this turns the data from a dataframe to a numpy array\n",
    "\n",
    "# Change back from numpy array to dataframe\n",
    "train_X4 = pd.DataFrame(train_X4, columns = column_names)\n",
    "valid_X4 = pd.DataFrame(valid_X4, columns = column_names)\n",
    "test_X4 = pd.DataFrame(test_X4, columns = column_names)\n",
    "\n",
    "test_data4 = pd.DataFrame(test_data4)\n",
    "\n",
    "# Change everything to a float\n",
    "\n",
    "train_X4 = train_X4.astype('float32')\n",
    "valid_X4 = valid_X4.astype('float32')\n",
    "test_X4 = test_X4.astype('float32')\n",
    "\n",
    "train_Y4 = train_Y4.astype('float32')\n",
    "valid_Y4 = valid_Y4.astype('float32')\n",
    "test_Y4 = test_Y4.astype('float32')\n",
    "\n",
    "test_data4 = test_data4.astype('float32')\n",
    "\n",
    "\n",
    "#print(valid_Y4)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Model 1 for XGBoost (XGBClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and testt sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'n_estimators': trial.suggest_int('n_estimators', 100, 600), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 3, 9), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.4, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.5, 1.0), # Default is 1\n",
    "#         #'colsample_bytree': trial.suggest_float('colsample_bytree', 0.4, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-5, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 2, log=True),  # Default is 1\n",
    "#     }\n",
    "\n",
    "#     model = xgb.XGBClassifier(**params)\n",
    "#     cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n",
    "#     score = cross_val_score(model, train_X1_full, train_Y1_temp_full, cv=cv, scoring='f1_micro', n_jobs=-1)\n",
    "#     f1 = score.mean()\n",
    "#     return f1       \n",
    "\n",
    "\n",
    "# study = optuna.create_study(direction='maximize')\n",
    "# study.optimize(objective, n_trials=100) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# # TOOK 1 HOUR AND 19 MIN UTES for 100 trials\n",
    "\n",
    "# # The trials can be found on the \"Optuna_Logs_XGBoost_Model1.txt\" file!!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get some information about the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Import necessary libraries\n",
    "# from xgboost import XGBClassifier, plot_importance\n",
    "# import matplotlib.pyplot as plt\n",
    "# import json\n",
    "\n",
    "# # Get the hyperparameters of the model and print them\n",
    "# best_params = study.best_trial.params\n",
    "\n",
    "# # Add the static parameters\n",
    "# best_params.update({\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'mlogloss',\n",
    "#     'random_state': 42,\n",
    "# })\n",
    "\n",
    "# # Write the hyperparameters to a new file\n",
    "# with open('hyperparameters_XGBoost_model1.json', 'w') as f:\n",
    "#     json.dump(best_params, f)\n",
    "\n",
    "# # Create the model with the best parameters\n",
    "# model1 = XGBClassifier(**best_params) # the ** operator is used to unpack a dictionary and pass it's key value pairs as keyword arguments to a function\n",
    "\n",
    "# # Fit the model to the full data\n",
    "# model1.fit(train_X1_full, train_Y1_temp_full)  # Make sure to change the name to match whatever model you are using\n",
    "\n",
    "# # Plot bar plot of the feature importances to visualize the model\n",
    "# plt.figure(figsize=(80, 20), dpi=200)\n",
    "\n",
    "# # Plot the feature importance\n",
    "# ax = plot_importance(model1)\n",
    "\n",
    "# # Decrease the font size of the y-axis labels since there are two many features\n",
    "# for label in ax.get_yticklabels():\n",
    "#     label.set_size(8)\n",
    "\n",
    "# # Adjust the layout\n",
    "# plt.tight_layout()\n",
    "\n",
    "# # Save the plot\n",
    "# plt.savefig('feature_importance_XGBoost_model1.png')\n",
    "\n",
    "# # Show the plot\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submitting Model 1. Score: 0.7472!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model1.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model1, 'saved_XGBoost_model1.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model1 = load('model1.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model1.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_1.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.72"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # [I 2024-01-10 00:04:59,965] Trial 82 finished with value: 0.7441874743381645 and parameters: {'n_estimators': 566, 'max_depth': 9, 'learning_rate': 0.06990259781853493, 'subsample': 0.8920015569105262, 'reg_alpha': 0.13284341131896668, 'reg_lambda': 1.6799006741969436e-05}. Best is trial 82 with value: 0.7441874743381645.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model2 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 82\n",
    "#     n_estimators=566,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.06990259781853493,\n",
    "#     subsample=0.8920015569105262,\n",
    "#     reg_alpha=0.13284341131896668,\n",
    "#     reg_lambda=1.6799006741969436e-05,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model2.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model2, 'saved_XGBoost_model2.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model2 = load('model2.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model2.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_2.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 3, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # # [I 2024-01-10 00:02:07,361] Trial 79 finished with value: 0.7441491014999942 and parameters: {'n_estimators': 571, 'max_depth': 9, 'learning_rate': 0.07411020030631262, 'subsample': 0.9084524981347166, 'reg_alpha': 1.5596644878161967e-05, 'reg_lambda': 1.0216930414411583e-05}. Best is trial 79 with value: 0.7441491014999942.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model3 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 79\n",
    "#     n_estimators=571,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.07411020030631262,\n",
    "#     subsample=0.9084524981347166,\n",
    "#     reg_alpha=1.5596644878161967e-05,\n",
    "#     reg_lambda=1.0216930414411583e-05,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model3.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model3, 'saved_XGBoost_model3.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model3 = load('model3.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model3.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_3.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 4, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.73"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-10 00:00:03,173] Trial 77 finished with value: 0.7441107286618239 and parameters: {'n_estimators': 586, 'max_depth': 9, 'learning_rate': 0.07372054199547849, 'subsample': 0.9084441669124462, 'reg_alpha': 4.5869925086802365e-05, 'reg_lambda': 1.2047816033954858e-05}. Best is trial 77 with value: 0.7441107286618239.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model4 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 77\n",
    "#     n_estimators=586,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.07372054199547849,\n",
    "#     subsample=0.9084441669124462,\n",
    "#     reg_alpha=4.5869925086802365e-05,\n",
    "#     reg_lambda=1.2047816033954858e-05,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model4.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model4, 'saved_XGBoost_model4.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model4 = load('model4.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model4.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_4.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 5, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.75!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-09 23:44:28,643] Trial 62 finished with value: 0.7440032847149474 and parameters: {'n_estimators': 543, 'max_depth': 9, 'learning_rate': 0.09107783861185915, 'subsample': 0.9231950980125518, 'reg_alpha': 1.0170523313529933e-05, 'reg_lambda': 0.00023219813720820273}. Best is trial 62 with value: 0.7440032847149474.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model5 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 62\n",
    "#     n_estimators=543,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model5.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model5, 'saved_XGBoost_model5.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model5 = load('model5.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model5.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_5.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 6, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-09 23:41:19,458] Trial 58 finished with value: 0.7437500239830239 and parameters: {'n_estimators': 553, 'max_depth': 9, 'learning_rate': 0.084354574047343, 'subsample': 0.877643098301532, 'reg_alpha': 1.0942230630211092e-05, 'reg_lambda': 0.0004413581362295261}. Best is trial 58 with value: 0.7437500239830239.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model6 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 58\n",
    "#     n_estimators=571,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.07411020030631262,\n",
    "#     subsample=0.9084524981347166,\n",
    "#     reg_alpha=1.5596644878161967e-05,\n",
    "#     reg_lambda=1.0216930414411583e-05,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model6.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model6, 'saved_XGBoost_model6.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model6 = load('model6.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model6.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_6.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 7, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.61"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-09 23:21:33,026] Trial 33 finished with value: 0.743596532630343 and parameters: {'n_estimators': 567, 'max_depth': 8, 'learning_rate': 0.10083473171825715, 'subsample': 0.8492514815691262, 'reg_alpha': 3.448161073426747e-05, 'reg_lambda': 0.0020333024383809638}. Best is trial 33 with value: 0.743596532630343.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model7 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 33\n",
    "#     n_estimators=567,\n",
    "#     max_depth=8,\n",
    "#     learning_rate=0.10083473171825715,\n",
    "#     subsample=0.8492514815691262,\n",
    "#     reg_alpha=3.448161073426747e-05,\n",
    "#     reg_lambda=0.0020333024383809638,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model7.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model7, 'saved_XGBoost_model7.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model7 = load('model7.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model7.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_7.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 8, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-09 23:17:51,875] Trial 28 finished with value: 0.7435121123863685 and parameters: {'n_estimators': 503, 'max_depth': 9, 'learning_rate': 0.1003129145568426, 'subsample': 0.9339489199783957, 'reg_alpha': 7.287435099971498e-05, 'reg_lambda': 0.00041200226288994816}. Best is trial 28 with value: 0.7435121123863685.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model8 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 28\n",
    "#     n_estimators=571,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.07411020030631262,\n",
    "#     subsample=0.9084524981347166,\n",
    "#     reg_alpha=1.5596644878161967e-05,\n",
    "#     reg_lambda=1.0216930414411583e-05,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model8.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model8, 'saved_XGBoost_model8.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model8 = load('model8.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model8.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_8.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 9, trying out some of the other good hyperparameters from the Model 1 optuna hyperparameter tuning. Score: 74.58"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-09 23:01:23,154] Trial 1 finished with value: 0.7433893193042237 and parameters: {'n_estimators': 587, 'max_depth': 9, 'learning_rate': 0.05088597132699915, 'subsample': 0.825629146173626, 'reg_alpha': 0.0001624842611518334, 'reg_lambda': 0.00012216763298662962}. Best is trial 1 with value: 0.7433893193042237.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model9 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # Best parameters from trial 1\n",
    "#     n_estimators=587,\n",
    "#     max_depth=9,\n",
    "#     learning_rate=0.05088597132699915,\n",
    "#     subsample=0.825629146173626,\n",
    "#     reg_alpha=0.0001624842611518334,\n",
    "#     reg_lambda=0.00012216763298662962,\n",
    "# )\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model9.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model9, 'saved_XGBoost_model9.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model9 = load('model9.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model9.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_9.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Another Optuna Trial: Try testing out higher max_depth and some other hyper parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and testt sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         'n_estimators': 543, # Default is 100\n",
    "#         'learning_rate': 0.09107783861185915, # Default is 0.3\n",
    "#         'subsample': 0.9231950980125518, # Default is 1\n",
    "#         'reg_alpha': 1.0170523313529933e-05, # Default is 0\n",
    "#         'reg_lambda': 0.00023219813720820273,  # Default is 1\n",
    "\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20), # Default is 6\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1), # Default is 1\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     model = xgb.XGBClassifier(**params)\n",
    "#     cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n",
    "#     score = cross_val_score(model, train_X1_full, train_Y1_temp_full, cv=cv, scoring='f1_micro', n_jobs=-1)\n",
    "#     f1 = score.mean()\n",
    "#     return f1       \n",
    "\n",
    "\n",
    "# study = optuna.create_study(direction='maximize')\n",
    "# study.optimize(objective, n_trials=100) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# # Took 63 minutes and I had to stop it since CPU was over heating (90 - 100 celcius and anything above 80 is high). I got up to trial 71\n",
    "\n",
    "# # The trials can be found on the \"Optuna_Logs_XGBoost_Model10.txt\" file!!!\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 10, get best trial from Optuna trial. Score: 74.79"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 20:26:11,314] Trial 66 finished with value: 0.7456533167562672 and parameters: {'max_depth': 11, 'min_child_weight': 4, 'gamma': 0.2336383419099693, 'colsample_bytree': 0.7218730892946489}. Best is trial 66 with value: 0.7456533167562672.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model10 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 66 \n",
    "#     max_depth=11,\n",
    "#     min_child_weight=4,\n",
    "#     gamma=0.2336383419099693,\n",
    "#     colsample_bytree=0.7218730892946489,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model10.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model10, 'saved_XGBoost_model10.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model10 = load('model10.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model10.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_10.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 11, get other good trials from Optuna. Score: 74.61"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 19:29:16,904] Trial 0 finished with value: 0.7456840183248545 and parameters: {'max_depth': 13, 'min_child_weight': 9, 'gamma': 0.1318407354171367, 'colsample_bytree': 0.7163120700192489}. Best is trial 0 with value: 0.7456840183248545.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model11 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 0 (of the failed attempt, on the bottom of the \"Optuna_Logs_XGBoost_Model10.txt\" document)\n",
    "#     max_depth=13,\n",
    "#     min_child_weight=9,\n",
    "#     gamma=0.1318407354171367,\n",
    "#     colsample_bytree=0.7163120700192489,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model11.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model11, 'saved_XGBoost_model11.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model11 = load('model11.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model11.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_11.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 12, get other good trials from Optuna. Score: 74.73"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 20:11:00,692] Trial 48 finished with value: 0.745607269350463 and parameters: {'max_depth': 10, 'min_child_weight': 5, 'gamma': 0.2067431237467906, 'colsample_bytree': 0.7155459291820985}. Best is trial 48 with value: 0.745607269350463.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model12 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 48 \n",
    "#     max_depth=10,\n",
    "#     min_child_weight=5,\n",
    "#     gamma=0.2067431237467906,\n",
    "#     colsample_bytree=0.7155459291820985,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model12.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model12, 'saved_XGBoost_model12.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model12 = load('model12.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model12.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_12.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 13, get other good trials from Optuna. Score: 74.71"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 20:07:56,662] Trial 44 finished with value: 0.7451851681305904 and parameters: {'max_depth': 11, 'min_child_weight': 7, 'gamma': 0.21294180730420548, 'colsample_bytree': 0.75951021933263}. Best is trial 44 with value: 0.7451851681305904.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model13 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 44\n",
    "#     max_depth=11,\n",
    "#     min_child_weight=7,\n",
    "#     gamma=0.21294180730420548,\n",
    "#     colsample_bytree=0.75951021933263,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model13.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model13, 'saved_XGBoost_model13.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model13 = load('model13.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model13.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_13.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 14, get other good trials from Optuna. Score: 74.82! Highest score so far!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 19:39:15,441] Trial 8 finished with value: 0.745123771589518 and parameters: {'max_depth': 10, 'min_child_weight': 10, 'gamma': 0.2240455435057339, 'colsample_bytree': 0.7009330927233471}. Best is trial 8 with value: 0.745123771589518.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model14 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 8\n",
    "#     max_depth=10,\n",
    "#     min_child_weight=10,\n",
    "#     gamma=0.2240455435057339,\n",
    "#     colsample_bytree=0.7009330927233471,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model14.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model14, 'saved_XGBoost_model14.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model14 = load('model14.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model14.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_14.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 15, get other good trials from Optuna. Score: 74.66"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 19:35:20,905] Trial 4 finished with value: 0.744456084205356 and parameters: {'max_depth': 10, 'min_child_weight': 9, 'gamma': 0.1204892873574776, 'colsample_bytree': 0.983077445033478}. Best is trial 4 with value: 0.744456084205356.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model15 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 66 \n",
    "#     max_depth=10,\n",
    "#     min_child_weight=9,\n",
    "#     gamma=0.1204892873574776,\n",
    "#     colsample_bytree=0.983077445033478,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model15.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model15, 'saved_XGBoost_model15.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model15 = load('model15.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model15.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_15.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Model 16, try Optuna again but with CV = 5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and testt sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         'n_estimators': 543, # Default is 100\n",
    "#         'learning_rate': 0.09107783861185915, # Default is 0.3\n",
    "#         'subsample': 0.9231950980125518, # Default is 1\n",
    "#         'reg_alpha': 1.0170523313529933e-05, # Default is 0\n",
    "#         'reg_lambda': 0.00023219813720820273,  # Default is 1\n",
    "\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20), # Default is 6\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1), # Default is 1\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     model = xgb.XGBClassifier(**params)\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     score = cross_val_score(model, train_X1_full, train_Y1_temp_full, cv=cv, scoring='f1_micro', n_jobs=4)\n",
    "#     f1 = score.mean()\n",
    "#     return f1       \n",
    "\n",
    "\n",
    "# study = optuna.create_study(direction='maximize')\n",
    "# study.optimize(objective, n_trials=50) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "\n",
    "\n",
    "# # The trials can be found on the \"Optuna_Logs_XGBoost_Model10.txt\" file!!!\n",
    "\n",
    "# # c:\\Users\\Micha\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
    "# #   from .autonotebook import tqdm as notebook_tqdm\n",
    "# # [I 2024-01-10 20:26:47,987] A new study created in memory with name: no-name-d5cbd81d-861c-404e-983f-e113bbc1aeb2\n",
    "# # [I 2024-01-10 20:27:47,171] Trial 0 finished with value: 0.7466548438861265 and parameters: {'max_depth': 13, 'min_child_weight': 10, 'gamma': 0.377206847613114, 'colsample_bytree': 0.8339161855632286}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:29:00,378] Trial 1 finished with value: 0.745307960196463 and parameters: {'max_depth': 14, 'min_child_weight': 2, 'gamma': 0.3103015639136577, 'colsample_bytree': 0.9190031191129362}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:32:44,147] Trial 2 finished with value: 0.731175245325819 and parameters: {'max_depth': 20, 'min_child_weight': 5, 'gamma': 0.07344816628567508, 'colsample_bytree': 0.8828090345804489}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:33:55,996] Trial 3 finished with value: 0.7465665900689232 and parameters: {'max_depth': 18, 'min_child_weight': 9, 'gamma': 0.47786972934464705, 'colsample_bytree': 0.7733168439787258}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:35:12,473] Trial 4 finished with value: 0.7464860087137399 and parameters: {'max_depth': 13, 'min_child_weight': 6, 'gamma': 0.23071171657194672, 'colsample_bytree': 0.7679305598380898}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:36:07,701] Trial 5 finished with value: 0.743684785784563 and parameters: {'max_depth': 9, 'min_child_weight': 5, 'gamma': 0.47446899100728207, 'colsample_bytree': 0.9356788297232858}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:37:16,536] Trial 6 finished with value: 0.7466778701069328 and parameters: {'max_depth': 12, 'min_child_weight': 2, 'gamma': 0.30654299104889127, 'colsample_bytree': 0.7976341034830331}. Best is trial 6 with value: 0.7466778701069328.\n",
    "# # [I 2024-01-10 20:39:28,533] Trial 7 finished with value: 0.7385389979627515 and parameters: {'max_depth': 19, 'min_child_weight': 10, 'gamma': 0.10661878802572344, 'colsample_bytree': 0.7943430858990438}. Best is trial 6 with value: 0.7466778701069328.\n",
    "# # [I 2024-01-10 20:40:29,887] Trial 8 finished with value: 0.7455880800451008 and parameters: {'max_depth': 16, 'min_child_weight': 9, 'gamma': 0.33645442238736545, 'colsample_bytree': 0.9417725575171005}. Best is trial 6 with value: 0.7466778701069328.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 16. Score: 74.71"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-10 20:37:16,536] Trial 6 finished with value: 0.7466778701069328 and parameters: {'max_depth': 12, 'min_child_weight': 2, 'gamma': 0.30654299104889127, 'colsample_bytree': 0.7976341034830331}. Best is trial 6 with value: 0.7466778701069328.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model16 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 6\n",
    "#     max_depth=12,\n",
    "#     min_child_weight=2,\n",
    "#     gamma=0.30654299104889127,\n",
    "#     colsample_bytree=0.7976341034830331,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model16.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model16, 'saved_XGBoost_model16.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model16 = load('model16.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model16.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_16.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 17, another good trial from Optuna used for Model 16. Score: 74.73"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # # [I 2024-01-10 20:27:47,171] Trial 0 finished with value: 0.7466548438861265 and parameters: {'max_depth': 13, 'min_child_weight': 10, 'gamma': 0.377206847613114, 'colsample_bytree': 0.8339161855632286}. Best is trial 0 with value: 0.7466548438861265.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model17 = XGBClassifier(\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 0\n",
    "#     max_depth=13,\n",
    "#     min_child_weight=10,\n",
    "#     gamma=0.377206847613114,\n",
    "#     colsample_bytree=0.8339161855632286,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model17.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model17, 'saved_XGBoost_model17.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model17 = load('model17.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model17.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_17.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optuna trials with GPU Acceleration (on XGBoost) now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# import pandas as pd\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import MedianPruner\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and test sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm\n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         #'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         #'n_estimators': 543, # Default is 100 , not needed because of the num_boost_round hyperparameter in the cross validation method\n",
    "#         'learning_rate': 0.09107783861185915, # Default is 0.3\n",
    "#         'subsample': 0.9231950980125518, # Default is 1\n",
    "#         'reg_alpha': 1.0170523313529933e-05, # Default is 0\n",
    "#         'reg_lambda': 0.00023219813720820273,  # Default is 1\n",
    "\n",
    "\n",
    "#         # These below are the ones we are testing and tuning\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20), # Default is 6\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#         #'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1), # Default is 1\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     dtrain = xgb.DMatrix(train_X1_full, label=train_Y1_temp_full) # This is change the pandas data into something our GPU can handle better (DMatrix)\n",
    "\n",
    "#     cv_results = xgb.cv(\n",
    "#         params,\n",
    "#         dtrain,\n",
    "#         num_boost_round=543,\n",
    "#         nfold=5,\n",
    "#         metrics={'merror'}, # This is for multiclass classification error\n",
    "#         early_stopping_rounds=10\n",
    "#     )\n",
    "\n",
    "#      # We want to maximize accuracy, so we return the negative of the error rate\n",
    "#     return 1 - cv_results['test-merror-mean'].min()\n",
    "\n",
    "# #sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# sampler = TPESampler()\n",
    "\n",
    "# pruner = optuna.pruners.MedianPruner(n_warmup_steps=10, interval_steps=1) # n_warmup_steps=10 means that it won't think about pruning until after 10 steps, and interval_steps=1 means each trial is a step\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', pruner=pruner, sampler = sampler)\n",
    "\n",
    "\n",
    "# study.optimize(objective, n_trials=50) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# # Took 60 minutes but because of GPU acceleration, my CPU was not hot! And neither was my GPU! It's so good for my computer health"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submit Model 18, which uses Optuna with GPU accelerated XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-14 18:27:57,799] Trial 17 finished with value: 0.746693225853537 and parameters: {'max_depth': 10, 'min_child_weight': 7, 'gamma': 0.01778659172817425}. Best is trial 17 with value: 0.746693225853537.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model18 = XGBClassifier( \n",
    "#     #tree_method = 'hist', It seems like turning off GPU acceleration for predictions improved score, at least for this instance\n",
    "#     #device = 'cuda', It seems like turning off GPU acceleration for predictions improved score, at least for this instance\n",
    "\n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 17\n",
    "#     max_depth=10,\n",
    "#     min_child_weight=7,\n",
    "#     gamma=0.01778659172817425,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model18.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model18, 'saved_XGBoost_model18.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model18 = load('model18.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model18.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_18.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")\n",
    "\n",
    "# # Successfully Submitted!\n",
    "# # /home/michaelye22/.local/lib/python3.10/site-packages/xgboost/core.py:160: UserWarning: [19:19:49] WARNING: /workspace/src/common/error_msg.cc:58: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.\n",
    "# # Potential solutions:\n",
    "# # - Use a data structure that matches the device ordinal in the booster.\n",
    "# # - Set the device for booster before call to inplace_predict.\n",
    "\n",
    "# # This warning will only be shown once.\n",
    "\n",
    "# #   warnings.warn(smsg, UserWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-14 18:27:57,799] Trial 17 finished with value: 0.746693225853537 and parameters: {'max_depth': 10, 'min_child_weight': 7, 'gamma': 0.01778659172817425}. Best is trial 17 with value: 0.746693225853537.\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "# #import cudf\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# model18 = XGBClassifier( \n",
    "#     tree_method = 'hist',  # Use histogram-based algorithm\n",
    "#     device = 'cuda',  # Use cuda for GPU acceleration\n",
    "#     gpu_id = 0,\n",
    "   \n",
    "#     objective = 'multi:softmax',\n",
    "#     num_class = 3,\n",
    "#     eval_metric = 'mlogloss',\n",
    "#     random_state = 42,\n",
    "\n",
    "#     # From Model 5\n",
    "#     n_estimators=543,\n",
    "#     learning_rate=0.09107783861185915,\n",
    "#     subsample=0.9231950980125518,\n",
    "#     reg_alpha=1.0170523313529933e-05,\n",
    "#     reg_lambda=0.00023219813720820273,\n",
    "\n",
    "#     # From trial 17\n",
    "#     max_depth=10,\n",
    "#     min_child_weight=7,\n",
    "#     gamma=0.01778659172817425,\n",
    "\n",
    "\n",
    "# )\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dtrain = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# params = {\n",
    "#     'tree_method': 'hist',\n",
    "#     #'device' : 'cuda',\n",
    "#     'gpu_id': 0,\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'mlogloss',\n",
    "#     'random_state': 42,\n",
    "#     'n_estimators': 543,\n",
    "#     'learning_rate': 0.09107783861185915,\n",
    "#     'subsample': 0.9231950980125518,\n",
    "#     'reg_alpha': 1.0170523313529933e-05,\n",
    "#     'reg_lambda': 0.00023219813720820273,\n",
    "#     'max_depth': 10,\n",
    "#     'min_child_weight': 7,\n",
    "#     'gamma': 0.01778659172817425\n",
    "# }\n",
    "\n",
    "# # Specify the gpu_id parameter\n",
    "# #params = {'tree_method': 'gpu_hist', 'gpu_id': 0}\n",
    "\n",
    "\n",
    "# # Now you can fit the model on the GPU data\n",
    "# model18 = xgb.train(params, dtrain)\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# #model18.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model18, 'saved_XGBoost_model18.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model18 = load('model18.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dtest = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model18.predict(dtest)\n",
    "\n",
    "# competition_y_pred = competition_y_pred.astype(int)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_18.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")\n",
    "\n",
    "# # Successfully Submitted!\n",
    "# # /home/michaelye22/.local/lib/python3.10/site-packages/xgboost/core.py:160: UserWarning: [19:19:49] WARNING: /workspace/src/common/error_msg.cc:58: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.\n",
    "# # Potential solutions:\n",
    "# # - Use a data structure that matches the device ordinal in the booster.\n",
    "# # - Set the device for booster before call to inplace_predict.\n",
    "\n",
    "# # This warning will only be shown once.\n",
    "\n",
    "# #   warnings.warn(smsg, UserWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and testt sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "\n",
    "#         'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         'n_estimators': 543, # Default is 100\n",
    "#         'learning_rate': 0.09107783861185915, # Default is 0.3\n",
    "#         'subsample': 0.9231950980125518, # Default is 1\n",
    "#         'reg_alpha': 1.0170523313529933e-05, # Default is 0\n",
    "#         'reg_lambda': 0.00023219813720820273,  # Default is 1\n",
    "\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20), # Default is 6\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#         #'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1), # Default is 1\n",
    "        \n",
    "#     }\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "#     model = xgb.XGBClassifier(tree_method='hist', # This is for GPU Acceleration\n",
    "#                               device = 'cuda', # This is for GPU Acceleration\n",
    "#                               **params # This is for the hyper parameters above\n",
    "#                               )\n",
    "\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     score = cross_val_score(model, train_X1_full, train_Y1_temp_full, cv=cv, scoring='f1_micro', n_jobs=-1)\n",
    "#     f1 = score.mean()\n",
    "#     return f1       \n",
    "\n",
    "\n",
    "# study = optuna.create_study(direction='maximize')\n",
    "# study.optimize(objective, n_trials=50) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# # Took 63 minutes and I had to stop it since CPU was over heating (90 - 100 celcius and anything above 80 is high). I got up to trial 71\n",
    "\n",
    "# # The trials can be found on the \"Optuna_Logs_XGBoost_Model18.txt\" file!!!\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try Optuna's integration with XGBoost, which is more efficient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from optuna.integration import XGBoostPruningCallback\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from optuna.pruners import MedianPruner\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "\n",
    "# train_X1_full = pd.concat([train_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     # params = {\n",
    "       \n",
    "#     #     'max_depth': trial.suggest_int('max_depth', 9, 20),\n",
    "#     #     'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n",
    "#     #     'gamma': trial.suggest_float('gamma', 0, 0.5),\n",
    "#     #     'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1),\n",
    "#     #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n",
    "#     #     'subsample': trial.suggest_float('subsample', 0.5, 1),\n",
    "#     #     'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-5, 1),\n",
    "#     #     'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-5, 1),\n",
    "#     # }\n",
    "    \n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm\n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         #'predictor': 'gpu_predictor',\n",
    "#         #'gpu_id' : 0,\n",
    "#         #'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'merror',\n",
    "#         #'eval_metric': 'mlogloss',\n",
    "      \n",
    "      \n",
    "#         #'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 10), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 400, 850) # This adds num_boost_round into hyper parameter tuning\n",
    "\n",
    "#     dtrain = xgb.DMatrix(train_X1_full, label=train_Y1_temp_full) # This is change the pandas data into something our GPU can handle better (DMatrix)\n",
    "#     dvalid = xgb.DMatrix(valid_X1, label=valid_Y1_temp)\n",
    "\n",
    "#     #pruning_callback = XGBoostPruningCallback(trial, \"validation-merror\")  # This is what makes this code \"Optuna's integration with XGBoost\"\n",
    "  \n",
    "\n",
    "#     watchlist = [(dtrain, 'train'), (dvalid, 'validation')]  # Specify what data to evaluate and when\n",
    "\n",
    "#     model = xgb.train(\n",
    "#         params,\n",
    "#         dtrain,\n",
    "#         num_boost_round=num_boost_round,\n",
    "#         evals=watchlist,\n",
    "#         early_stopping_rounds=10,\n",
    "#         #callbacks=[pruning_callback],\n",
    "#         verbose_eval=False  # verbose\n",
    "#     )\n",
    "\n",
    "#     # Use validation data for evaluation\n",
    "#     preds = model.predict(dvalid)\n",
    "#     error = (preds != valid_Y1_temp).sum() / len(valid_Y1_temp)\n",
    "\n",
    "#     return 1 - error\n",
    "\n",
    "# #pruner = MedianPruner(n_startup_trials=10, n_warmup_steps=10)\n",
    "# pruner = HyperbandPruner(min_resource=10) # Pruning won't be considered until after 10 iterations for each trial\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', \n",
    "#                             #pruner=pruner, \n",
    "#                             sampler=sampler)\n",
    "# study.optimize(objective, n_trials=300)\n",
    "\n",
    "# # print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# print(f'Best trial: trial number {study.best_trial.number}, score {study.best_trial.value}, params {study.best_trial.params}')\n",
    "\n",
    "# # Took 11 minutes for 300 trials. XGBoostPruningCallBack and CmaEsSampler\n",
    "\n",
    "# # Took 37 min for 300 trials. Hyperband pruning and TPESampler\n",
    "\n",
    "# # Took 37 min for 300 trials. Hyperbad pruning and CmaEsSampler\n",
    "\n",
    "# # All logs can be found in the \"Optuna_Logs_XGBoost_Model19.txt\" file. The 11 minute one is on the bottom of the document.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from optuna.integration import XGBoostPruningCallback\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from optuna.pruners import MedianPruner\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and testt sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     # params = {\n",
    "       \n",
    "#     #     'max_depth': trial.suggest_int('max_depth', 9, 20),\n",
    "#     #     'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n",
    "#     #     'gamma': trial.suggest_float('gamma', 0, 0.5),\n",
    "#     #     'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1),\n",
    "#     #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n",
    "#     #     'subsample': trial.suggest_float('subsample', 0.5, 1),\n",
    "#     #     'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-5, 1),\n",
    "#     #     'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-5, 1),\n",
    "#     # }\n",
    "    \n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm\n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         #'predictor': 'gpu_predictor',\n",
    "#         #'gpu_id' : 0,\n",
    "#         #'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'merror',\n",
    "#         #'eval_metric': 'mlogloss',\n",
    "      \n",
    "      \n",
    "#         #'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 10), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 400, 850) # This adds num_boost_round into hyper parameter tuning\n",
    "\n",
    "#     dtrain = xgb.DMatrix(train_X1_full, label=train_Y1_temp_full) # This is change the pandas data into something our GPU can handle better (DMatrix)\n",
    "\n",
    "#     pruning_callback = XGBoostPruningCallback(trial, \"test-merror-mean\") # This is what makes this code \"Optuna's integration with XGBoost\"\n",
    "  \n",
    "\n",
    "#     cv_results = xgb.cv(\n",
    "#         params,\n",
    "#         dtrain,\n",
    "#         num_boost_round = num_boost_round,\n",
    "#         nfold=3,\n",
    "#         metrics={'merror'},\n",
    "#         early_stopping_rounds=10,\n",
    "#         callbacks=[pruning_callback]\n",
    "#     )\n",
    "\n",
    "#     return 1 - cv_results['test-merror-mean'].min()\n",
    "\n",
    "# #pruner = MedianPruner()\n",
    "# pruner = HyperbandPruner()\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', pruner=pruner, sampler=sampler)\n",
    "# study.optimize(objective, n_trials=200)\n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 19. Score: 0.7393"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-14 22:04:05,626] Trial 1 finished with value: 0.737477615758506 and parameters: {'max_depth': 8, 'learning_rate': 0.0493605952014681, 'subsample': 0.8050435191563892, 'colsample_bytree': 0.854112103631072, 'reg_alpha': 0.00020118715081702792, 'reg_lambda': 0.006404389225491458, 'min_child_weight': 14, 'gamma': 0.29607599793557965, 'num_boost_round': 549}. Best is trial 1 with value: 0.737477615758506.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# params = {\n",
    "#     'tree_method': 'hist',\n",
    "#     'device': 'cuda',\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'merror',\n",
    "#     'random_state': 42,\n",
    "\n",
    "\n",
    "\n",
    "#     'max_depth': 8,\n",
    "#     'learning_rate': 0.0493605952014681,\n",
    "#     'subsample': 0.8050435191563892,\n",
    "#     'colsample_bytree': 0.854112103631072,\n",
    "#     'reg_alpha': 0.00020118715081702792,\n",
    "#     'reg_lambda': 0.006404389225491458,\n",
    "#     'min_child_weight': 14,\n",
    "#     'gamma': 0.29607599793557965,\n",
    "    \n",
    "# }\n",
    "\n",
    "# # Number of boosting rounds\n",
    "# num_boost_round = 549\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# model19 = xgb.train(params, dfull, num_boost_round)\n",
    "\n",
    "\n",
    "# # Save the model to a file in JSON format\n",
    "# model19.save_model('saved_XGBoost_model19.json')\n",
    "\n",
    "\n",
    "# # # Load the model from the JSON file\n",
    "# # model19 = xgb.Booster()\n",
    "# # model19.load_model('saved_XGBoost_model19.json')\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model19.predict(dcompetition_test_X)\n",
    "# competition_y_pred = np.round(competition_y_pred).astype(int)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_19.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-14 22:04:05,626] Trial 1 finished with value: 0.737477615758506 and parameters: {'max_depth': 8, 'learning_rate': 0.0493605952014681, 'subsample': 0.8050435191563892, 'colsample_bytree': 0.854112103631072, 'reg_alpha': 0.00020118715081702792, 'reg_lambda': 0.006404389225491458, 'min_child_weight': 14, 'gamma': 0.29607599793557965, 'num_boost_round': 549}. Best is trial 1 with value: 0.737477615758506.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# params = {\n",
    "#     'tree_method': 'hist',\n",
    "#     'device': 'cuda',\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'merror',\n",
    "#     'random_state': 42,\n",
    "\n",
    "\n",
    "\n",
    "#     'max_depth': 8,\n",
    "#     'learning_rate': 0.0493605952014681,\n",
    "#     'subsample': 0.8050435191563892,\n",
    "#     'colsample_bytree': 0.854112103631072,\n",
    "#     'reg_alpha': 0.00020118715081702792,\n",
    "#     'reg_lambda': 0.006404389225491458,\n",
    "#     'min_child_weight': 14,\n",
    "#     'gamma': 0.29607599793557965,\n",
    "    \n",
    "# }\n",
    "\n",
    "# # Number of boosting rounds\n",
    "# num_boost_round = 549\n",
    "\n",
    "\n",
    "# model19 = XGBClassifier(**params)\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "\n",
    "\n",
    "# # Refit the model on the full dataset\n",
    "# model19.fit(full_X, full_Y_temp)\n",
    "\n",
    "# # Save the model to a file using joblib\n",
    "# from joblib import dump\n",
    "# dump(model19, 'saved_XGBoost_model19.joblib') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "\n",
    "# # # This is how to load the model from joblib\n",
    "# # from joblib import load\n",
    "# # model19 = load('model19.joblib')  # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model19.predict(competition_test_X)\n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_19.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\")\n",
    "\n",
    "# # Successfully Submitted!\n",
    "# # /home/michaelye22/.local/lib/python3.10/site-packages/xgboost/core.py:160: UserWarning: [19:19:49] WARNING: /workspace/src/common/error_msg.cc:58: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.\n",
    "# # Potential solutions:\n",
    "# # - Use a data structure that matches the device ordinal in the booster.\n",
    "# # - Set the device for booster before call to inplace_predict.\n",
    "\n",
    "# # This warning will only be shown once.\n",
    "\n",
    "# #   warnings.warn(smsg, UserWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 20, using first round of Optuna (on the bottom of the \"Optuna_Logs_XGBoost_Model19.txt\" file)\n",
    "\n",
    "#### Score: 74.74"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-14 22:34:29,248] Trial 4 finished with value: 0.7419800460475825 and parameters: {'max_depth': 8, 'learning_rate': 0.09660978171950639, 'subsample': 0.8811659484112643, 'colsample_bytree': 0.8451090426407317, 'reg_alpha': 0.0007659971588599838, 'reg_lambda': 0.012751505327279422, 'min_child_weight': 12, 'gamma': 0.1667116512451981, 'num_boost_round': 813}. Best is trial 4 with value: 0.7419800460475825.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# params = {\n",
    "#     'tree_method': 'hist',\n",
    "#     'device': 'cuda',\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'merror',\n",
    "#     'random_state': 42,\n",
    "\n",
    "\n",
    "\n",
    "#     'max_depth': 8,\n",
    "#     'learning_rate': 0.09660978171950639,\n",
    "#     'subsample': 0.8811659484112643,\n",
    "#     'colsample_bytree': 0.8451090426407317,\n",
    "#     'reg_alpha': 0.0007659971588599838,\n",
    "#     'reg_lambda': 0.012751505327279422,\n",
    "#     'min_child_weight': 12,\n",
    "#     'gamma': 0.1667116512451981,\n",
    "    \n",
    "# }\n",
    "\n",
    "# # Number of boosting rounds\n",
    "# num_boost_round = 813\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Making the model using xgb\n",
    "# model20 = xgb.train(params, dfull, num_boost_round)\n",
    "\n",
    "\n",
    "# # Save the model to a file in JSON format\n",
    "# model20.save_model('saved_XGBoost_model20.json')\n",
    "\n",
    "\n",
    "# # # Load the model from the JSON file\n",
    "# # model20 = xgb.Booster()\n",
    "# # model20.load_model('saved_XGBoost_model20.json') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model20.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "# competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_20.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\") \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optuna with XGBoost. Logs found in \"Optuna_Logs_XGBoost_Model21.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from optuna.integration import XGBoostPruningCallback\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from optuna.pruners import MedianPruner\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "\n",
    "# train_X1_full = pd.concat([train_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, test_Y1_temp]) # I didn't realize this but since I added my test data here instead of in validation, I have 85/15 split instead of a good 70/30 split\n",
    "\n",
    "# def objective(trial):\n",
    "#     # params = {\n",
    "       \n",
    "#     #     'max_depth': trial.suggest_int('max_depth', 9, 20),\n",
    "#     #     'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n",
    "#     #     'gamma': trial.suggest_float('gamma', 0, 0.5),\n",
    "#     #     'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1),\n",
    "#     #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n",
    "#     #     'subsample': trial.suggest_float('subsample', 0.5, 1),\n",
    "#     #     'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-5, 1),\n",
    "#     #     'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-5, 1),\n",
    "#     # }\n",
    "    \n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm\n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         #'predictor': 'gpu_predictor',\n",
    "#         #'gpu_id' : 0,\n",
    "#         #'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'merror',\n",
    "#         #'eval_metric': 'mlogloss',\n",
    "      \n",
    "      \n",
    "#         #'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 20), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 400, 900) # This adds num_boost_round into hyper parameter tuning\n",
    "\n",
    "#     dtrain = xgb.DMatrix(train_X1_full, label=train_Y1_temp_full) # This is change the pandas data into something our GPU can handle better (DMatrix)\n",
    "#     dvalid = xgb.DMatrix(valid_X1, label=valid_Y1_temp)\n",
    "\n",
    "#     #pruning_callback = XGBoostPruningCallback(trial, \"validation-merror\")  # This is what makes this code \"Optuna's integration with XGBoost\"\n",
    "  \n",
    "\n",
    "#     watchlist = [(dtrain, 'train'), (dvalid, 'validation')]  # Specify what data to evaluate and when\n",
    "\n",
    "#     model = xgb.train(\n",
    "#         params,\n",
    "#         dtrain,\n",
    "#         num_boost_round=num_boost_round,\n",
    "#         evals=watchlist,\n",
    "#         early_stopping_rounds=10,\n",
    "#         #callbacks=[pruning_callback],\n",
    "#         verbose_eval=False  # verbose\n",
    "#     )\n",
    "\n",
    "#     # Use validation data for evaluation\n",
    "#     preds = model.predict(dvalid)\n",
    "#     error = (preds != valid_Y1_temp).sum() / len(valid_Y1_temp)\n",
    "\n",
    "#     return 1 - error\n",
    "\n",
    "# #pruner = MedianPruner(n_startup_trials=10, n_warmup_steps=10)\n",
    "# pruner = HyperbandPruner(min_resource=10) # Pruning won't be considered until after 10 iterations for each trial\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', \n",
    "#                             #pruner=pruner, \n",
    "#                             sampler=sampler)\n",
    "# study.optimize(objective, n_trials=600)\n",
    "\n",
    "# # print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# print(f'Best trial: trial number {study.best_trial.number}, score {study.best_trial.value}, params {study.best_trial.params}')\n",
    "\n",
    "# # Took 11 minutes for 300 trials. XGBoostPruningCallBack and CmaEsSampler\n",
    "\n",
    "# # Took 37 min for 300 trials. Hyperband pruning and TPESampler\n",
    "\n",
    "# # Took 37 min for 300 trials. Hyperbad pruning and CmaEsSampler\n",
    "\n",
    "# # Took 161 min (2 hours and 41 min) for 600 trials. Hyperband pruning and CmaEsSampler. Increased num_boost_round and max_depth. \n",
    "\n",
    "# # All logs can be found in the \"Optuna_Logs_XGBoost_Model21.txt\" file. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 21, from \"Optuna_Logs_XGBoost_Model21.txt.\" Score: 74.29\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # [I 2024-01-17 21:09:21,975] Trial 538 finished with value: 0.7506779227423894 and parameters: {'max_depth': 20, 'learning_rate': 0.04735883403404956, 'subsample': 0.820644662125095, 'colsample_bytree': 0.7184349909513894, 'reg_alpha': 0.011578588979534711, 'reg_lambda': 0.0039715599430452725, 'min_child_weight': 13, 'gamma': 0.16377835407138885, 'num_boost_round': 815}. Best is trial 538 with value: 0.7506779227423894.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# params = {\n",
    "#     'tree_method': 'hist',\n",
    "#     'device': 'cuda',\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'merror',\n",
    "#     'random_state': 42,\n",
    "\n",
    "\n",
    "\n",
    "#     'max_depth': 20,\n",
    "#     'learning_rate': 0.04735883403404956,\n",
    "#     'subsample': 0.820644662125095,\n",
    "#     'colsample_bytree': 0.7184349909513894,\n",
    "#     'reg_alpha': 0.011578588979534711,\n",
    "#     'reg_lambda': 0.0039715599430452725,\n",
    "#     'min_child_weight': 13,\n",
    "#     'gamma': 0.16377835407138885,\n",
    "    \n",
    "# }\n",
    "\n",
    "# # Number of boosting rounds\n",
    "# num_boost_round = 815\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Making the model using xgb\n",
    "# model21 = xgb.train(params, dfull, num_boost_round)\n",
    "\n",
    "\n",
    "# # Save the model to a file in JSON format\n",
    "# model21.save_model('saved_XGBoost_model21.bin') # Saving it to be binary (bin) since json was too large\n",
    "\n",
    "\n",
    "# # # Load the model from the JSON file\n",
    "# # model21 = xgb.Booster()\n",
    "# # model21.load_model('saved_XGBoost_model21.json') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Load the model from the bin (binary) file\n",
    "# model21 = xgb.Booster()\n",
    "# model21.load_model('saved_XGBoost_model21.bin')\n",
    "\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model21.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "# competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_21.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\") \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 22, with CPU predictor, from \"Optuna_Logs_XGBoost_Model21.txt.\" Score: 74.86. HIGHEST Score so far!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 74.29% [I 2024-01-17 20:34:55,147] Trial 434 finished with value: 0.7503965208493221 and parameters: {'max_depth': 20, 'learning_rate': 0.05240086193952971, 'subsample': 0.7775131152851413, 'colsample_bytree': 0.7080302282921819, 'reg_alpha': 0.005031582823108817, 'reg_lambda': 0.004109741559817778, 'min_child_weight': 12, 'gamma': 0.14380399018568563, 'num_boost_round': 768}. Best is trial 434 with value: 0.7503965208493221.\n",
    "# # 74.19% [I 2024-01-17 20:15:01,147] Trial 363 finished with value: 0.7503453568687644 and parameters: {'max_depth': 19, 'learning_rate': 0.06226811843938903, 'subsample': 0.7963573343501883, 'colsample_bytree': 0.7288693410290388, 'reg_alpha': 0.0011147331327687727, 'reg_lambda': 0.003696015096232239, 'min_child_weight': 13, 'gamma': 0.09897851695964469, 'num_boost_round': 768}. Best is trial 363 with value: 0.7503453568687644.\n",
    "# # 73.94% [I 2024-01-17 20:00:09,251] Trial 304 finished with value: 0.7502686108979278 and parameters: {'max_depth': 18, 'learning_rate': 0.06949760197050864, 'subsample': 0.7871296138515518, 'colsample_bytree': 0.7223808145597003, 'reg_alpha': 0.0028247977944178774, 'reg_lambda': 0.000577550039269003, 'min_child_weight': 9, 'gamma': 0.097092580471978, 'num_boost_round': 705}. Best is trial 304 with value: 0.7502686108979278.\n",
    "# # 74.37% [I 2024-01-17 19:54:18,937] Trial 280 finished with value: 0.7497313891020722 and parameters: {'max_depth': 19, 'learning_rate': 0.0507538135551261, 'subsample': 0.7757033990592374, 'colsample_bytree': 0.7033304767300838, 'reg_alpha': 0.0033088574343809146, 'reg_lambda': 0.010811444965629302, 'min_child_weight': 13, 'gamma': 0.14001008988371658, 'num_boost_round': 791}. Best is trial 280 with value: 0.7497313891020722.\n",
    "# # 74.12% [I 2024-01-17 19:32:37,856] Trial 200 finished with value: 0.7496290611409568 and parameters: {'max_depth': 19, 'learning_rate': 0.0654529656924255, 'subsample': 0.7924918656264801, 'colsample_bytree': 0.7299869960283065, 'reg_alpha': 0.0002183208300517004, 'reg_lambda': 0.002460830534207808, 'min_child_weight': 10, 'gamma': 0.11695512242880172, 'num_boost_round': 655}. Best is trial 200 with value: 0.7496290611409568.\n",
    "# # [I 2024-01-17 19:05:25,715] Trial 73 finished with value: 0.7495011511895626 and parameters: {'max_depth': 16, 'learning_rate': 0.07804953080726278, 'subsample': 0.7372446311060505, 'colsample_bytree': 0.7313209356500971, 'reg_alpha': 0.0004963207814185677, 'reg_lambda': 0.00040658603133315256, 'min_child_weight': 11, 'gamma': 0.10405169615573975, 'num_boost_round': 497}. Best is trial 73 with value: 0.7495011511895626.\n",
    "# # [I 2024-01-17 19:01:40,996] Trial 57 finished with value: 0.74911742133538 and parameters: {'max_depth': 17, 'learning_rate': 0.05783367331655646, 'subsample': 0.8224779216422303, 'colsample_bytree': 0.7274126154655153, 'reg_alpha': 0.001936343415557244, 'reg_lambda': 0.002112405454336705, 'min_child_weight': 8, 'gamma': 0.23352989217655448, 'num_boost_round': 557}. Best is trial 57 with value: 0.74911742133538.\n",
    "# # [I 2024-01-17 18:57:06,865] Trial 36 finished with value: 0.7485546175492453 and parameters: {'max_depth': 19, 'learning_rate': 0.04729565720609144, 'subsample': 0.8073063971695645, 'colsample_bytree': 0.7447703594205985, 'reg_alpha': 0.003247889923636391, 'reg_lambda': 0.00039516710765441314, 'min_child_weight': 11, 'gamma': 0.1654429468212822, 'num_boost_round': 486}. Best is trial 36 with value: 0.7485546175492453.\n",
    "# # 74.81% [I 2024-01-17 18:51:08,796] Trial 9 finished with value: 0.7485034535686876 and parameters: {'max_depth': 13, 'learning_rate': 0.05336935110261098, 'subsample': 0.8834652325679093, 'colsample_bytree': 0.8220555703793428, 'reg_alpha': 0.011173016288011785, 'reg_lambda': 0.0008103757391725349, 'min_child_weight': 18, 'gamma': 0.24717434178747674, 'num_boost_round': 557}. Best is trial 9 with value: 0.7485034535686876.\n",
    "\n",
    "\n",
    "# # [I 2024-01-17 18:57:06,865] Trial 36 finished with value: 0.7485546175492453 and parameters: {'max_depth': 19, 'learning_rate': 0.04729565720609144, 'subsample': 0.8073063971695645, 'colsample_bytree': 0.7447703594205985, 'reg_alpha': 0.003247889923636391, 'reg_lambda': 0.00039516710765441314, 'min_child_weight': 11, 'gamma': 0.1654429468212822, 'num_boost_round': 486}. Best is trial 36 with value: 0.7485546175492453.\n",
    "# # [I 2024-01-17 19:04:09,335] Trial 68 finished with value: 0.7487848554617549 and parameters: {'max_depth': 18, 'learning_rate': 0.08264145597007437, 'subsample': 0.732058828773006, 'colsample_bytree': 0.7194204017091105, 'reg_alpha': 0.0005707300188849352, 'reg_lambda': 0.002782286325783357, 'min_child_weight': 11, 'gamma': 0.1538405594371126, 'num_boost_round': 557}. Best is trial 57 with value: 0.74911742133538.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# params = {\n",
    "#     'tree_method': 'hist',\n",
    "#     'device': 'cuda',\n",
    "#     'objective': 'multi:softmax',\n",
    "#     'num_class': 3,\n",
    "#     'eval_metric': 'merror',\n",
    "#     'random_state': 42,\n",
    "\n",
    "\n",
    "\n",
    "#     'max_depth': 13, #11 got 74.76. 15 got 74.75\n",
    "#     'learning_rate': 0.05336935110261098,\n",
    "#     'subsample': 0.8834652325679093,\n",
    "#     'colsample_bytree': 0.8220555703793428,\n",
    "#     'reg_alpha': 0.011173016288011785,\n",
    "#     'reg_lambda': 0.0008103757391725349,\n",
    "#     'min_child_weight': 18,\n",
    "#     'gamma': 0.24717434178747674,\n",
    "  \n",
    "    \n",
    "\n",
    "    \n",
    "# }\n",
    "\n",
    "# # Number of boosting rounds\n",
    "# num_boost_round = 750 #660 got 74.84. 750 got 74.86. 850 got 74.79. 800 got 74.83. 775 got 74.85. 786 got 74.85. This is based on Trial 9 but with modified num_boost_round\n",
    "\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Making the model using xgb\n",
    "# model22 = xgb.train(params, dfull, num_boost_round)\n",
    "\n",
    "\n",
    "# # Save the model to a file in binary (bin) format\n",
    "# model22.save_model('saved_XGBoost_model22.bin')\n",
    "\n",
    "\n",
    "# # # Load the model from the bin (binary) file\n",
    "# model22 = xgb.Booster()\n",
    "# model22.load_model('saved_XGBoost_model22.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Predict on the competition test data\n",
    "# competition_y_pred = model22.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "# competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "# # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "# competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "# # Create a DataFrame for submission\n",
    "# submission = pd.DataFrame({\n",
    "#     'building_id': competition_test_building_ids,\n",
    "#     'damage_grade': competition_y_pred\n",
    "# })\n",
    "\n",
    "# # Save the submission DataFrame to a CSV file for submission\n",
    "# submission.to_csv('submission_XGBoost_22_.csv', index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "# print(\"Successfully Submitted!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 22, with GPU (instead of CPU) predictor, from \"Optuna_Logs_XGBoost_Model21.txt.\" Score: 74.84"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 74.29% [I 2024-01-17 20:34:55,147] Trial 434 finished with value: 0.7503965208493221 and parameters: {'max_depth': 20, 'learning_rate': 0.05240086193952971, 'subsample': 0.7775131152851413, 'colsample_bytree': 0.7080302282921819, 'reg_alpha': 0.005031582823108817, 'reg_lambda': 0.004109741559817778, 'min_child_weight': 12, 'gamma': 0.14380399018568563, 'num_boost_round': 768}. Best is trial 434 with value: 0.7503965208493221.\n",
    "# # 74.19% [I 2024-01-17 20:15:01,147] Trial 363 finished with value: 0.7503453568687644 and parameters: {'max_depth': 19, 'learning_rate': 0.06226811843938903, 'subsample': 0.7963573343501883, 'colsample_bytree': 0.7288693410290388, 'reg_alpha': 0.0011147331327687727, 'reg_lambda': 0.003696015096232239, 'min_child_weight': 13, 'gamma': 0.09897851695964469, 'num_boost_round': 768}. Best is trial 363 with value: 0.7503453568687644.\n",
    "# # 73.94% [I 2024-01-17 20:00:09,251] Trial 304 finished with value: 0.7502686108979278 and parameters: {'max_depth': 18, 'learning_rate': 0.06949760197050864, 'subsample': 0.7871296138515518, 'colsample_bytree': 0.7223808145597003, 'reg_alpha': 0.0028247977944178774, 'reg_lambda': 0.000577550039269003, 'min_child_weight': 9, 'gamma': 0.097092580471978, 'num_boost_round': 705}. Best is trial 304 with value: 0.7502686108979278.\n",
    "# # 74.37% [I 2024-01-17 19:54:18,937] Trial 280 finished with value: 0.7497313891020722 and parameters: {'max_depth': 19, 'learning_rate': 0.0507538135551261, 'subsample': 0.7757033990592374, 'colsample_bytree': 0.7033304767300838, 'reg_alpha': 0.0033088574343809146, 'reg_lambda': 0.010811444965629302, 'min_child_weight': 13, 'gamma': 0.14001008988371658, 'num_boost_round': 791}. Best is trial 280 with value: 0.7497313891020722.\n",
    "# # 74.12% [I 2024-01-17 19:32:37,856] Trial 200 finished with value: 0.7496290611409568 and parameters: {'max_depth': 19, 'learning_rate': 0.0654529656924255, 'subsample': 0.7924918656264801, 'colsample_bytree': 0.7299869960283065, 'reg_alpha': 0.0002183208300517004, 'reg_lambda': 0.002460830534207808, 'min_child_weight': 10, 'gamma': 0.11695512242880172, 'num_boost_round': 655}. Best is trial 200 with value: 0.7496290611409568.\n",
    "# # [I 2024-01-17 19:05:25,715] Trial 73 finished with value: 0.7495011511895626 and parameters: {'max_depth': 16, 'learning_rate': 0.07804953080726278, 'subsample': 0.7372446311060505, 'colsample_bytree': 0.7313209356500971, 'reg_alpha': 0.0004963207814185677, 'reg_lambda': 0.00040658603133315256, 'min_child_weight': 11, 'gamma': 0.10405169615573975, 'num_boost_round': 497}. Best is trial 73 with value: 0.7495011511895626.\n",
    "# # [I 2024-01-17 19:01:40,996] Trial 57 finished with value: 0.74911742133538 and parameters: {'max_depth': 17, 'learning_rate': 0.05783367331655646, 'subsample': 0.8224779216422303, 'colsample_bytree': 0.7274126154655153, 'reg_alpha': 0.001936343415557244, 'reg_lambda': 0.002112405454336705, 'min_child_weight': 8, 'gamma': 0.23352989217655448, 'num_boost_round': 557}. Best is trial 57 with value: 0.74911742133538.\n",
    "# # [I 2024-01-17 18:57:06,865] Trial 36 finished with value: 0.7485546175492453 and parameters: {'max_depth': 19, 'learning_rate': 0.04729565720609144, 'subsample': 0.8073063971695645, 'colsample_bytree': 0.7447703594205985, 'reg_alpha': 0.003247889923636391, 'reg_lambda': 0.00039516710765441314, 'min_child_weight': 11, 'gamma': 0.1654429468212822, 'num_boost_round': 486}. Best is trial 36 with value: 0.7485546175492453.\n",
    "# # [I 2024-01-17 18:51:08,796] Trial 9 finished with value: 0.7485034535686876 and parameters: {'max_depth': 15, 'learning_rate': 0.05336935110261098, 'subsample': 0.8834652325679093, 'colsample_bytree': 0.8220555703793428, 'reg_alpha': 0.011173016288011785, 'reg_lambda': 0.0008103757391725349, 'min_child_weight': 18, 'gamma': 0.24717434178747674, 'num_boost_round': 557}. Best is trial 9 with value: 0.7485034535686876.\n",
    "\n",
    "\n",
    "# # # [I 2024-01-17 19:04:09,335] Trial 68 finished with value: 0.7487848554617549 and parameters: {'max_depth': 18, 'learning_rate': 0.08264145597007437, 'subsample': 0.732058828773006, 'colsample_bytree': 0.7194204017091105, 'reg_alpha': 0.0005707300188849352, 'reg_lambda': 0.002782286325783357, 'min_child_weight': 11, 'gamma': 0.1538405594371126, 'num_boost_round': 557}. Best is trial 57 with value: 0.74911742133538.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Define the parameters for each model\n",
    "# params = [\n",
    "#     {'max_depth': 20, 'learning_rate': 0.05240086193952971, 'subsample': 0.7775131152851413, 'colsample_bytree': 0.7080302282921819, 'reg_alpha': 0.005031582823108817, 'reg_lambda': 0.004109741559817778, 'min_child_weight': 12, 'gamma': 0.14380399018568563},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.06226811843938903, 'subsample': 0.7963573343501883, 'colsample_bytree': 0.7288693410290388, 'reg_alpha': 0.0011147331327687727, 'reg_lambda': 0.003696015096232239, 'min_child_weight': 13, 'gamma': 0.09897851695964469},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.06949760197050864, 'subsample': 0.7871296138515518, 'colsample_bytree': 0.7223808145597003, 'reg_alpha': 0.0028247977944178774, 'reg_lambda': 0.000577550039269003, 'min_child_weight': 9, 'gamma': 0.097092580471978},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.0507538135551261, 'subsample': 0.7757033990592374, 'colsample_bytree': 0.7033304767300838, 'reg_alpha': 0.0033088574343809146, 'reg_lambda': 0.010811444965629302, 'min_child_weight': 13, 'gamma': 0.14001008988371658},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.0654529656924255, 'subsample': 0.7924918656264801, 'colsample_bytree': 0.7299869960283065, 'reg_alpha': 0.0002183208300517004, 'reg_lambda': 0.002460830534207808, 'min_child_weight': 10, 'gamma': 0.11695512242880172},\n",
    "#     {'max_depth': 16, 'learning_rate': 0.07804953080726278, 'subsample': 0.7372446311060505, 'colsample_bytree': 0.7313209356500971, 'reg_alpha': 0.0004963207814185677, 'reg_lambda': 0.00040658603133315256, 'min_child_weight': 11, 'gamma': 0.10405169615573975},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.05783367331655646, 'subsample': 0.8224779216422303, 'colsample_bytree': 0.7274126154655153, 'reg_alpha': 0.001936343415557244, 'reg_lambda': 0.002112405454336705, 'min_child_weight': 8, 'gamma': 0.23352989217655448},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.04729565720609144, 'subsample': 0.8073063971695645, 'colsample_bytree': 0.7447703594205985, 'reg_alpha': 0.003247889923636391, 'reg_lambda': 0.00039516710765441314, 'min_child_weight': 11, 'gamma': 0.1654429468212822},\n",
    "#     {'max_depth': 15, 'learning_rate': 0.05336935110261098, 'subsample': 0.8834652325679093, 'colsample_bytree': 0.8220555703793428, 'reg_alpha': 0.011173016288011785, 'reg_lambda': 0.0008103757391725349, 'min_child_weight': 18, 'gamma': 0.24717434178747674},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.08264145597007437, 'subsample': 0.732058828773006, 'colsample_bytree': 0.7194204017091105, 'reg_alpha': 0.0005707300188849352, 'reg_lambda': 0.002782286325783357, 'min_child_weight': 11, 'gamma': 0.1538405594371126},\n",
    "# ]\n",
    "\n",
    "# # This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# # The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "# for param in params:\n",
    "#     param.update({\n",
    "#         'tree_method': 'hist',\n",
    "#         'device': 'cuda',\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "#     })\n",
    "\n",
    "\n",
    "# num_boost_round = [768, 768, 705, 791, 655, 497, 557, 486, 557, 557]\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "# for i in range(len(params)):\n",
    "\n",
    "#     # Create the XGBoost models\n",
    "#     model22 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "#     # Save the model to a file in JSON format\n",
    "#     #model22.save_model('saved_XGBoost_model22.json')\n",
    "\n",
    "\n",
    "#     # # Load the model from the bin (binary) file\n",
    "#     # model22 = xgb.Booster()\n",
    "#     # model22.load_model('saved_XGBoost_model22.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "#     # Predict on the competition test data\n",
    "#     competition_y_pred = model22.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "#     competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "#     # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "#     competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "#     # Create a DataFrame for submission\n",
    "#     submission = pd.DataFrame({\n",
    "#         'building_id': competition_test_building_ids,\n",
    "#         'damage_grade': competition_y_pred\n",
    "#     })\n",
    "\n",
    "#     # Save the submission DataFrame to a CSV file for submission\n",
    "#     submission.to_csv(f\"submission_XGBoost_22__{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "#     print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 23, from \"Optuna_Logs_XGBoost_Model21_Kaggle.txt.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 0. [I 2024-01-24 19:46:12,639] Trial 442 finished with value: 0.7504988488104375 and parameters: {'max_depth': 24, 'learning_rate': 0.05499358322455664, 'subsample': 0.9953760617217673, 'colsample_bytree': 0.7003539269061473, 'reg_alpha': 0.5685576654400151, 'reg_lambda': 0.00010880959224634605, 'min_child_weight': 12, 'gamma': 0.4061534464520698, 'num_boost_round': 593}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 1. [I 2024-01-24 18:58:45,804] Trial 243 finished with value: 0.750422102839601 and parameters: {'max_depth': 26, 'learning_rate': 0.05806796580696987, 'subsample': 0.983071751147048, 'colsample_bytree': 0.7237525527298514, 'reg_alpha': 0.26534117163076976, 'reg_lambda': 0.00013147205610001023, 'min_child_weight': 16, 'gamma': 0.4147177374695941, 'num_boost_round': 552}. Best is trial 243 with value: 0.750422102839601.\n",
    "# # 2. [I 2024-01-24 18:18:19,700] Trial 68 finished with value: 0.7502174469173701 and parameters: {'max_depth': 21, 'learning_rate': 0.07092241783523516, 'subsample': 0.9426411272656379, 'colsample_bytree': 0.7624376090904599, 'reg_alpha': 0.9657127210093687, 'reg_lambda': 0.485514659006626, 'min_child_weight': 13, 'gamma': 0.483339100910917, 'num_boost_round': 471}. Best is trial 68 with value: 0.7502174469173701.\n",
    "# # 3. [I 2024-01-24 18:10:26,115] Trial 31 finished with value: 0.7496802251215144 and parameters: {'max_depth': 23, 'learning_rate': 0.05140777212279913, 'subsample': 0.9748724577801192, 'colsample_bytree': 0.7012107163416342, 'reg_alpha': 0.039956318878116445, 'reg_lambda': 1.8811109619649502e-05, 'min_child_weight': 15, 'gamma': 0.4193938630627251, 'num_boost_round': 532}. Best is trial 31 with value: 0.7496802251215144.\n",
    "# # 4. [I 2024-01-24 18:08:51,456] Trial 24 finished with value: 0.7495523151701202 and parameters: {'max_depth': 26, 'learning_rate': 0.06783052647468646, 'subsample': 0.9798367091372376, 'colsample_bytree': 0.7010160783415671, 'reg_alpha': 0.7374805449485975, 'reg_lambda': 3.28949409824742e-05, 'min_child_weight': 18, 'gamma': 0.3915768941242742, 'num_boost_round': 559}. Best is trial 24 with value: 0.7495523151701202.\n",
    "# # 5. [I 2024-01-24 18:08:09,859] Trial 20 finished with value: 0.7488104374520338 and parameters: {'max_depth': 27, 'learning_rate': 0.09868002290593986, 'subsample': 0.963621041443613, 'colsample_bytree': 0.7073429734309913, 'reg_alpha': 0.05140684953573958, 'reg_lambda': 0.00015388768781174578, 'min_child_weight': 17, 'gamma': 0.377471928752876, 'num_boost_round': 548}. Best is trial 20 with value: 0.7488104374520338.\n",
    "# # 6. [I 2024-01-24 19:01:44,642] Trial 256 finished with value: 0.7502686108979278 and parameters: {'max_depth': 26, 'learning_rate': 0.06726060544922906, 'subsample': 0.9820538632956979, 'colsample_bytree': 0.7043520484549479, 'reg_alpha': 0.40649923487285705, 'reg_lambda': 0.00016764654563650248, 'min_child_weight': 16, 'gamma': 0.3895163408985203, 'num_boost_round': 551}. Best is trial 243 with value: 0.750422102839601.\n",
    "# # 7. [I 2024-01-24 19:10:09,021] Trial 288 finished with value: 0.7503709388590433 and parameters: {'max_depth': 25, 'learning_rate': 0.06715237055522526, 'subsample': 0.945590316652707, 'colsample_bytree': 0.7135553801977628, 'reg_alpha': 0.9876689629009342, 'reg_lambda': 7.876659586701334e-05, 'min_child_weight': 7, 'gamma': 0.4149073094841512, 'num_boost_round': 555}. Best is trial 243 with value: 0.750422102839601.\n",
    "\n",
    "# # 8. [I 2024-01-24 19:28:17,376] Trial 364 finished with value: 0.7503453568687644 and parameters: {'max_depth': 24, 'learning_rate': 0.060610486998204566, 'subsample': 0.9906638467056794, 'colsample_bytree': 0.7203240315165619, 'reg_alpha': 0.9767827681959972, 'reg_lambda': 6.711384214253154e-05, 'min_child_weight': 17, 'gamma': 0.4206162915330285, 'num_boost_round': 526}. Best is trial 243 with value: 0.750422102839601.\n",
    "# # 9. [I 2024-01-24 19:55:57,086] Trial 478 finished with value: 0.7500383729854183 and parameters: {'max_depth': 24, 'learning_rate': 0.06041101577303755, 'subsample': 0.9888228555334335, 'colsample_bytree': 0.7150019841646881, 'reg_alpha': 0.2674887787992473, 'reg_lambda': 0.00010053010959983926, 'min_child_weight': 16, 'gamma': 0.41638784582483046, 'num_boost_round': 488}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 10. [I 2024-01-24 20:02:16,306] Trial 505 finished with value: 0.7500383729854183 and parameters: {'max_depth': 24, 'learning_rate': 0.059612069536797554, 'subsample': 0.9795208776643566, 'colsample_bytree': 0.7298653887440224, 'reg_alpha': 0.6346810374545601, 'reg_lambda': 0.0001155358276653963, 'min_child_weight': 18, 'gamma': 0.418621124667366, 'num_boost_round': 474}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 11. [I 2024-01-24 20:23:24,363] Trial 589 finished with value: 0.7492453312867742 and parameters: {'max_depth': 23, 'learning_rate': 0.05809613537400483, 'subsample': 0.9837747070040155, 'colsample_bytree': 0.7245127355157561, 'reg_alpha': 0.3101223279106794, 'reg_lambda': 8.823623262958104e-05, 'min_child_weight': 15, 'gamma': 0.4099843927878516, 'num_boost_round': 668}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 12. [I 2024-01-24 20:22:25,941] Trial 584 finished with value: 0.7495523151701202 and parameters: {'max_depth': 26, 'learning_rate': 0.05254526918840954, 'subsample': 0.993591518286756, 'colsample_bytree': 0.7004339868082838, 'reg_alpha': 3.512226068319261e-06, 'reg_lambda': 2.523432703114378e-05, 'min_child_weight': 17, 'gamma': 0.3991927746992376, 'num_boost_round': 439}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 13. [I 2024-01-24 20:18:12,432] Trial 567 finished with value: 0.7491941673062164 and parameters: {'max_depth': 29, 'learning_rate': 0.06131249483502395, 'subsample': 0.9879091510864554, 'colsample_bytree': 0.7309065159005248, 'reg_alpha': 0.28244864516385093, 'reg_lambda': 0.00011451852564120219, 'min_child_weight': 16, 'gamma': 0.432069464401094, 'num_boost_round': 560}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 14. [I 2024-01-24 20:17:18,587] Trial 564 finished with value: 0.7496034791506779 and parameters: {'max_depth': 25, 'learning_rate': 0.06459305611188691, 'subsample': 0.9812130308114865, 'colsample_bytree': 0.7058561659332866, 'reg_alpha': 0.2245536421652947, 'reg_lambda': 2.113502392626074e-05, 'min_child_weight': 18, 'gamma': 0.4177117713753982, 'num_boost_round': 584}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 15. [I 2024-01-24 20:17:04,832] Trial 563 finished with value: 0.7493476592478895 and parameters: {'max_depth': 26, 'learning_rate': 0.058023688410146976, 'subsample': 0.9469405138894762, 'colsample_bytree': 0.7240069160140927, 'reg_alpha': 0.7721111370953628, 'reg_lambda': 3.802969159342665e-05, 'min_child_weight': 15, 'gamma': 0.44152453730970886, 'num_boost_round': 434}. Best is trial 442 with value: 0.7504988488104375.\n",
    "\n",
    "# # 16. [I 2024-01-24 20:15:59,536] Trial 558 finished with value: 0.7495523151701202 and parameters: {'max_depth': 26, 'learning_rate': 0.053142416675934556, 'subsample': 0.9757310873748694, 'colsample_bytree': 0.7175471174827269, 'reg_alpha': 0.4297993567042098, 'reg_lambda': 2.9382860650213248e-05, 'min_child_weight': 19, 'gamma': 0.3885595025054114, 'num_boost_round': 578}. Best is trial 442 with value: 0.7504988488104375.\n",
    "# # 17. [I 2024-01-24 20:15:44,329] Trial 557 finished with value: 0.7496034791506779 and parameters: {'max_depth': 29, 'learning_rate': 0.09870055235606595, 'subsample': 0.999941326541075, 'colsample_bytree': 0.7050817572846517, 'reg_alpha': 0.9980584089048694, 'reg_lambda': 8.18821974915161e-05, 'min_child_weight': 17, 'gamma': 0.399677286739808, 'num_boost_round': 471}. Best is trial 442 with value: 0.7504988488104375.\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Define the parameters for each model\n",
    "# params = [\n",
    "#     {'max_depth': 24, 'learning_rate': 0.05499358322455664, 'subsample': 0.9953760617217673, 'colsample_bytree': 0.7003539269061473, 'reg_alpha': 0.5685576654400151, 'reg_lambda': 0.00010880959224634605, 'min_child_weight': 12, 'gamma': 0.4061534464520698},\n",
    "#     {'max_depth': 26, 'learning_rate': 0.05806796580696987, 'subsample': 0.983071751147048, 'colsample_bytree': 0.7237525527298514, 'reg_alpha': 0.26534117163076976, 'reg_lambda': 0.00013147205610001023, 'min_child_weight': 16, 'gamma': 0.4147177374695941},\n",
    "#     {'max_depth': 21, 'learning_rate': 0.07092241783523516, 'subsample': 0.9426411272656379, 'colsample_bytree': 0.7624376090904599, 'reg_alpha': 0.9657127210093687, 'reg_lambda': 0.485514659006626, 'min_child_weight': 13, 'gamma': 0.483339100910917},\n",
    "#     {'max_depth': 23, 'learning_rate': 0.05140777212279913, 'subsample': 0.9748724577801192, 'colsample_bytree': 0.7012107163416342, 'reg_alpha': 0.039956318878116445, 'reg_lambda': 1.8811109619649502e-05, 'min_child_weight': 15, 'gamma': 0.4193938630627251},\n",
    "#     {'max_depth': 26, 'learning_rate': 0.06783052647468646, 'subsample': 0.9798367091372376, 'colsample_bytree': 0.7010160783415671, 'reg_alpha': 0.7374805449485975, 'reg_lambda': 3.28949409824742e-05, 'min_child_weight': 18, 'gamma': 0.3915768941242742},\n",
    "#     {'max_depth': 27, 'learning_rate': 0.09868002290593986, 'subsample': 0.963621041443613, 'colsample_bytree': 0.7073429734309913, 'reg_alpha': 0.05140684953573958, 'reg_lambda': 0.00015388768781174578, 'min_child_weight': 17, 'gamma': 0.377471928752876},\n",
    "#     {'max_depth': 26, 'learning_rate': 0.06726060544922906, 'subsample': 0.9820538632956979, 'colsample_bytree': 0.7043520484549479, 'reg_alpha': 0.40649923487285705, 'reg_lambda': 0.00016764654563650248, 'min_child_weight': 16, 'gamma': 0.3895163408985203},\n",
    "#     {'max_depth': 25, 'learning_rate': 0.06715237055522526, 'subsample': 0.945590316652707, 'colsample_bytree': 0.7135553801977628, 'reg_alpha': 0.9876689629009342, 'reg_lambda': 7.876659586701334e-05, 'min_child_weight': 7, 'gamma': 0.4149073094841512},\n",
    "#     {'max_depth': 24, 'learning_rate': 0.060610486998204566, 'subsample': 0.9906638467056794, 'colsample_bytree': 0.7203240315165619, 'reg_alpha': 0.9767827681959972, 'reg_lambda': 6.711384214253154e-05, 'min_child_weight': 17, 'gamma': 0.4206162915330285},\n",
    "#     {'max_depth': 24, 'learning_rate': 0.06041101577303755, 'subsample': 0.9888228555334335, 'colsample_bytree': 0.7150019841646881, 'reg_alpha': 0.2674887787992473, 'reg_lambda': 0.00010053010959983926, 'min_child_weight': 16, 'gamma': 0.41638784582483046},\n",
    "#     {'max_depth': 24, 'learning_rate': 0.059612069536797554, 'subsample': 0.9795208776643566, 'colsample_bytree': 0.7298653887440224, 'reg_alpha': 0.6346810374545601, 'reg_lambda': 0.0001155358276653963, 'min_child_weight': 18, 'gamma': 0.418621124667366},\n",
    "#     {'max_depth': 23, 'learning_rate': 0.05809613537400483, 'subsample': 0.9837747070040155, 'colsample_bytree': 0.7245127355157561, 'reg_alpha': 0.3101223279106794, 'reg_lambda': 8.823623262958104e-05, 'min_child_weight': 15, 'gamma': 0.4099843927878516},\n",
    "#     {'max_depth': 26, 'learning_rate': 0.05254526918840954, 'subsample': 0.993591518286756, 'colsample_bytree': 0.7004339868082838, 'reg_alpha': 3.512226068319261e-06, 'reg_lambda': 2.523432703114378e-05, 'min_child_weight': 17, 'gamma': 0.3991927746992376},\n",
    "#     {'max_depth': 29, 'learning_rate': 0.06131249483502395, 'subsample': 0.9879091510864554, 'colsample_bytree': 0.7309065159005248, 'reg_alpha': 0.28244864516385093, 'reg_lambda': 0.00011451852564120219, 'min_child_weight': 16, 'gamma': 0.432069464401094},\n",
    "#     {'max_depth': 25, 'learning_rate': 0.06459305611188691, 'subsample': 0.9812130308114865, 'colsample_bytree': 0.7058561659332866, 'reg_alpha': 0.2245536421652947, 'reg_lambda': 2.113502392626074e-05, 'min_child_weight': 18, 'gamma': 0.4177117713753982},\n",
    "#     {'max_depth': 26, 'learning_rate': 0.058023688410146976, 'subsample': 0.9469405138894762, 'colsample_bytree': 0.7240069160140927, 'reg_alpha': 0.7721111370953628, 'reg_lambda': 3.802969159342665e-05, 'min_child_weight': 15, 'gamma': 0.44152453730970886},\n",
    "#     {'max_depth': 26, 'learning_rate': 0.053142416675934556, 'subsample': 0.9757310873748694, 'colsample_bytree': 0.7175471174827269, 'reg_alpha': 0.4297993567042098, 'reg_lambda': 2.9382860650213248e-05, 'min_child_weight': 19, 'gamma': 0.3885595025054114},\n",
    "#     {'max_depth': 29, 'learning_rate': 0.09870055235606595, 'subsample': 0.999941326541075, 'colsample_bytree': 0.7050817572846517, 'reg_alpha': 0.9980584089048694, 'reg_lambda': 8.18821974915161e-05, 'min_child_weight': 17, 'gamma': 0.399677286739808},\n",
    "# ]\n",
    "\n",
    "# # This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# # The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "# for param in params:\n",
    "#     param.update({\n",
    "#         'tree_method': 'hist',\n",
    "#         'device': 'cuda',\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "#     })\n",
    "\n",
    "\n",
    "# num_boost_round = [593, 552, 471, 532, 559, 548, 551, 555, 526, 488, 474, 668, 439, 560, 584, 434, 578, 471]\n",
    "\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "# for i in range(len(params)):\n",
    "\n",
    "#     # Create the XGBoost models\n",
    "#     model23 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "#     # Save the model to a file in JSON format\n",
    "#     #model23.save_model('saved_XGBoost_model23.json')\n",
    "\n",
    "\n",
    "#     # # Load the model from the bin (binary) file\n",
    "#     # model23 = xgb.Booster()\n",
    "#     # model23.load_model('saved_XGBoost_model23.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "#     # Predict on the competition test data\n",
    "#     competition_y_pred = model23.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "#     competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "#     # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "#     competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "#     # Create a DataFrame for submission\n",
    "#     submission = pd.DataFrame({\n",
    "#         'building_id': competition_test_building_ids,\n",
    "#         'damage_grade': competition_y_pred\n",
    "#     })\n",
    "\n",
    "#     # Save the submission DataFrame to a CSV file for submission\n",
    "#     submission.to_csv(f\"submission_XGBoost_23_{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "#     print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 24, with CPU, from \"Optuna_Logs_XGBoost_Model21_GitHubCodeSpaces.txt.\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # can you make 8 seperate XGB models for each line? And make it so that n_estimators is it's own variable for each set of param. So like n_estimator_1 and n_estimator_2\n",
    "\n",
    "# # 0. [I 2024-01-25 01:33:30,957] Trial 566 finished with value: 0.7500127909951394 and parameters: {'max_depth': 20, 'learning_rate': 0.06320245480949405, 'subsample': 0.8188272349235466, 'colsample_bytree': 0.7127641606598758, 'reg_alpha': 0.08424063460596642, 'reg_lambda': 0.0010426300127364537, 'min_child_weight': 15, 'gamma': 0.1279168805879946, 'num_boost_round': 659}. Best is trial 566 with value: 0.7500127909951394.\n",
    "# # 1. [I 2024-01-24 23:02:49,314] Trial 250 finished with value: 0.7498848810437452 and parameters: {'max_depth': 18, 'learning_rate': 0.07282838028748469, 'subsample': 0.8022684987091674, 'colsample_bytree': 0.7188889119870725, 'reg_alpha': 0.026699754804712122, 'reg_lambda': 2.079328254162567e-06, 'min_child_weight': 8, 'gamma': 0.27211990552088783, 'num_boost_round': 680}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 2. [I 2024-01-24 22:40:56,910] Trial 173 finished with value: 0.7495523151701202 and parameters: {'max_depth': 19, 'learning_rate': 0.06852535553109501, 'subsample': 0.8142291200127127, 'colsample_bytree': 0.761985965271594, 'reg_alpha': 0.01608942918515435, 'reg_lambda': 1.4710947888626936e-05, 'min_child_weight': 12, 'gamma': 0.29303790141636005, 'num_boost_round': 731}. Best is trial 173 with value: 0.7495523151701202.\n",
    "# # 3. [I 2024-01-24 22:03:44,975] Trial 66 finished with value: 0.7492197492964953 and parameters: {'max_depth': 14, 'learning_rate': 0.08072418755309368, 'subsample': 0.8763456228289015, 'colsample_bytree': 0.7778282818904712, 'reg_alpha': 0.003225029142847914, 'reg_lambda': 7.784882855510882e-05, 'min_child_weight': 13, 'gamma': 0.1845128349290207, 'num_boost_round': 727}. Best is trial 66 with value: 0.7492197492964953.\n",
    "# # 4. [I 2024-01-24 21:59:01,029] Trial 54 finished with value: 0.7491941673062164 and parameters: {'max_depth': 12, 'learning_rate': 0.08510240645508052, 'subsample': 0.8118792369913675, 'colsample_bytree': 0.8144660820958751, 'reg_alpha': 0.0052169265809395545, 'reg_lambda': 2.1496005254384563e-05, 'min_child_weight': 13, 'gamma': 0.15679567874549755, 'num_boost_round': 685}. Best is trial 54 with value: 0.7491941673062164.\n",
    "# # 5. [I 2024-01-24 21:38:30,821] Trial 8 finished with value: 0.7486313635200819 and parameters: {'max_depth': 12, 'learning_rate': 0.05758828222614718, 'subsample': 0.7804226954425081, 'colsample_bytree': 0.8755348548132729, 'reg_alpha': 0.00017682974532703887, 'reg_lambda': 0.002043455632809802, 'min_child_weight': 12, 'gamma': 0.2367087180889414, 'num_boost_round': 647}. Best is trial 8 with value: 0.7486313635200819.\n",
    "# # 6. [I 2024-01-25 01:43:29,009] Trial 589 finished with value: 0.749424405218726 and parameters: {'max_depth': 20, 'learning_rate': 0.03800343485475528, 'subsample': 0.8202951515109791, 'colsample_bytree': 0.738661252490032, 'reg_alpha': 0.2242356003763232, 'reg_lambda': 0.0020704697932251615, 'min_child_weight': 13, 'gamma': 0.07638731745657272, 'num_boost_round': 649}. Best is trial 566 with value: 0.7500127909951394.\n",
    "# # 7. [I 2024-01-25 01:38:35,106] Trial 578 finished with value: 0.749091839345101 and parameters: {'max_depth': 19, 'learning_rate': 0.03690271078018128, 'subsample': 0.8269855360074767, 'colsample_bytree': 0.7233009713934764, 'reg_alpha': 0.17321621623842617, 'reg_lambda': 0.0006845771256753008, 'min_child_weight': 12, 'gamma': 0.14467238682175698, 'num_boost_round': 664}. Best is trial 566 with value: 0.7500127909951394.\n",
    "\n",
    "# # 8. [I 2024-01-25 01:20:05,861] Trial 534 finished with value: 0.74911742133538 and parameters: {'max_depth': 19, 'learning_rate': 0.04337748228501226, 'subsample': 0.8254008781432502, 'colsample_bytree': 0.7607409145997176, 'reg_alpha': 0.05062786196636236, 'reg_lambda': 0.00011755759434240179, 'min_child_weight': 14, 'gamma': 0.16854728464675092, 'num_boost_round': 751}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 9. [I 2024-01-25 01:18:06,265] Trial 529 finished with value: 0.7496290611409568 and parameters: {'max_depth': 18, 'learning_rate': 0.03559377553825265, 'subsample': 0.825572200337752, 'colsample_bytree': 0.7246358884874327, 'reg_alpha': 0.5222479206121815, 'reg_lambda': 0.00022562611856304275, 'min_child_weight': 13, 'gamma': 0.16700788798953314, 'num_boost_round': 720}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 10. [I 2024-01-25 01:17:04,207] Trial 527 finished with value: 0.7495523151701202 and parameters: {'max_depth': 19, 'learning_rate': 0.03777380643073618, 'subsample': 0.8268734985591043, 'colsample_bytree': 0.7551672310239684, 'reg_alpha': 0.0954073906788678, 'reg_lambda': 7.716334032574376e-05, 'min_child_weight': 12, 'gamma': 0.1987798231252018, 'num_boost_round': 728}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 11. [I 2024-01-25 01:15:25,117] Trial 524 finished with value: 0.7490406753645433 and parameters: {'max_depth': 18, 'learning_rate': 0.042656936444501706, 'subsample': 0.8238608907524858, 'colsample_bytree': 0.7493009881293785, 'reg_alpha': 0.9255446138049785, 'reg_lambda': 0.0009087495291381847, 'min_child_weight': 13, 'gamma': 0.1060124184848753, 'num_boost_round': 775}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 12. [I 2024-01-25 01:12:46,909] Trial 519 finished with value: 0.749091839345101 and parameters: {'max_depth': 19, 'learning_rate': 0.038262703503635066, 'subsample': 0.8252605400529552, 'colsample_bytree': 0.773969163547315, 'reg_alpha': 0.10931342695413239, 'reg_lambda': 0.0004657496400790118, 'min_child_weight': 11, 'gamma': 0.16224800867614694, 'num_boost_round': 773}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 13. [I 2024-01-25 01:10:41,601] Trial 515 finished with value: 0.749091839345101 and parameters: {'max_depth': 20, 'learning_rate': 0.03901218723700183, 'subsample': 0.8081442759647638, 'colsample_bytree': 0.7110060770246625, 'reg_alpha': 0.09776966139611619, 'reg_lambda': 0.00011872123338303836, 'min_child_weight': 14, 'gamma': 0.11033818326362488, 'num_boost_round': 679}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 14. [I 2024-01-25 01:08:57,357] Trial 512 finished with value: 0.7491430033256588 and parameters: {'max_depth': 19, 'learning_rate': 0.03794093740819866, 'subsample': 0.8243302463210778, 'colsample_bytree': 0.7581050205518861, 'reg_alpha': 0.4157919570151736, 'reg_lambda': 0.000292465051823018, 'min_child_weight': 12, 'gamma': 0.15841116001980188, 'num_boost_round': 787}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 15. [I 2024-01-25 01:03:05,886] Trial 500 finished with value: 0.7490406753645433 and parameters: {'max_depth': 19, 'learning_rate': 0.04211408805214449, 'subsample': 0.8209966390375378, 'colsample_bytree': 0.7516431544682141, 'reg_alpha': 0.16791966160729843, 'reg_lambda': 0.0003699346040178164, 'min_child_weight': 13, 'gamma': 0.1306439754724737, 'num_boost_round': 796}. Best is trial 250 with value: 0.7498848810437452.\n",
    "\n",
    "# # 16. [I 2024-01-25 00:56:39,537] Trial 488 finished with value: 0.7493220772576106 and parameters: {'max_depth': 18, 'learning_rate': 0.031086245113979383, 'subsample': 0.8554882405655578, 'colsample_bytree': 0.7634830821567467, 'reg_alpha': 0.13536133073726545, 'reg_lambda': 0.00020792364882006904, 'min_child_weight': 12, 'gamma': 0.12373727131876128, 'num_boost_round': 836}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 17. [I 2024-01-25 00:50:17,334] Trial 476 finished with value: 0.7490150933742645 and parameters: {'max_depth': 20, 'learning_rate': 0.026036363785614178, 'subsample': 0.8438358217232308, 'colsample_bytree': 0.7704072531466309, 'reg_alpha': 0.11480760469479345, 'reg_lambda': 0.00011417867992155103, 'min_child_weight': 11, 'gamma': 0.12741945208978078, 'num_boost_round': 798}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 18. [I 2024-01-25 00:01:41,936] Trial 389 finished with value: 0.7491685853159376 and parameters: {'max_depth': 19, 'learning_rate': 0.0313588592826631, 'subsample': 0.830229868310942, 'colsample_bytree': 0.7461967990241131, 'reg_alpha': 0.018966919233564618, 'reg_lambda': 1.3191606835570239e-05, 'min_child_weight': 11, 'gamma': 0.13199649255166215, 'num_boost_round': 706}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 19. [I 2024-01-24 23:15:57,669] Trial 293 finished with value: 0.749091839345101 and parameters: {'max_depth': 19, 'learning_rate': 0.05429824837408812, 'subsample': 0.799197551215953, 'colsample_bytree': 0.7434159715642802, 'reg_alpha': 0.022325362316784458, 'reg_lambda': 2.5583426571902403e-05, 'min_child_weight': 12, 'gamma': 0.16660268463376066, 'num_boost_round': 704}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 20. [I 2024-01-24 23:04:50,282] Trial 256 finished with value: 0.749577897160399 and parameters: {'max_depth': 18, 'learning_rate': 0.10167138577038133, 'subsample': 0.8163202577478864, 'colsample_bytree': 0.7094428404341258, 'reg_alpha': 0.01031993947063547, 'reg_lambda': 1.6163969849635279e-06, 'min_child_weight': 11, 'gamma': 0.35938090529360256, 'num_boost_round': 689}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 21. [I 2024-01-24 22:58:34,759] Trial 236 finished with value: 0.749091839345101 and parameters: {'max_depth': 17, 'learning_rate': 0.06755970424987041, 'subsample': 0.8217113084043229, 'colsample_bytree': 0.7549707111995404, 'reg_alpha': 0.04323825951344119, 'reg_lambda': 6.149413011693923e-06, 'min_child_weight': 10, 'gamma': 0.2505610034254141, 'num_boost_round': 699}. Best is trial 173 with value: 0.7495523151701202.\n",
    "# # 22. [I 2024-01-24 22:19:59,904] Trial 111 finished with value: 0.749091839345101 and parameters: {'max_depth': 17, 'learning_rate': 0.07690575672906713, 'subsample': 0.8304088943416862, 'colsample_bytree': 0.8032684114382151, 'reg_alpha': 0.15305796613509787, 'reg_lambda': 7.2352676380118895e-06, 'min_child_weight': 15, 'gamma': 0.29252545702316546, 'num_boost_round': 679}. Best is trial 66 with value: 0.7492197492964953.\n",
    "\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Define the parameters for each model\n",
    "# params = [\n",
    "#     {'max_depth': 20, 'learning_rate': 0.06320245480949405, 'subsample': 0.8188272349235466, 'colsample_bytree': 0.7127641606598758, 'reg_alpha': 0.08424063460596642, 'reg_lambda': 0.0010426300127364537, 'min_child_weight': 15, 'gamma': 0.1279168805879946},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.07282838028748469, 'subsample': 0.8022684987091674, 'colsample_bytree': 0.7188889119870725, 'reg_alpha': 0.026699754804712122, 'reg_lambda': 2.079328254162567e-06, 'min_child_weight': 8, 'gamma': 0.27211990552088783},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.06852535553109501, 'subsample': 0.8142291200127127, 'colsample_bytree': 0.761985965271594, 'reg_alpha': 0.01608942918515435, 'reg_lambda': 1.4710947888626936e-05, 'min_child_weight': 12, 'gamma': 0.29303790141636005},\n",
    "#     {'max_depth': 14, 'learning_rate': 0.08072418755309368, 'subsample': 0.8763456228289015, 'colsample_bytree': 0.7778282818904712, 'reg_alpha': 0.003225029142847914, 'reg_lambda': 7.784882855510882e-05, 'min_child_weight': 13, 'gamma': 0.1845128349290207},\n",
    "#     {'max_depth': 12, 'learning_rate': 0.08510240645508052, 'subsample': 0.8118792369913675, 'colsample_bytree': 0.8144660820958751, 'reg_alpha': 0.0052169265809395545, 'reg_lambda': 2.1496005254384563e-05, 'min_child_weight': 13, 'gamma': 0.15679567874549755},\n",
    "#     {'max_depth': 12, 'learning_rate': 0.05758828222614718, 'subsample': 0.7804226954425081, 'colsample_bytree': 0.8755348548132729, 'reg_alpha': 0.00017682974532703887, 'reg_lambda': 0.002043455632809802, 'min_child_weight': 12, 'gamma': 0.2367087180889414},\n",
    "#     {'max_depth': 20, 'learning_rate': 0.03800343485475528, 'subsample': 0.8202951515109791, 'colsample_bytree': 0.738661252490032, 'reg_alpha': 0.2242356003763232, 'reg_lambda': 0.0020704697932251615, 'min_child_weight': 13, 'gamma': 0.07638731745657272},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.03690271078018128, 'subsample': 0.8269855360074767, 'colsample_bytree': 0.7233009713934764, 'reg_alpha': 0.17321621623842617, 'reg_lambda': 0.0006845771256753008, 'min_child_weight': 12, 'gamma': 0.14467238682175698},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.04337748228501226, 'subsample': 0.8254008781432502, 'colsample_bytree': 0.7607409145997176, 'reg_alpha': 0.05062786196636236, 'reg_lambda': 0.00011755759434240179, 'min_child_weight': 14, 'gamma': 0.16854728464675092},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.03559377553825265, 'subsample': 0.825572200337752, 'colsample_bytree': 0.7246358884874327, 'reg_alpha': 0.5222479206121815, 'reg_lambda': 0.00022562611856304275, 'min_child_weight': 13, 'gamma': 0.16700788798953314},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.03777380643073618, 'subsample': 0.8268734985591043, 'colsample_bytree': 0.7551672310239684, 'reg_alpha': 0.0954073906788678, 'reg_lambda': 7.716334032574376e-05, 'min_child_weight': 12, 'gamma': 0.1987798231252018},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.042656936444501706, 'subsample': 0.8238608907524858, 'colsample_bytree': 0.7493009881293785, 'reg_alpha': 0.9255446138049785, 'reg_lambda': 0.0009087495291381847, 'min_child_weight': 13, 'gamma': 0.1060124184848753},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.038262703503635066, 'subsample': 0.8252605400529552, 'colsample_bytree': 0.773969163547315, 'reg_alpha': 0.10931342695413239, 'reg_lambda': 0.0004657496400790118, 'min_child_weight': 11, 'gamma': 0.16224800867614694},\n",
    "#     {'max_depth': 20, 'learning_rate': 0.03901218723700183, 'subsample': 0.8081442759647638, 'colsample_bytree': 0.7110060770246625, 'reg_alpha': 0.09776966139611619, 'reg_lambda': 0.00011872123338303836, 'min_child_weight': 14, 'gamma': 0.11033818326362488},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.03794093740819866, 'subsample': 0.8243302463210778, 'colsample_bytree': 0.7581050205518861, 'reg_alpha': 0.4157919570151736, 'reg_lambda': 0.000292465051823018, 'min_child_weight': 12, 'gamma': 0.15841116001980188},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.04211408805214449, 'subsample': 0.8209966390375378, 'colsample_bytree': 0.7516431544682141, 'reg_alpha': 0.16791966160729843, 'reg_lambda': 0.0003699346040178164, 'min_child_weight': 13, 'gamma': 0.1306439754724737},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.031086245113979383, 'subsample': 0.8554882405655578, 'colsample_bytree': 0.7634830821567467, 'reg_alpha': 0.13536133073726545, 'reg_lambda': 0.00020792364882006904, 'min_child_weight': 12, 'gamma': 0.12373727131876128},\n",
    "#     {'max_depth': 20, 'learning_rate': 0.026036363785614178, 'subsample': 0.8438358217232308, 'colsample_bytree': 0.7704072531466309, 'reg_alpha': 0.11480760469479345, 'reg_lambda': 0.00011417867992155103, 'min_child_weight': 11, 'gamma': 0.12741945208978078},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.0313588592826631, 'subsample': 0.830229868310942, 'colsample_bytree': 0.7461967990241131, 'reg_alpha': 0.018966919233564618, 'reg_lambda': 1.3191606835570239e-05, 'min_child_weight': 11, 'gamma': 0.13199649255166215},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.05429824837408812, 'subsample': 0.799197551215953, 'colsample_bytree': 0.7434159715642802, 'reg_alpha': 0.022325362316784458, 'reg_lambda': 2.5583426571902403e-05, 'min_child_weight': 12, 'gamma': 0.16660268463376066},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.10167138577038133, 'subsample': 0.8163202577478864, 'colsample_bytree': 0.7094428404341258, 'reg_alpha': 0.01031993947063547, 'reg_lambda': 1.6163969849635279e-06, 'min_child_weight': 11, 'gamma': 0.35938090529360256},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.06755970424987041, 'subsample': 0.8217113084043229, 'colsample_bytree': 0.7549707111995404, 'reg_alpha': 0.04323825951344119, 'reg_lambda': 6.149413011693923e-06, 'min_child_weight': 10, 'gamma': 0.2505610034254141},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.07690575672906713, 'subsample': 0.8304088943416862, 'colsample_bytree': 0.8032684114382151, 'reg_alpha': 0.15305796613509787, 'reg_lambda': 7.2352676380118895e-06, 'min_child_weight': 15, 'gamma': 0.29252545702316546},\n",
    "# ]\n",
    "\n",
    "# # This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# # The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "# for param in params:\n",
    "#     param.update({\n",
    "#         #'tree_method': 'hist', # I trained the model in githubcodespaces with default tree_method of auto\n",
    "#         'device': 'cpu', # use CPU since GitHubCodeSpaces optuna training didn't have GPU so it trained on CPU\n",
    "#         #'predictor': 'cpu_predictor', # predictor hyperparameter is not used anymore in xgb. outdated\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "#     })\n",
    "\n",
    "\n",
    "# num_boost_round = [659, 680, 731, 727, 685, 647, 649, 664, 751, 720, 728, 775, 773, 679, 787, 796, 836, 798, 706, 704, 689, 699, 679]\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "# for i in range(len(params)):\n",
    "\n",
    "#     # Create the XGBoost models\n",
    "#     model24 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "#     # Save the model to a file in JSON format\n",
    "#     #model24.save_model('saved_XGBoost_model24.json')\n",
    "\n",
    "\n",
    "#     # # Load the model from the bin (binary) file\n",
    "#     # model24 = xgb.Booster()\n",
    "#     # model24.load_model('saved_XGBoost_model24.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "#     # Predict on the competition test data\n",
    "#     competition_y_pred = model24.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "#     competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "#     # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "#     competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "#     # Create a DataFrame for submission\n",
    "#     submission = pd.DataFrame({\n",
    "#         'building_id': competition_test_building_ids,\n",
    "#         'damage_grade': competition_y_pred\n",
    "#     })\n",
    "\n",
    "#     # Save the submission DataFrame to a CSV file for submission\n",
    "#     submission.to_csv(f\"submission_XGBoost_24_{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "#     print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 25, same as Model 24 but with CPU instead of GPU, from \"Optuna_Logs_XGBoost_Model21_GitHubCodeSpaces.txt.\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # can you make 8 seperate XGB models for each line? And make it so that n_estimators is it's own variable for each set of param. So like n_estimator_1 and n_estimator_2\n",
    "\n",
    "# # 0. [I 2024-01-25 01:33:30,957] Trial 566 finished with value: 0.7500127909951394 and parameters: {'max_depth': 20, 'learning_rate': 0.06320245480949405, 'subsample': 0.8188272349235466, 'colsample_bytree': 0.7127641606598758, 'reg_alpha': 0.08424063460596642, 'reg_lambda': 0.0010426300127364537, 'min_child_weight': 15, 'gamma': 0.1279168805879946, 'num_boost_round': 659}. Best is trial 566 with value: 0.7500127909951394.\n",
    "# # 1. [I 2024-01-24 23:02:49,314] Trial 250 finished with value: 0.7498848810437452 and parameters: {'max_depth': 18, 'learning_rate': 0.07282838028748469, 'subsample': 0.8022684987091674, 'colsample_bytree': 0.7188889119870725, 'reg_alpha': 0.026699754804712122, 'reg_lambda': 2.079328254162567e-06, 'min_child_weight': 8, 'gamma': 0.27211990552088783, 'num_boost_round': 680}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 2. [I 2024-01-24 22:40:56,910] Trial 173 finished with value: 0.7495523151701202 and parameters: {'max_depth': 19, 'learning_rate': 0.06852535553109501, 'subsample': 0.8142291200127127, 'colsample_bytree': 0.761985965271594, 'reg_alpha': 0.01608942918515435, 'reg_lambda': 1.4710947888626936e-05, 'min_child_weight': 12, 'gamma': 0.29303790141636005, 'num_boost_round': 731}. Best is trial 173 with value: 0.7495523151701202.\n",
    "# # 3. [I 2024-01-24 22:03:44,975] Trial 66 finished with value: 0.7492197492964953 and parameters: {'max_depth': 14, 'learning_rate': 0.08072418755309368, 'subsample': 0.8763456228289015, 'colsample_bytree': 0.7778282818904712, 'reg_alpha': 0.003225029142847914, 'reg_lambda': 7.784882855510882e-05, 'min_child_weight': 13, 'gamma': 0.1845128349290207, 'num_boost_round': 727}. Best is trial 66 with value: 0.7492197492964953.\n",
    "# # 4. [I 2024-01-24 21:59:01,029] Trial 54 finished with value: 0.7491941673062164 and parameters: {'max_depth': 12, 'learning_rate': 0.08510240645508052, 'subsample': 0.8118792369913675, 'colsample_bytree': 0.8144660820958751, 'reg_alpha': 0.0052169265809395545, 'reg_lambda': 2.1496005254384563e-05, 'min_child_weight': 13, 'gamma': 0.15679567874549755, 'num_boost_round': 685}. Best is trial 54 with value: 0.7491941673062164.\n",
    "# # 5. [I 2024-01-24 21:38:30,821] Trial 8 finished with value: 0.7486313635200819 and parameters: {'max_depth': 12, 'learning_rate': 0.05758828222614718, 'subsample': 0.7804226954425081, 'colsample_bytree': 0.8755348548132729, 'reg_alpha': 0.00017682974532703887, 'reg_lambda': 0.002043455632809802, 'min_child_weight': 12, 'gamma': 0.2367087180889414, 'num_boost_round': 647}. Best is trial 8 with value: 0.7486313635200819.\n",
    "# # 6. [I 2024-01-25 01:43:29,009] Trial 589 finished with value: 0.749424405218726 and parameters: {'max_depth': 20, 'learning_rate': 0.03800343485475528, 'subsample': 0.8202951515109791, 'colsample_bytree': 0.738661252490032, 'reg_alpha': 0.2242356003763232, 'reg_lambda': 0.0020704697932251615, 'min_child_weight': 13, 'gamma': 0.07638731745657272, 'num_boost_round': 649}. Best is trial 566 with value: 0.7500127909951394.\n",
    "# # 7. [I 2024-01-25 01:38:35,106] Trial 578 finished with value: 0.749091839345101 and parameters: {'max_depth': 19, 'learning_rate': 0.03690271078018128, 'subsample': 0.8269855360074767, 'colsample_bytree': 0.7233009713934764, 'reg_alpha': 0.17321621623842617, 'reg_lambda': 0.0006845771256753008, 'min_child_weight': 12, 'gamma': 0.14467238682175698, 'num_boost_round': 664}. Best is trial 566 with value: 0.7500127909951394.\n",
    "\n",
    "# # 8. [I 2024-01-25 01:20:05,861] Trial 534 finished with value: 0.74911742133538 and parameters: {'max_depth': 19, 'learning_rate': 0.04337748228501226, 'subsample': 0.8254008781432502, 'colsample_bytree': 0.7607409145997176, 'reg_alpha': 0.05062786196636236, 'reg_lambda': 0.00011755759434240179, 'min_child_weight': 14, 'gamma': 0.16854728464675092, 'num_boost_round': 751}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 9. [I 2024-01-25 01:18:06,265] Trial 529 finished with value: 0.7496290611409568 and parameters: {'max_depth': 18, 'learning_rate': 0.03559377553825265, 'subsample': 0.825572200337752, 'colsample_bytree': 0.7246358884874327, 'reg_alpha': 0.5222479206121815, 'reg_lambda': 0.00022562611856304275, 'min_child_weight': 13, 'gamma': 0.16700788798953314, 'num_boost_round': 720}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 10. [I 2024-01-25 01:17:04,207] Trial 527 finished with value: 0.7495523151701202 and parameters: {'max_depth': 19, 'learning_rate': 0.03777380643073618, 'subsample': 0.8268734985591043, 'colsample_bytree': 0.7551672310239684, 'reg_alpha': 0.0954073906788678, 'reg_lambda': 7.716334032574376e-05, 'min_child_weight': 12, 'gamma': 0.1987798231252018, 'num_boost_round': 728}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 11. [I 2024-01-25 01:15:25,117] Trial 524 finished with value: 0.7490406753645433 and parameters: {'max_depth': 18, 'learning_rate': 0.042656936444501706, 'subsample': 0.8238608907524858, 'colsample_bytree': 0.7493009881293785, 'reg_alpha': 0.9255446138049785, 'reg_lambda': 0.0009087495291381847, 'min_child_weight': 13, 'gamma': 0.1060124184848753, 'num_boost_round': 775}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 12. [I 2024-01-25 01:12:46,909] Trial 519 finished with value: 0.749091839345101 and parameters: {'max_depth': 19, 'learning_rate': 0.038262703503635066, 'subsample': 0.8252605400529552, 'colsample_bytree': 0.773969163547315, 'reg_alpha': 0.10931342695413239, 'reg_lambda': 0.0004657496400790118, 'min_child_weight': 11, 'gamma': 0.16224800867614694, 'num_boost_round': 773}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 13. [I 2024-01-25 01:10:41,601] Trial 515 finished with value: 0.749091839345101 and parameters: {'max_depth': 20, 'learning_rate': 0.03901218723700183, 'subsample': 0.8081442759647638, 'colsample_bytree': 0.7110060770246625, 'reg_alpha': 0.09776966139611619, 'reg_lambda': 0.00011872123338303836, 'min_child_weight': 14, 'gamma': 0.11033818326362488, 'num_boost_round': 679}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 14. [I 2024-01-25 01:08:57,357] Trial 512 finished with value: 0.7491430033256588 and parameters: {'max_depth': 19, 'learning_rate': 0.03794093740819866, 'subsample': 0.8243302463210778, 'colsample_bytree': 0.7581050205518861, 'reg_alpha': 0.4157919570151736, 'reg_lambda': 0.000292465051823018, 'min_child_weight': 12, 'gamma': 0.15841116001980188, 'num_boost_round': 787}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 15. [I 2024-01-25 01:03:05,886] Trial 500 finished with value: 0.7490406753645433 and parameters: {'max_depth': 19, 'learning_rate': 0.04211408805214449, 'subsample': 0.8209966390375378, 'colsample_bytree': 0.7516431544682141, 'reg_alpha': 0.16791966160729843, 'reg_lambda': 0.0003699346040178164, 'min_child_weight': 13, 'gamma': 0.1306439754724737, 'num_boost_round': 796}. Best is trial 250 with value: 0.7498848810437452.\n",
    "\n",
    "# # 16. [I 2024-01-25 00:56:39,537] Trial 488 finished with value: 0.7493220772576106 and parameters: {'max_depth': 18, 'learning_rate': 0.031086245113979383, 'subsample': 0.8554882405655578, 'colsample_bytree': 0.7634830821567467, 'reg_alpha': 0.13536133073726545, 'reg_lambda': 0.00020792364882006904, 'min_child_weight': 12, 'gamma': 0.12373727131876128, 'num_boost_round': 836}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 17. [I 2024-01-25 00:50:17,334] Trial 476 finished with value: 0.7490150933742645 and parameters: {'max_depth': 20, 'learning_rate': 0.026036363785614178, 'subsample': 0.8438358217232308, 'colsample_bytree': 0.7704072531466309, 'reg_alpha': 0.11480760469479345, 'reg_lambda': 0.00011417867992155103, 'min_child_weight': 11, 'gamma': 0.12741945208978078, 'num_boost_round': 798}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 18. [I 2024-01-25 00:01:41,936] Trial 389 finished with value: 0.7491685853159376 and parameters: {'max_depth': 19, 'learning_rate': 0.0313588592826631, 'subsample': 0.830229868310942, 'colsample_bytree': 0.7461967990241131, 'reg_alpha': 0.018966919233564618, 'reg_lambda': 1.3191606835570239e-05, 'min_child_weight': 11, 'gamma': 0.13199649255166215, 'num_boost_round': 706}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 19. [I 2024-01-24 23:15:57,669] Trial 293 finished with value: 0.749091839345101 and parameters: {'max_depth': 19, 'learning_rate': 0.05429824837408812, 'subsample': 0.799197551215953, 'colsample_bytree': 0.7434159715642802, 'reg_alpha': 0.022325362316784458, 'reg_lambda': 2.5583426571902403e-05, 'min_child_weight': 12, 'gamma': 0.16660268463376066, 'num_boost_round': 704}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 20. [I 2024-01-24 23:04:50,282] Trial 256 finished with value: 0.749577897160399 and parameters: {'max_depth': 18, 'learning_rate': 0.10167138577038133, 'subsample': 0.8163202577478864, 'colsample_bytree': 0.7094428404341258, 'reg_alpha': 0.01031993947063547, 'reg_lambda': 1.6163969849635279e-06, 'min_child_weight': 11, 'gamma': 0.35938090529360256, 'num_boost_round': 689}. Best is trial 250 with value: 0.7498848810437452.\n",
    "# # 21. [I 2024-01-24 22:58:34,759] Trial 236 finished with value: 0.749091839345101 and parameters: {'max_depth': 17, 'learning_rate': 0.06755970424987041, 'subsample': 0.8217113084043229, 'colsample_bytree': 0.7549707111995404, 'reg_alpha': 0.04323825951344119, 'reg_lambda': 6.149413011693923e-06, 'min_child_weight': 10, 'gamma': 0.2505610034254141, 'num_boost_round': 699}. Best is trial 173 with value: 0.7495523151701202.\n",
    "# # 22. [I 2024-01-24 22:19:59,904] Trial 111 finished with value: 0.749091839345101 and parameters: {'max_depth': 17, 'learning_rate': 0.07690575672906713, 'subsample': 0.8304088943416862, 'colsample_bytree': 0.8032684114382151, 'reg_alpha': 0.15305796613509787, 'reg_lambda': 7.2352676380118895e-06, 'min_child_weight': 15, 'gamma': 0.29252545702316546, 'num_boost_round': 679}. Best is trial 66 with value: 0.7492197492964953.\n",
    "\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Define the parameters for each model\n",
    "# params = [\n",
    "#     {'max_depth': 20, 'learning_rate': 0.06320245480949405, 'subsample': 0.8188272349235466, 'colsample_bytree': 0.7127641606598758, 'reg_alpha': 0.08424063460596642, 'reg_lambda': 0.0010426300127364537, 'min_child_weight': 15, 'gamma': 0.1279168805879946},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.07282838028748469, 'subsample': 0.8022684987091674, 'colsample_bytree': 0.7188889119870725, 'reg_alpha': 0.026699754804712122, 'reg_lambda': 2.079328254162567e-06, 'min_child_weight': 8, 'gamma': 0.27211990552088783},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.06852535553109501, 'subsample': 0.8142291200127127, 'colsample_bytree': 0.761985965271594, 'reg_alpha': 0.01608942918515435, 'reg_lambda': 1.4710947888626936e-05, 'min_child_weight': 12, 'gamma': 0.29303790141636005},\n",
    "#     {'max_depth': 14, 'learning_rate': 0.08072418755309368, 'subsample': 0.8763456228289015, 'colsample_bytree': 0.7778282818904712, 'reg_alpha': 0.003225029142847914, 'reg_lambda': 7.784882855510882e-05, 'min_child_weight': 13, 'gamma': 0.1845128349290207},\n",
    "#     {'max_depth': 12, 'learning_rate': 0.08510240645508052, 'subsample': 0.8118792369913675, 'colsample_bytree': 0.8144660820958751, 'reg_alpha': 0.0052169265809395545, 'reg_lambda': 2.1496005254384563e-05, 'min_child_weight': 13, 'gamma': 0.15679567874549755},\n",
    "#     {'max_depth': 12, 'learning_rate': 0.05758828222614718, 'subsample': 0.7804226954425081, 'colsample_bytree': 0.8755348548132729, 'reg_alpha': 0.00017682974532703887, 'reg_lambda': 0.002043455632809802, 'min_child_weight': 12, 'gamma': 0.2367087180889414},\n",
    "#     {'max_depth': 20, 'learning_rate': 0.03800343485475528, 'subsample': 0.8202951515109791, 'colsample_bytree': 0.738661252490032, 'reg_alpha': 0.2242356003763232, 'reg_lambda': 0.0020704697932251615, 'min_child_weight': 13, 'gamma': 0.07638731745657272},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.03690271078018128, 'subsample': 0.8269855360074767, 'colsample_bytree': 0.7233009713934764, 'reg_alpha': 0.17321621623842617, 'reg_lambda': 0.0006845771256753008, 'min_child_weight': 12, 'gamma': 0.14467238682175698},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.04337748228501226, 'subsample': 0.8254008781432502, 'colsample_bytree': 0.7607409145997176, 'reg_alpha': 0.05062786196636236, 'reg_lambda': 0.00011755759434240179, 'min_child_weight': 14, 'gamma': 0.16854728464675092},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.03559377553825265, 'subsample': 0.825572200337752, 'colsample_bytree': 0.7246358884874327, 'reg_alpha': 0.5222479206121815, 'reg_lambda': 0.00022562611856304275, 'min_child_weight': 13, 'gamma': 0.16700788798953314},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.03777380643073618, 'subsample': 0.8268734985591043, 'colsample_bytree': 0.7551672310239684, 'reg_alpha': 0.0954073906788678, 'reg_lambda': 7.716334032574376e-05, 'min_child_weight': 12, 'gamma': 0.1987798231252018},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.042656936444501706, 'subsample': 0.8238608907524858, 'colsample_bytree': 0.7493009881293785, 'reg_alpha': 0.9255446138049785, 'reg_lambda': 0.0009087495291381847, 'min_child_weight': 13, 'gamma': 0.1060124184848753},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.038262703503635066, 'subsample': 0.8252605400529552, 'colsample_bytree': 0.773969163547315, 'reg_alpha': 0.10931342695413239, 'reg_lambda': 0.0004657496400790118, 'min_child_weight': 11, 'gamma': 0.16224800867614694},\n",
    "#     {'max_depth': 20, 'learning_rate': 0.03901218723700183, 'subsample': 0.8081442759647638, 'colsample_bytree': 0.7110060770246625, 'reg_alpha': 0.09776966139611619, 'reg_lambda': 0.00011872123338303836, 'min_child_weight': 14, 'gamma': 0.11033818326362488},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.03794093740819866, 'subsample': 0.8243302463210778, 'colsample_bytree': 0.7581050205518861, 'reg_alpha': 0.4157919570151736, 'reg_lambda': 0.000292465051823018, 'min_child_weight': 12, 'gamma': 0.15841116001980188},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.04211408805214449, 'subsample': 0.8209966390375378, 'colsample_bytree': 0.7516431544682141, 'reg_alpha': 0.16791966160729843, 'reg_lambda': 0.0003699346040178164, 'min_child_weight': 13, 'gamma': 0.1306439754724737},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.031086245113979383, 'subsample': 0.8554882405655578, 'colsample_bytree': 0.7634830821567467, 'reg_alpha': 0.13536133073726545, 'reg_lambda': 0.00020792364882006904, 'min_child_weight': 12, 'gamma': 0.12373727131876128},\n",
    "#     {'max_depth': 20, 'learning_rate': 0.026036363785614178, 'subsample': 0.8438358217232308, 'colsample_bytree': 0.7704072531466309, 'reg_alpha': 0.11480760469479345, 'reg_lambda': 0.00011417867992155103, 'min_child_weight': 11, 'gamma': 0.12741945208978078},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.0313588592826631, 'subsample': 0.830229868310942, 'colsample_bytree': 0.7461967990241131, 'reg_alpha': 0.018966919233564618, 'reg_lambda': 1.3191606835570239e-05, 'min_child_weight': 11, 'gamma': 0.13199649255166215},\n",
    "#     {'max_depth': 19, 'learning_rate': 0.05429824837408812, 'subsample': 0.799197551215953, 'colsample_bytree': 0.7434159715642802, 'reg_alpha': 0.022325362316784458, 'reg_lambda': 2.5583426571902403e-05, 'min_child_weight': 12, 'gamma': 0.16660268463376066},\n",
    "#     {'max_depth': 18, 'learning_rate': 0.10167138577038133, 'subsample': 0.8163202577478864, 'colsample_bytree': 0.7094428404341258, 'reg_alpha': 0.01031993947063547, 'reg_lambda': 1.6163969849635279e-06, 'min_child_weight': 11, 'gamma': 0.35938090529360256},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.06755970424987041, 'subsample': 0.8217113084043229, 'colsample_bytree': 0.7549707111995404, 'reg_alpha': 0.04323825951344119, 'reg_lambda': 6.149413011693923e-06, 'min_child_weight': 10, 'gamma': 0.2505610034254141},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.07690575672906713, 'subsample': 0.8304088943416862, 'colsample_bytree': 0.8032684114382151, 'reg_alpha': 0.15305796613509787, 'reg_lambda': 7.2352676380118895e-06, 'min_child_weight': 15, 'gamma': 0.29252545702316546},\n",
    "# ]\n",
    "\n",
    "# # This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# # The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "# for param in params:\n",
    "#     param.update({\n",
    "#         'tree_method': 'hist',\n",
    "#         'device': 'cuda',\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "#     })\n",
    "\n",
    "# num_boost_round = [659, 680, 731, 727, 685, 647, 649, 664, 751, 720, 728, 775, 773, 679, 787, 796, 836, 798, 706, 704, 689, 699, 679]\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "# for i in range(len(params)):\n",
    "\n",
    "#     # Create the XGBoost models\n",
    "#     model25 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "#     # Save the model to a file in JSON format\n",
    "#     #model25.save_model('saved_XGBoost_model25.json')\n",
    "\n",
    "\n",
    "#     # # Load the model from the bin (binary) file\n",
    "#     # model25 = xgb.Booster()\n",
    "#     # model25.load_model('saved_XGBoost_model25.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "#     # Predict on the competition test data\n",
    "#     competition_y_pred = model25.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "#     competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "#     # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "#     competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "#     # Create a DataFrame for submission\n",
    "#     submission = pd.DataFrame({\n",
    "#         'building_id': competition_test_building_ids,\n",
    "#         'damage_grade': competition_y_pred\n",
    "#     })\n",
    "\n",
    "#     # Save the submission DataFrame to a CSV file for submission\n",
    "#     submission.to_csv(f\"submission_XGBoost_25_{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "#     print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 26, from \"Optuna_Logs_XGBoost_Model21_GoogleColab.txt.\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # can you make 8 seperate XGB models for each line? And make it so that n_estimators is it's own variable for each set of param. So like n_estimator_1 and n_estimator_2\n",
    "\n",
    "# # [I 2024-01-24 19:35:51,266] Trial 222 finished with value: 0.7501662829368125 and parameters: {'max_depth': 242, 'learning_rate': 0.04393371416973458, 'subsample': 0.9439960873720802, 'colsample_bytree': 0.7003290355739655, 'reg_alpha': 0.0003237412959478171, 'reg_lambda': 0.09095702630543905, 'min_child_weight': 16, 'gamma': 0.269374952772387, 'num_boost_round': 517}. Best is trial 222 with value: 0.7501662829368125.\n",
    "# # [I 2024-01-24 19:30:37,313] Trial 206 finished with value: 0.7497058071117932 and parameters: {'max_depth': 223, 'learning_rate': 0.048939863364769194, 'subsample': 0.9475937420536319, 'colsample_bytree': 0.7060667515442907, 'reg_alpha': 0.0029684355124299463, 'reg_lambda': 0.021456480801960092, 'min_child_weight': 16, 'gamma': 0.26531910644058576, 'num_boost_round': 525}. Best is trial 206 with value: 0.7497058071117932.\n",
    "# # [I 2024-01-24 18:55:03,719] Trial 92 finished with value: 0.749424405218726 and parameters: {'max_depth': 143, 'learning_rate': 0.05993667069032727, 'subsample': 0.9637983918508871, 'colsample_bytree': 0.7419399834064909, 'reg_alpha': 0.005532485879576461, 'reg_lambda': 0.09086636000755648, 'min_child_weight': 19, 'gamma': 0.2872609748192018, 'num_boost_round': 531}. Best is trial 92 with value: 0.749424405218726.\n",
    "# # [I 2024-01-24 18:53:33,431] Trial 85 finished with value: 0.7490662573548221 and parameters: {'max_depth': 125, 'learning_rate': 0.07535772315203983, 'subsample': 0.949255407954068, 'colsample_bytree': 0.7269918820032547, 'reg_alpha': 0.005279657252171642, 'reg_lambda': 0.025915206740100248, 'min_child_weight': 19, 'gamma': 0.17212166343102384, 'num_boost_round': 527}. Best is trial 85 with value: 0.7490662573548221.\n",
    "# # [I 2024-01-24 18:33:05,218] Trial 14 finished with value: 0.7489639293937068 and parameters: {'max_depth': 165, 'learning_rate': 0.06160489226822765, 'subsample': 0.9564838150525058, 'colsample_bytree': 0.8220719375877771, 'reg_alpha': 0.894707293379902, 'reg_lambda': 1.1290231588400649e-06, 'min_child_weight': 12, 'gamma': 0.4859320416349858, 'num_boost_round': 898}. Best is trial 14 with value: 0.7489639293937068.\n",
    "\n",
    "# # [I 2024-01-24 21:49:27,125] Trial 599 finished with value: 0.749424405218726 and parameters: {'max_depth': 17, 'learning_rate': 0.058329032960236324, 'subsample': 0.837491717486544, 'colsample_bytree': 0.734244645417882, 'reg_alpha': 0.006197458206437538, 'reg_lambda': 0.2693280266743663, 'min_child_weight': 16, 'gamma': 0.22414355364915867, 'num_boost_round': 702}. Best is trial 222 with value: 0.7501662829368125.\n",
    "# # [I 2024-01-24 21:49:11,354] Trial 598 finished with value: 0.749424405218726 and parameters: {'max_depth': 191, 'learning_rate': 0.06045740493865895, 'subsample': 0.8556739826131183, 'colsample_bytree': 0.7303385703236094, 'reg_alpha': 0.0032054922789221335, 'reg_lambda': 0.3251034992865143, 'min_child_weight': 14, 'gamma': 0.22699566124078455, 'num_boost_round': 498}. Best is trial 222 with value: 0.7501662829368125.\n",
    "# # [I 2024-01-24 21:47:30,025] Trial 593 finished with value: 0.7491685853159376 and parameters: {'max_depth': 174, 'learning_rate': 0.033664165615095935, 'subsample': 0.9822358623809594, 'colsample_bytree': 0.723775182264819, 'reg_alpha': 0.0018767232743644457, 'reg_lambda': 0.001689687407182799, 'min_child_weight': 15, 'gamma': 0.2314629007519745, 'num_boost_round': 443}. Best is trial 222 with value: 0.7501662829368125.\n",
    "# # [I 2024-01-24 21:38:45,016] Trial 571 finished with value: 0.7494499872090048 and parameters: {'max_depth': 190, 'learning_rate': 0.040027036867377117, 'subsample': 0.9743889960626464, 'colsample_bytree': 0.7274095238501245, 'reg_alpha': 0.005472212990485103, 'reg_lambda': 0.11103156230760185, 'min_child_weight': 14, 'gamma': 0.18049174864340387, 'num_boost_round': 465}. Best is trial 222 with value: 0.7501662829368125.\n",
    "# # [I 2024-01-24 19:41:49,015] Trial 241 finished with value: 0.7491941673062164 and parameters: {'max_depth': 104, 'learning_rate': 0.05965888861863485, 'subsample': 0.9655698429621326, 'colsample_bytree': 0.7178870863068045, 'reg_alpha': 0.6193618691386176, 'reg_lambda': 0.07086922170445695, 'min_child_weight': 17, 'gamma': 0.28253415114875857, 'num_boost_round': 506}. Best is trial 222 with value: 0.7501662829368125.\n",
    "# # [I 2024-01-24 19:41:33,838] Trial 240 finished with value: 0.7495011511895626 and parameters: {'max_depth': 276, 'learning_rate': 0.060973641350517874, 'subsample': 0.9648951661189773, 'colsample_bytree': 0.719309089328096, 'reg_alpha': 0.702865687561501, 'reg_lambda': 0.07096053408883235, 'min_child_weight': 17, 'gamma': 0.2783358993030983, 'num_boost_round': 507}. Best is trial 222 with value: 0.7501662829368125.\n",
    "\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Define the parameters for each model\n",
    "# params = [\n",
    "#     {'max_depth': 242, 'learning_rate': 0.04393371416973458, 'subsample': 0.9439960873720802, 'colsample_bytree': 0.7003290355739655, 'reg_alpha': 0.0003237412959478171, 'reg_lambda': 0.09095702630543905, 'min_child_weight': 16, 'gamma': 0.269374952772387},\n",
    "#     {'max_depth': 223, 'learning_rate': 0.048939863364769194, 'subsample': 0.9475937420536319, 'colsample_bytree': 0.7060667515442907, 'reg_alpha': 0.0029684355124299463, 'reg_lambda': 0.021456480801960092, 'min_child_weight': 16, 'gamma': 0.26531910644058576},\n",
    "#     {'max_depth': 143, 'learning_rate': 0.05993667069032727, 'subsample': 0.9637983918508871, 'colsample_bytree': 0.7419399834064909, 'reg_alpha': 0.005532485879576461, 'reg_lambda': 0.09086636000755648, 'min_child_weight': 19, 'gamma': 0.2872609748192018},\n",
    "#     {'max_depth': 125, 'learning_rate': 0.07535772315203983, 'subsample': 0.949255407954068, 'colsample_bytree': 0.7269918820032547, 'reg_alpha': 0.005279657252171642, 'reg_lambda': 0.025915206740100248, 'min_child_weight': 19, 'gamma': 0.17212166343102384},\n",
    "#     {'max_depth': 165, 'learning_rate': 0.06160489226822765, 'subsample': 0.9564838150525058, 'colsample_bytree': 0.8220719375877771, 'reg_alpha': 0.894707293379902, 'reg_lambda': 1.1290231588400649e-06, 'min_child_weight': 12, 'gamma': 0.4859320416349858},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.058329032960236324, 'subsample': 0.837491717486544, 'colsample_bytree': 0.734244645417882, 'reg_alpha': 0.006197458206437538, 'reg_lambda': 0.2693280266743663, 'min_child_weight': 16, 'gamma': 0.22414355364915867},\n",
    "#     {'max_depth': 191, 'learning_rate': 0.06045740493865895, 'subsample': 0.8556739826131183, 'colsample_bytree': 0.7303385703236094, 'reg_alpha': 0.0032054922789221335, 'reg_lambda': 0.3251034992865143, 'min_child_weight': 14, 'gamma': 0.22699566124078455},\n",
    "#     {'max_depth': 174, 'learning_rate': 0.033664165615095935, 'subsample': 0.9822358623809594, 'colsample_bytree': 0.723775182264819, 'reg_alpha': 0.0018767232743644457, 'reg_lambda': 0.001689687407182799, 'min_child_weight': 15, 'gamma': 0.2314629007519745},\n",
    "#     {'max_depth': 190, 'learning_rate': 0.040027036867377117, 'subsample': 0.9743889960626464, 'colsample_bytree': 0.7274095238501245, 'reg_alpha': 0.005472212990485103, 'reg_lambda': 0.11103156230760185, 'min_child_weight': 14, 'gamma': 0.18049174864340387},\n",
    "#     {'max_depth': 104, 'learning_rate': 0.05965888861863485, 'subsample': 0.9655698429621326, 'colsample_bytree': 0.7178870863068045, 'reg_alpha': 0.6193618691386176, 'reg_lambda': 0.07086922170445695, 'min_child_weight': 17, 'gamma': 0.28253415114875857},\n",
    "#     {'max_depth': 276, 'learning_rate': 0.060973641350517874, 'subsample': 0.9648951661189773, 'colsample_bytree': 0.719309089328096, 'reg_alpha': 0.702865687561501, 'reg_lambda': 0.07096053408883235, 'min_child_weight': 17, 'gamma': 0.2783358993030983},\n",
    "# ]\n",
    "\n",
    "# # This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# # The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "# for param in params:\n",
    "#     param.update({\n",
    "#         'tree_method': 'hist',\n",
    "#         'device': 'cuda',\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "#     })\n",
    "\n",
    "# num_boost_round = [517, 525, 531, 527, 898, 702, 498, 443, 465, 506, 507]\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "# for i in range(len(params)):\n",
    "\n",
    "#     # Create the XGBoost models\n",
    "#     model26 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "#     # Save the model to a file in JSON format\n",
    "#     #model26.save_model('saved_XGBoost_model26.json')\n",
    "\n",
    "\n",
    "#     # # Load the model from the bin (binary) file\n",
    "#     # model26 = xgb.Booster()\n",
    "#     # model26.load_model('saved_XGBoost_model26.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "#     # Predict on the competition test data\n",
    "#     competition_y_pred = model26.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "#     competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "#     # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "#     competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "#     # Create a DataFrame for submission\n",
    "#     submission = pd.DataFrame({\n",
    "#         'building_id': competition_test_building_ids,\n",
    "#         'damage_grade': competition_y_pred\n",
    "#     })\n",
    "\n",
    "#     # Save the submission DataFrame to a CSV file for submission\n",
    "#     submission.to_csv(f\"submission_XGBoost_26_{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "#     print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying Cross Validation with Optuna GPU XGBoost. Logs can be found in \"Optuna_Logs_XGBoost_Model27.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# import pandas as pd\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import MedianPruner\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "# from sklearn.metrics import f1_score\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and test sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm. this turns on GPU \n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "\n",
    "        \n",
    "#         #'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 20), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 100, 1000)\n",
    "\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     scores = []\n",
    "\n",
    "#     for train_idx, valid_idx in cv.split(train_X1_full, train_Y1_temp_full):\n",
    "#         dtrain = xgb.DMatrix(train_X1_full.iloc[train_idx], label=train_Y1_temp_full.iloc[train_idx])\n",
    "#         dvalid = xgb.DMatrix(train_X1_full.iloc[valid_idx], label=train_Y1_temp_full.iloc[valid_idx])\n",
    "\n",
    "#         model = xgb.train(params, \n",
    "#                           dtrain, \n",
    "#                           num_boost_round=num_boost_round, \n",
    "#                           evals=[(dvalid, 'eval')], \n",
    "#                           early_stopping_rounds=10, \n",
    "#                           verbose_eval=False)\n",
    "        \n",
    "#         pred = model.predict(dvalid)\n",
    "#         score = f1_score(train_Y1_temp_full.iloc[valid_idx], pred, average='micro')\n",
    "#         scores.append(score)\n",
    "\n",
    "#     return np.mean(scores)\n",
    "\n",
    "# #pruner = MedianPruner(n_startup_trials=10, n_warmup_steps=10)\n",
    "# #pruner = HyperbandPruner(min_resource=10) # Pruning won't be considered until after 10 iterations for each trial\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', \n",
    "#                             #pruner=pruner, \n",
    "#                             sampler=sampler)\n",
    "# study.optimize(objective, n_trials=100)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 27"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # [I 2024-01-27 22:53:43,733] Trial 73 finished with value: 0.7467776361142529 and parameters: {'max_depth': 15, 'learning_rate': 0.07176584445592332, 'subsample': 0.819649190296673, 'colsample_bytree': 0.7297695708758003, 'reg_alpha': 0.00014021976520202774, 'reg_lambda': 0.09814516207267494, 'min_child_weight': 12, 'gamma': 0.30110742817450914, 'num_boost_round': 906}. Best is trial 73 with value: 0.7467776361142529.\n",
    "# # [I 2024-01-27 22:08:06,981] Trial 44 finished with value: 0.7466126316143483 and parameters: {'max_depth': 15, 'learning_rate': 0.07936442699137237, 'subsample': 0.7949699342315747, 'colsample_bytree': 0.7613849248673744, 'reg_alpha': 0.000766044875617753, 'reg_lambda': 0.0010892361858568964, 'min_child_weight': 8, 'gamma': 0.34383084613428017, 'num_boost_round': 657}. Best is trial 44 with value: 0.7466126316143483.\n",
    "# # [I 2024-01-27 22:00:06,534] Trial 41 finished with value: 0.7465934443854103 and parameters: {'max_depth': 12, 'learning_rate': 0.14084749989982814, 'subsample': 0.8072078653877525, 'colsample_bytree': 0.8249375706521678, 'reg_alpha': 0.023652060157623934, 'reg_lambda': 9.058634703024376e-05, 'min_child_weight': 8, 'gamma': 0.3209248849920897, 'num_boost_round': 762}. Best is trial 41 with value: 0.7465934443854103.\n",
    "# # [I 2024-01-27 21:42:17,011] Trial 33 finished with value: 0.7463632105957825 and parameters: {'max_depth': 16, 'learning_rate': 0.0994546401703216, 'subsample': 0.7817439075169768, 'colsample_bytree': 0.7367816591414275, 'reg_alpha': 0.016362034800666784, 'reg_lambda': 0.021599712662850625, 'min_child_weight': 10, 'gamma': 0.32245819714232826, 'num_boost_round': 626}. Best is trial 33 with value: 0.7463632105957825.\n",
    "# # [I 2024-01-27 21:02:50,816] Trial 10 finished with value: 0.7454767983873879 and parameters: {'max_depth': 13, 'learning_rate': 0.08350914590440645, 'subsample': 0.8418658565504189, 'colsample_bytree': 0.8600372561243069, 'reg_alpha': 0.006764083283242511, 'reg_lambda': 0.006382130086589043, 'min_child_weight': 10, 'gamma': 0.321614403741767, 'num_boost_round': 674}. Best is trial 10 with value: 0.7454767983873879.\n",
    "# # [I 2024-01-27 20:48:07,919] Trial 4 finished with value: 0.7451966768454236 and parameters: {'max_depth': 14, 'learning_rate': 0.05721337634497906, 'subsample': 0.8718504353045916, 'colsample_bytree': 0.8515755939321006, 'reg_alpha': 0.0008527774360401653, 'reg_lambda': 8.112546604603403e-05, 'min_child_weight': 5, 'gamma': 0.2346411059752564, 'num_boost_round': 360}. Best is trial 4 with value: 0.7451966768454236.\n",
    "\n",
    "# # [I 2024-01-27 23:27:51,570] Trial 96 finished with value: 0.7464821680285454 and parameters: {'max_depth': 12, 'learning_rate': 0.10783073662547868, 'subsample': 0.8254646597226531, 'colsample_bytree': 0.7365674364458886, 'reg_alpha': 0.0014424420647900114, 'reg_lambda': 0.09276639114468592, 'min_child_weight': 10, 'gamma': 0.3669478259851947, 'num_boost_round': 771}. Best is trial 73 with value: 0.7467776361142529.\n",
    "# # [I 2024-01-27 23:26:32,179] Trial 95 finished with value: 0.7461099564015565 and parameters: {'max_depth': 17, 'learning_rate': 0.10551156888520756, 'subsample': 0.7925493203685217, 'colsample_bytree': 0.7111997463273205, 'reg_alpha': 0.001173920569309059, 'reg_lambda': 0.10076589629635299, 'min_child_weight': 10, 'gamma': 0.35731063735810553, 'num_boost_round': 685}. Best is trial 73 with value: 0.7467776361142529.\n",
    "# # [I 2024-01-27 23:13:21,547] Trial 87 finished with value: 0.7463248410706399 and parameters: {'max_depth': 15, 'learning_rate': 0.09758473415695909, 'subsample': 0.8409084662738137, 'colsample_bytree': 0.8132505291533378, 'reg_alpha': 0.0005129766309031195, 'reg_lambda': 0.03149739692791368, 'min_child_weight': 9, 'gamma': 0.2901807954002583, 'num_boost_round': 733}. Best is trial 73 with value: 0.7467776361142529.\n",
    "# # [I 2024-01-27 22:36:41,874] Trial 57 finished with value: 0.7464015839493152 and parameters: {'max_depth': 15, 'learning_rate': 0.07096461632674658, 'subsample': 0.819632108870837, 'colsample_bytree': 0.7871285646418766, 'reg_alpha': 0.0006332850151794863, 'reg_lambda': 0.04513117998049593, 'min_child_weight': 12, 'gamma': 0.3391684849671152, 'num_boost_round': 638}. Best is trial 44 with value: 0.7466126316143483.\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# import pandas as pd\n",
    "# from sklearn.preprocessing import LabelEncoder\n",
    "# from xgboost import XGBClassifier\n",
    "# import xgboost as xgb\n",
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Define the parameters for each model\n",
    "# params = [\n",
    "#     {'max_depth': 15, 'learning_rate': 0.07176584445592332, 'subsample': 0.819649190296673, 'colsample_bytree': 0.7297695708758003, 'reg_alpha': 0.00014021976520202774, 'reg_lambda': 0.09814516207267494, 'min_child_weight': 12, 'gamma': 0.30110742817450914},\n",
    "#     {'max_depth': 15, 'learning_rate': 0.07936442699137237, 'subsample': 0.7949699342315747, 'colsample_bytree': 0.7613849248673744, 'reg_alpha': 0.000766044875617753, 'reg_lambda': 0.0010892361858568964, 'min_child_weight': 8, 'gamma': 0.34383084613428017},\n",
    "#     {'max_depth': 12, 'learning_rate': 0.14084749989982814, 'subsample': 0.8072078653877525, 'colsample_bytree': 0.8249375706521678, 'reg_alpha': 0.023652060157623934, 'reg_lambda': 9.058634703024376e-05, 'min_child_weight': 8, 'gamma': 0.3209248849920897},\n",
    "#     {'max_depth': 16, 'learning_rate': 0.0994546401703216, 'subsample': 0.7817439075169768, 'colsample_bytree': 0.7367816591414275, 'reg_alpha': 0.016362034800666784, 'reg_lambda': 0.021599712662850625, 'min_child_weight': 10, 'gamma': 0.32245819714232826},\n",
    "#     {'max_depth': 13, 'learning_rate': 0.08350914590440645, 'subsample': 0.8418658565504189, 'colsample_bytree': 0.8600372561243069, 'reg_alpha': 0.006764083283242511, 'reg_lambda': 0.006382130086589043, 'min_child_weight': 10, 'gamma': 0.321614403741767},\n",
    "#     {'max_depth': 14, 'learning_rate': 0.05721337634497906, 'subsample': 0.8718504353045916, 'colsample_bytree': 0.8515755939321006, 'reg_alpha': 0.0008527774360401653, 'reg_lambda': 8.112546604603403e-05, 'min_child_weight': 5, 'gamma': 0.2346411059752564},\n",
    "#      {'max_depth': 12, 'learning_rate': 0.10783073662547868, 'subsample': 0.8254646597226531, 'colsample_bytree': 0.7365674364458886, 'reg_alpha': 0.0014424420647900114, 'reg_lambda': 0.09276639114468592, 'min_child_weight': 10, 'gamma': 0.3669478259851947},\n",
    "#     {'max_depth': 17, 'learning_rate': 0.10551156888520756, 'subsample': 0.7925493203685217, 'colsample_bytree': 0.7111997463273205, 'reg_alpha': 0.001173920569309059, 'reg_lambda': 0.10076589629635299, 'min_child_weight': 10, 'gamma': 0.35731063735810553},\n",
    "#     {'max_depth': 15, 'learning_rate': 0.09758473415695909, 'subsample': 0.8409084662738137, 'colsample_bytree': 0.8132505291533378, 'reg_alpha': 0.0005129766309031195, 'reg_lambda': 0.03149739692791368, 'min_child_weight': 9, 'gamma': 0.2901807954002583},\n",
    "#     {'max_depth': 15, 'learning_rate': 0.07096461632674658, 'subsample': 0.819632108870837, 'colsample_bytree': 0.7871285646418766, 'reg_alpha': 0.0006332850151794863, 'reg_lambda': 0.04513117998049593, 'min_child_weight': 12, 'gamma': 0.3391684849671152},\n",
    "# ]\n",
    "\n",
    "# # This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# # The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "# for param in params:\n",
    "#     param.update({\n",
    "#         'tree_method': 'hist',\n",
    "#         'device': 'cuda',\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "#     })\n",
    "\n",
    "# num_boost_round = [906, 657, 762, 626, 674, 360, 771, 685, 733, 638]\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate the datasets\n",
    "# full_X = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# full_Y_temp = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# # Separate building_ids and features in the test data\n",
    "# competition_test_building_ids = test_data1['building_id']\n",
    "# competition_test_X = test_data1.drop('building_id', axis=1)\n",
    "\n",
    "# dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# # Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "# for i in range(len(params)):\n",
    "\n",
    "#     # Create the XGBoost models\n",
    "#     model27 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "#     # Save the model to a file in bin format\n",
    "#     model27.save_model('saved_XGBoost_model27.bin')\n",
    "\n",
    "\n",
    "#     # # Load the model from the bin (binary) file\n",
    "#     # model27 = xgb.Booster()\n",
    "#     # model27.load_model('saved_XGBoost_model27.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "#     # Predict on the competition test data\n",
    "#     competition_y_pred = model27.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "#     competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "#     # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "#     competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "#     # Create a DataFrame for submission\n",
    "#     submission = pd.DataFrame({\n",
    "#         'building_id': competition_test_building_ids,\n",
    "#         'damage_grade': competition_y_pred\n",
    "#     })\n",
    "\n",
    "#     # Save the submission DataFrame to a CSV file for submission\n",
    "#     submission.to_csv(f\"submission_XGBoost_27_{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "#     print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure out how to do SQL and see if it's free. and make sure the rest of the Optuna are put into a SQL, since there is much more info there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'random_state': 42,\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         'n_estimators': 543,\n",
    "#         'learning_rate': 0.09107783861185915,\n",
    "#         'subsample': 0.9231950980125518,\n",
    "#         'reg_alpha': 1.0170523313529933e-05,\n",
    "#         'reg_lambda': 0.00023219813720820273,\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20),\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5),\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1),\n",
    "#         'tree_method': 'gpu_hist'  # Use GPU accelerated algorithm\n",
    "#     }\n",
    "\n",
    "#     model = xgb.XGBClassifier(**params)\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     score = cross_val_score(model, train_X1_full, train_Y1_temp_full, cv=cv, scoring='f1_micro', n_jobs=-1)\n",
    "#     f1 = score.mean()\n",
    "#     return f1\n",
    "\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Assume you have X and y as your features and labels\n",
    "# X, y = ...\n",
    "\n",
    "# # Create the StratifiedKFold object\n",
    "# stratified_kfold = StratifiedKFold(n_splits=5)\n",
    "\n",
    "# # Use the object to generate train/test indices\n",
    "# for train_index, test_index in stratified_kfold.split(X, y):\n",
    "#     X_train, X_test = X[train_index], X[test_index]\n",
    "#     y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "#     # Create and train the XGBoost model\n",
    "#     model = xgb.XGBClassifier(...)\n",
    "#     model.fit(X_train, y_train)\n",
    "\n",
    "#     # Now you can use the model to make predictions\n",
    "#     predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# import pandas as pd\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import MedianPruner\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and test sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm\n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         #'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         #'n_estimators': 543, # Default is 100 , not needed because of the num_boost_round hyperparameter in the cross validation method\n",
    "#         'learning_rate': 0.09107783861185915, # Default is 0.3\n",
    "#         'subsample': 0.9231950980125518, # Default is 1\n",
    "#         'reg_alpha': 1.0170523313529933e-05, # Default is 0\n",
    "#         'reg_lambda': 0.00023219813720820273,  # Default is 1\n",
    "\n",
    "\n",
    "#         # These below are the ones we are testing and tuning\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20), # Default is 6\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#         #'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1), # Default is 1\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     dtrain = xgb.DMatrix(train_X1_full, label=train_Y1_temp_full) # This is change the pandas data into something our GPU can handle better (DMatrix)\n",
    "\n",
    "#     cv_results = xgb.cv(\n",
    "#         params,\n",
    "#         dtrain,\n",
    "#         num_boost_round=543,\n",
    "#         nfold=5,\n",
    "#         metrics={'merror'}, # This is for multiclass classification error\n",
    "#         early_stopping_rounds=10\n",
    "#     )\n",
    "\n",
    "#      # We want to maximize accuracy, so we return the negative of the error rate\n",
    "#     return 1 - cv_results['test-merror-mean'].min()\n",
    "\n",
    "# #sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# sampler = TPESampler()\n",
    "\n",
    "# pruner = optuna.pruners.MedianPruner(n_warmup_steps=10, interval_steps=1) # n_warmup_steps=10 means that it won't think about pruning until after 10 steps, and interval_steps=1 means each trial is a step\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', pruner=pruner, sampler = sampler)\n",
    "\n",
    "\n",
    "# study.optimize(objective, n_trials=50) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# # Took 60 minutes but because of GPU acceleration, my CPU was not hot! And neither was my GPU! It's so good for my computer health"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and testt sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'mlogloss',\n",
    "#         'n_estimators': 543, # Default is 100\n",
    "#         'learning_rate': 0.09107783861185915, # Default is 0.3\n",
    "#         'subsample': 0.9231950980125518, # Default is 1\n",
    "#         'reg_alpha': 1.0170523313529933e-05, # Default is 0\n",
    "#         'reg_lambda': 0.00023219813720820273,  # Default is 1\n",
    "\n",
    "#         'max_depth': trial.suggest_int('max_depth', 9, 20), # Default is 6\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 10), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1), # Default is 1\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     model = xgb.XGBClassifier(**params)\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     score = cross_val_score(model, train_X1_full, train_Y1_temp_full, cv=cv, scoring='f1_micro', n_jobs=4)\n",
    "#     f1 = score.mean()\n",
    "#     return f1       \n",
    "\n",
    "\n",
    "# study = optuna.create_study(direction='maximize')\n",
    "# study.optimize(objective, n_trials=50) \n",
    "\n",
    "# print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "\n",
    "\n",
    "# # The trials can be found on the \"Optuna_Logs_XGBoost_Model10.txt\" file!!!\n",
    "\n",
    "# # c:\\Users\\Micha\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
    "# #   from .autonotebook import tqdm as notebook_tqdm\n",
    "# # [I 2024-01-10 20:26:47,987] A new study created in memory with name: no-name-d5cbd81d-861c-404e-983f-e113bbc1aeb2\n",
    "# # [I 2024-01-10 20:27:47,171] Trial 0 finished with value: 0.7466548438861265 and parameters: {'max_depth': 13, 'min_child_weight': 10, 'gamma': 0.377206847613114, 'colsample_bytree': 0.8339161855632286}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:29:00,378] Trial 1 finished with value: 0.745307960196463 and parameters: {'max_depth': 14, 'min_child_weight': 2, 'gamma': 0.3103015639136577, 'colsample_bytree': 0.9190031191129362}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:32:44,147] Trial 2 finished with value: 0.731175245325819 and parameters: {'max_depth': 20, 'min_child_weight': 5, 'gamma': 0.07344816628567508, 'colsample_bytree': 0.8828090345804489}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:33:55,996] Trial 3 finished with value: 0.7465665900689232 and parameters: {'max_depth': 18, 'min_child_weight': 9, 'gamma': 0.47786972934464705, 'colsample_bytree': 0.7733168439787258}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:35:12,473] Trial 4 finished with value: 0.7464860087137399 and parameters: {'max_depth': 13, 'min_child_weight': 6, 'gamma': 0.23071171657194672, 'colsample_bytree': 0.7679305598380898}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:36:07,701] Trial 5 finished with value: 0.743684785784563 and parameters: {'max_depth': 9, 'min_child_weight': 5, 'gamma': 0.47446899100728207, 'colsample_bytree': 0.9356788297232858}. Best is trial 0 with value: 0.7466548438861265.\n",
    "# # [I 2024-01-10 20:37:16,536] Trial 6 finished with value: 0.7466778701069328 and parameters: {'max_depth': 12, 'min_child_weight': 2, 'gamma': 0.30654299104889127, 'colsample_bytree': 0.7976341034830331}. Best is trial 6 with value: 0.7466778701069328.\n",
    "# # [I 2024-01-10 20:39:28,533] Trial 7 finished with value: 0.7385389979627515 and parameters: {'max_depth': 19, 'min_child_weight': 10, 'gamma': 0.10661878802572344, 'colsample_bytree': 0.7943430858990438}. Best is trial 6 with value: 0.7466778701069328.\n",
    "# # [I 2024-01-10 20:40:29,887] Trial 8 finished with value: 0.7455880800451008 and parameters: {'max_depth': 16, 'min_child_weight': 9, 'gamma': 0.33645442238736545, 'colsample_bytree': 0.9417725575171005}. Best is trial 6 with value: 0.7466778701069328.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Doing same thing as before but without any pruning or early stoppings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# import pandas as pd\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import MedianPruner\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "# from sklearn.metrics import f1_score\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and test sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm. this turns on GPU \n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "\n",
    "        \n",
    "#         'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 20), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 100, 1000)\n",
    "\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     scores = []\n",
    "\n",
    "#     for train_idx, valid_idx in cv.split(train_X1_full, train_Y1_temp_full):\n",
    "#         dtrain = xgb.DMatrix(train_X1_full.iloc[train_idx], label=train_Y1_temp_full.iloc[train_idx])\n",
    "#         dvalid = xgb.DMatrix(train_X1_full.iloc[valid_idx], label=train_Y1_temp_full.iloc[valid_idx])\n",
    "\n",
    "#         model = xgb.train(params, \n",
    "#                           dtrain, \n",
    "#                           num_boost_round=num_boost_round, \n",
    "#                           evals=[(dvalid, 'eval')], \n",
    "#                           #early_stopping_rounds=10, \n",
    "#                           verbose_eval=False)\n",
    "        \n",
    "#         pred = model.predict(dvalid)\n",
    "#         score = f1_score(train_Y1_temp_full.iloc[valid_idx], pred, average='micro')\n",
    "#         scores.append(score)\n",
    "\n",
    "#     return np.mean(scores)\n",
    "\n",
    "# #pruner = MedianPruner(n_startup_trials=10, n_warmup_steps=10)\n",
    "# #pruner = HyperbandPruner(min_resource=10) # Pruning won't be considered until after 10 iterations for each trial\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', \n",
    "#                             #pruner=pruner, \n",
    "#                             sampler=sampler)\n",
    "# study.optimize(objective, n_trials=600)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying test set (train_X4) all features. Logs found in \"Optuna_Logs_XGBoost_Model28.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# from optuna.integration import XGBoostPruningCallback\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from optuna.pruners import MedianPruner\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y4_temp = train_Y4['damage_grade'] - 1  \n",
    "# valid_Y4_temp = valid_Y4['damage_grade'] - 1\n",
    "# test_Y4_temp = test_Y4['damage_grade'] - 1\n",
    "\n",
    "\n",
    "# train_X4_full = train_X4 # 70% used for training, and 30% for validation\n",
    "# train_Y4_temp_full = train_Y4_temp\n",
    "\n",
    "# valid_X4_full = pd.concat([valid_X4, test_X4]) # Now I have 30% of data for the validation \n",
    "# valid_Y4_temp_full = pd.concat([valid_Y4_temp, test_Y4_temp])\n",
    "\n",
    "\n",
    "\n",
    "# def objective(trial):\n",
    "#     # params = {\n",
    "       \n",
    "#     #     'max_depth': trial.suggest_int('max_depth', 9, 20),\n",
    "#     #     'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n",
    "#     #     'gamma': trial.suggest_float('gamma', 0, 0.5),\n",
    "#     #     'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1),\n",
    "#     #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n",
    "#     #     'subsample': trial.suggest_float('subsample', 0.5, 1),\n",
    "#     #     'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-5, 1),\n",
    "#     #     'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-5, 1),\n",
    "#     # }\n",
    "    \n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm\n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         #'predictor': 'gpu_predictor',\n",
    "#         #'gpu_id' : 0,\n",
    "#         #'predictor': 'cpu_predictor', # This is to make the GPU acceleration more smooth. Making the CPU as the predictor will lighten the load for the GPU\n",
    "\n",
    "#         #These below are the hyperparameters from Model 5, which has the highest score (74.75) so far\n",
    "#         'objective': 'multi:softmax',  # Default is 'binary:logistic'\n",
    "#         'num_class': 3, # Default is 1 (for binary classification)\n",
    "#         'random_state': 42,  # Default is 0\n",
    "#         'eval_metric': 'merror',\n",
    "#         #'eval_metric': 'mlogloss',\n",
    "      \n",
    "      \n",
    "#         #'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 20), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "\n",
    "        \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 400, 900) # This adds num_boost_round into hyper parameter tuning\n",
    "\n",
    "#     dtrain = xgb.DMatrix(train_X4_full, label=train_Y4_temp_full) # This is change the pandas data into something our GPU can handle better (DMatrix)\n",
    "#     dvalid = xgb.DMatrix(valid_X4_full, label=valid_Y4_temp_full)\n",
    "\n",
    "#     #pruning_callback = XGBoostPruningCallback(trial, \"validation-merror\")  # This is what makes this code \"Optuna's integration with XGBoost\"\n",
    "  \n",
    "\n",
    "#     watchlist = [(dtrain, 'train'), (dvalid, 'validation')]  # Specify what data to evaluate and when\n",
    "\n",
    "#     model = xgb.train(\n",
    "#         params,\n",
    "#         dtrain,\n",
    "#         num_boost_round=num_boost_round,\n",
    "#         evals=watchlist,\n",
    "#         early_stopping_rounds=10,\n",
    "#         #callbacks=[pruning_callback],\n",
    "#         verbose_eval=False  # verbose\n",
    "#     )\n",
    "\n",
    "#     # Use validation data for evaluation\n",
    "#     preds = model.predict(dvalid)\n",
    "#     error = (preds != valid_Y4_temp_full).sum() / len(valid_Y4_temp_full)\n",
    "\n",
    "#     return 1 - error\n",
    "\n",
    "# #pruner = MedianPruner(n_startup_trials=10, n_warmup_steps=10)\n",
    "# #pruner = HyperbandPruner(min_resource=10) # Pruning won't be considered until after 10 iterations for each trial\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', \n",
    "#                             #pruner=pruner, \n",
    "#                             sampler=sampler)\n",
    "# study.optimize(objective, n_trials=600)\n",
    "\n",
    "# # print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))\n",
    "\n",
    "# print(f'Best trial: trial number {study.best_trial.number}, score {study.best_trial.value}, params {study.best_trial.params}')\n",
    "\n",
    "# # Took 96 min for 185 trials\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submitting Model 28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully Submitted Model 0!\n",
      "Successfully Submitted Model 1!\n",
      "Successfully Submitted Model 2!\n",
      "Successfully Submitted Model 3!\n",
      "Successfully Submitted Model 4!\n",
      "Successfully Submitted Model 5!\n",
      "Successfully Submitted Model 6!\n",
      "Successfully Submitted Model 7!\n",
      "Successfully Submitted Model 8!\n",
      "Successfully Submitted Model 9!\n",
      "Successfully Submitted Model 10!\n"
     ]
    }
   ],
   "source": [
    "# [I 2024-01-28 21:18:14,989] Trial 132 finished with value: 0.7412158964454278 and parameters: {'max_depth': 18, 'learning_rate': 0.0334091846845558, 'subsample': 0.828458861169777, 'colsample_bytree': 0.7672705664131164, 'reg_alpha': 0.02049334001509153, 'reg_lambda': 0.021298573207234165, 'min_child_weight': 8, 'gamma': 0.3745657865155657, 'num_boost_round': 683}. Best is trial 132 with value: 0.7412158964454278.\n",
    "# [I 2024-01-28 20:45:43,105] Trial 70 finished with value: 0.7412031056138959 and parameters: {'max_depth': 16, 'learning_rate': 0.04846684228670431, 'subsample': 0.8890783223897608, 'colsample_bytree': 0.8005769952366119, 'reg_alpha': 9.860719384681922e-05, 'reg_lambda': 0.001221067566921893, 'min_child_weight': 15, 'gamma': 0.3063328781623517, 'num_boost_round': 646}. Best is trial 70 with value: 0.7412031056138959.\n",
    "# [I 2024-01-28 20:36:20,093] Trial 51 finished with value: 0.7405507732057661 and parameters: {'max_depth': 13, 'learning_rate': 0.0725851024781176, 'subsample': 0.8638814561924124, 'colsample_bytree': 0.7887399318950916, 'reg_alpha': 0.00013132489746638579, 'reg_lambda': 0.023399582015872177, 'min_child_weight': 13, 'gamma': 0.3390085919908514, 'num_boost_round': 588}. Best is trial 51 with value: 0.7405507732057661.\n",
    "# [I 2024-01-28 20:30:22,148] Trial 40 finished with value: 0.7403461199012548 and parameters: {'max_depth': 18, 'learning_rate': 0.03894650695632271, 'subsample': 0.845255799397075, 'colsample_bytree': 0.8262047308121397, 'reg_alpha': 0.00010877082152035756, 'reg_lambda': 0.004843354468185792, 'min_child_weight': 13, 'gamma': 0.23246286970665334, 'num_boost_round': 563}. Best is trial 40 with value: 0.7403461199012548.\n",
    "# [I 2024-01-28 20:24:42,938] Trial 31 finished with value: 0.7401798390913393 and parameters: {'max_depth': 13, 'learning_rate': 0.06399046740732188, 'subsample': 0.945670948202838, 'colsample_bytree': 0.8040895683625671, 'reg_alpha': 0.000604499951056913, 'reg_lambda': 0.0029989143741818193, 'min_child_weight': 13, 'gamma': 0.2867005241464392, 'num_boost_round': 636}. Best is trial 31 with value: 0.7401798390913393.\n",
    "# [I 2024-01-28 20:18:58,427] Trial 20 finished with value: 0.7398984407976363 and parameters: {'max_depth': 15, 'learning_rate': 0.13898829811580257, 'subsample': 0.8621427394786444, 'colsample_bytree': 0.8486782399346747, 'reg_alpha': 2.362593089442652e-05, 'reg_lambda': 0.00018554888520539667, 'min_child_weight': 15, 'gamma': 0.27905201068181423, 'num_boost_round': 663}. Best is trial 20 with value: 0.7398984407976363.\n",
    "# [I 2024-01-28 20:08:57,948] Trial 2 finished with value: 0.7397065783246568 and parameters: {'max_depth': 14, 'learning_rate': 0.06446823059426308, 'subsample': 0.8941417163013202, 'colsample_bytree': 0.9167268549422605, 'reg_alpha': 0.000687011613903676, 'reg_lambda': 0.00011195720551338815, 'min_child_weight': 14, 'gamma': 0.2417876463734283, 'num_boost_round': 598}. Best is trial 2 with value: 0.7397065783246568.\n",
    "\n",
    "\n",
    "# [I 2024-01-28 21:42:15,400] Trial 183 finished with value: 0.7409984523093847 and parameters: {'max_depth': 18, 'learning_rate': 0.07111172954745441, 'subsample': 0.8296304545380478, 'colsample_bytree': 0.7142802387066199, 'reg_alpha': 0.0013903131055939472, 'reg_lambda': 0.007879991873551153, 'min_child_weight': 8, 'gamma': 0.28261299947117163, 'num_boost_round': 667}. Best is trial 132 with value: 0.7412158964454278.\n",
    "# [I 2024-01-28 21:41:12,264] Trial 181 finished with value: 0.7409472889832568 and parameters: {'max_depth': 18, 'learning_rate': 0.06168056632024283, 'subsample': 0.8841461392812814, 'colsample_bytree': 0.7342270820261512, 'reg_alpha': 0.01211332478174504, 'reg_lambda': 0.003558453819436312, 'min_child_weight': 12, 'gamma': 0.336250637426218, 'num_boost_round': 517}. Best is trial 132 with value: 0.7412158964454278.\n",
    "# [I 2024-01-28 21:01:56,823] Trial 102 finished with value: 0.7409472889832568 and parameters: {'max_depth': 20, 'learning_rate': 0.055600352950813316, 'subsample': 0.8517442111860183, 'colsample_bytree': 0.7900987679326942, 'reg_alpha': 0.004788061817239992, 'reg_lambda': 0.0023600650752695, 'min_child_weight': 13, 'gamma': 0.34249776566857565, 'num_boost_round': 677}. Best is trial 70 with value: 0.7412031056138959.\n",
    "# [I 2024-01-28 20:51:53,460] Trial 80 finished with value: 0.7409856614778527 and parameters: {'max_depth': 19, 'learning_rate': 0.04626744083022719, 'subsample': 0.857897322085892, 'colsample_bytree': 0.7986643221006146, 'reg_alpha': 1.6513371933884545e-05, 'reg_lambda': 0.003029956009282107, 'min_child_weight': 14, 'gamma': 0.4085645602675048, 'num_boost_round': 708}. Best is trial 70 with value: 0.7412031056138959.\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "# Define the parameters for each model\n",
    "params = [\n",
    "    {'max_depth': 18, 'learning_rate': 0.0334091846845558, 'subsample': 0.828458861169777, 'colsample_bytree': 0.7672705664131164, 'reg_alpha': 0.02049334001509153, 'reg_lambda': 0.021298573207234165, 'min_child_weight': 8, 'gamma': 0.3745657865155657},\n",
    "    {'max_depth': 16, 'learning_rate': 0.04846684228670431, 'subsample': 0.8890783223897608, 'colsample_bytree': 0.8005769952366119, 'reg_alpha': 9.860719384681922e-05, 'reg_lambda': 0.001221067566921893, 'min_child_weight': 15, 'gamma': 0.3063328781623517},\n",
    "    {'max_depth': 13, 'learning_rate': 0.0725851024781176, 'subsample': 0.8638814561924124, 'colsample_bytree': 0.7887399318950916, 'reg_alpha': 0.00013132489746638579, 'reg_lambda': 0.023399582015872177, 'min_child_weight': 13, 'gamma': 0.3390085919908514},\n",
    "    {'max_depth': 18, 'learning_rate': 0.03894650695632271, 'subsample': 0.845255799397075, 'colsample_bytree': 0.8262047308121397, 'reg_alpha': 0.00010877082152035756, 'reg_lambda': 0.004843354468185792, 'min_child_weight': 13, 'gamma': 0.23246286970665334},\n",
    "    {'max_depth': 13, 'learning_rate': 0.06399046740732188, 'subsample': 0.945670948202838, 'colsample_bytree': 0.8040895683625671, 'reg_alpha': 0.000604499951056913, 'reg_lambda': 0.0029989143741818193, 'min_child_weight': 13, 'gamma': 0.2867005241464392},\n",
    "    {'max_depth': 15, 'learning_rate': 0.13898829811580257, 'subsample': 0.8621427394786444, 'colsample_bytree': 0.8486782399346747, 'reg_alpha': 2.362593089442652e-05, 'reg_lambda': 0.00018554888520539667, 'min_child_weight': 15, 'gamma': 0.27905201068181423},\n",
    "    {'max_depth': 14, 'learning_rate': 0.06446823059426308, 'subsample': 0.8941417163013202, 'colsample_bytree': 0.9167268549422605, 'reg_alpha': 0.000687011613903676, 'reg_lambda': 0.00011195720551338815, 'min_child_weight': 14, 'gamma': 0.2417876463734283},\n",
    "     {'max_depth': 18, 'learning_rate': 0.07111172954745441, 'subsample': 0.8296304545380478, 'colsample_bytree': 0.7142802387066199, 'reg_alpha': 0.0013903131055939472, 'reg_lambda': 0.007879991873551153, 'min_child_weight': 8, 'gamma': 0.28261299947117163},\n",
    "    {'max_depth': 18, 'learning_rate': 0.06168056632024283, 'subsample': 0.8841461392812814, 'colsample_bytree': 0.7342270820261512, 'reg_alpha': 0.01211332478174504, 'reg_lambda': 0.003558453819436312, 'min_child_weight': 12, 'gamma': 0.336250637426218},\n",
    "    {'max_depth': 20, 'learning_rate': 0.055600352950813316, 'subsample': 0.8517442111860183, 'colsample_bytree': 0.7900987679326942, 'reg_alpha': 0.004788061817239992, 'reg_lambda': 0.0023600650752695, 'min_child_weight': 13, 'gamma': 0.34249776566857565},\n",
    "    {'max_depth': 19, 'learning_rate': 0.04626744083022719, 'subsample': 0.857897322085892, 'colsample_bytree': 0.7986643221006146, 'reg_alpha': 1.6513371933884545e-05, 'reg_lambda': 0.003029956009282107, 'min_child_weight': 14, 'gamma': 0.4085645602675048},\n",
    "]\n",
    "\n",
    "# This code below adds the non-tuning hyperparameters (which are the same for each model) into each dictionary above, so each model will have these hyperparameters\n",
    "# The reason I didn't manually add these hyperparameters is because it will make it much harder for people to read the dictionaries above. It will make the dictionaries even longer and messier\n",
    "for param in params:\n",
    "    param.update({\n",
    "        'tree_method': 'hist',\n",
    "        'device': 'cuda',\n",
    "        'objective': 'multi:softmax',\n",
    "        'num_class': 3,\n",
    "        'eval_metric': 'merror',\n",
    "        'random_state': 42,\n",
    "    })\n",
    "\n",
    "\n",
    "num_boost_round = [683, 646, 588, 563, 636, 663, 598, 667, 517, 677, 708]\n",
    "\n",
    "# XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "train_Y4_temp = train_Y4['damage_grade'] - 1  \n",
    "valid_Y4_temp = valid_Y4['damage_grade'] - 1\n",
    "test_Y4_temp = test_Y4['damage_grade'] - 1\n",
    "\n",
    "# Concatenate the datasets\n",
    "full_X = pd.concat([train_X4, valid_X4, test_X4])\n",
    "full_Y_temp = pd.concat([train_Y4_temp, valid_Y4_temp, test_Y4_temp])\n",
    "\n",
    "dfull = xgb.DMatrix(full_X, label=full_Y_temp)\n",
    "\n",
    "# Separate building_ids and features in the test data\n",
    "competition_test_building_ids = test_data4['building_id']\n",
    "competition_test_X = test_data4.drop('building_id', axis=1)\n",
    "\n",
    "dcompetition_test_X = xgb.DMatrix(competition_test_X)\n",
    "\n",
    "# Train each model with it's unique hyperparameters and make a submission file for each unique prediction\n",
    "for i in range(len(params)):\n",
    "\n",
    "    # Create the XGBoost models\n",
    "    model28 = xgb.train(params[i], dfull, num_boost_round[i])\n",
    "\n",
    "    # # Save the model to a file in bin (binary) format\n",
    "    # model28.save_model(f'saved_XGBoost_model28_{i}.bin')\n",
    "\n",
    "\n",
    "    # Load the model from the bin (binary) file\n",
    "    #model28 = xgb.Booster()\n",
    "    #model28.load_model('saved_XGBoost_model28.bin') # Make sure to change the name of the file to match up with the model number!\n",
    "\n",
    "    # Predict on the competition test data\n",
    "    competition_y_pred = model28.predict(dcompetition_test_X)  # Make sure to change the name of the file to match up with the model number!\n",
    "    competition_y_pred = np.round(competition_y_pred).astype(int) \n",
    "\n",
    "    # Since the competition expects labels in the range 1-3, add 1 to the predictions\n",
    "    competition_y_pred = competition_y_pred + 1\n",
    "\n",
    "    # Create a DataFrame for submission\n",
    "    submission = pd.DataFrame({\n",
    "        'building_id': competition_test_building_ids,\n",
    "        'damage_grade': competition_y_pred\n",
    "    })\n",
    "\n",
    "    # Save the submission DataFrame to a CSV file for submission\n",
    "    submission.to_csv(f\"submission_XGBoost_28_{i}.csv\", index=False) # Make sure to change the name of the submission file to match up with the model number!\n",
    "    print(f\"Successfully Submitted Model {i}!\") \n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# import xgboost as xgb\n",
    "# import pandas as pd\n",
    "# from optuna.samplers import CmaEsSampler\n",
    "# from optuna.samplers import TPESampler\n",
    "# from optuna.pruners import MedianPruner\n",
    "# from optuna.pruners import HyperbandPruner\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "# from sklearn.metrics import f1_score\n",
    "\n",
    "# !pip install numpy==1.22.0\n",
    "# !pip install cmaes\n",
    "\n",
    "# # XGBoost expects the labels data to only include one column. Currently, they include two columns: building_id and damage_grade.\n",
    "# # So, I am making it so the labels data only includes one column (damage_grade) and changing the range from 1-3 to 0-2 since XGBoost wants to start at 0\n",
    "# train_Y1_temp = train_Y1['damage_grade'] - 1  \n",
    "# valid_Y1_temp = valid_Y1['damage_grade'] - 1\n",
    "# test_Y1_temp = test_Y1['damage_grade'] - 1\n",
    "\n",
    "# # Concatenate training and validation and test sets so that I have more data to fit the model on during GridSearchCV\n",
    "# train_X1_full = pd.concat([train_X1, valid_X1, test_X1])\n",
    "# train_Y1_temp_full = pd.concat([train_Y1_temp, valid_Y1_temp, test_Y1_temp])\n",
    "\n",
    "# def objective(trial):\n",
    "#     params = {\n",
    "#         'tree_method': 'hist',  # Use histogram-based algorithm. this turns on GPU \n",
    "#         'device': 'cuda',  # Use cuda for GPU acceleration\n",
    "#         'objective': 'multi:softmax',\n",
    "#         'num_class': 3,\n",
    "#         'eval_metric': 'merror',\n",
    "#         'random_state': 42,\n",
    "\n",
    "        \n",
    "#         #'n_estimators': trial.suggest_int('n_estimators', 400, 850), # Default is 100\n",
    "#         'max_depth': trial.suggest_int('max_depth', 5, 20), # Default is 6\n",
    "#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True), # Default is 0.3\n",
    "#         'subsample': trial.suggest_float('subsample', 0.7, 1.0), # Default is 1\n",
    "#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.7, 1.0), # Default is 1\n",
    "#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-6, 1, log=True), # Default is 0\n",
    "#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-6, 2, log=True),  # Default is 1\n",
    "#         'min_child_weight': trial.suggest_int('min_child_weight', 1, 20), # Default is 1\n",
    "#         'gamma': trial.suggest_float('gamma', 0, 0.5), # Default is 0 \n",
    "#     }\n",
    "\n",
    "#     num_boost_round = trial.suggest_int('num_boost_round', 100, 1000)\n",
    "\n",
    "#     cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "#     scores = []\n",
    "\n",
    "#     for train_idx, valid_idx in cv.split(train_X1_full, train_Y1_temp_full):\n",
    "#         dtrain = xgb.DMatrix(train_X1_full.iloc[train_idx], label=train_Y1_temp_full.iloc[train_idx])\n",
    "#         dvalid = xgb.DMatrix(train_X1_full.iloc[valid_idx], label=train_Y1_temp_full.iloc[valid_idx])\n",
    "\n",
    "#         model = xgb.train(params, \n",
    "#                           dtrain, \n",
    "#                           num_boost_round=num_boost_round, \n",
    "#                           evals=[(dvalid, 'eval')], \n",
    "#                           #early_stopping_rounds=10, \n",
    "#                           verbose_eval=False)\n",
    "        \n",
    "#         pred = model.predict(dvalid)\n",
    "#         score = f1_score(train_Y1_temp_full.iloc[valid_idx], pred, average='micro')\n",
    "#         scores.append(score)\n",
    "\n",
    "#     return np.mean(scores)\n",
    "\n",
    "# #pruner = MedianPruner(n_startup_trials=10, n_warmup_steps=10)\n",
    "# #pruner = HyperbandPruner(min_resource=10) # Pruning won't be considered until after 10 iterations for each trial\n",
    "\n",
    "# sampler = CmaEsSampler() # Think about using CmaEsSampler, especially when you use ALL the features, since it's good at handling that. Also good when we are trying to tune 10+ parameters at the same time\n",
    "# #sampler = TPESampler()\n",
    "\n",
    "# study = optuna.create_study(direction='maximize', \n",
    "#                             #pruner=pruner, \n",
    "#                             sampler=sampler)\n",
    "# study.optimize(objective, n_trials=100)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maybe try using ALL features in another model and then doing Optuna on it\n",
    "\n",
    "# Make the Optuna model try and optimize F1 Micro score instead of accuracy (mean error)\n",
    "\n",
    "# Figure out which sampler and Pruner did the best (between VSCODE, Kaggle, Codespaces, and Colab) and continue to use that one\n",
    "\n",
    "# Maybe try using GPU for XGBoost (GPU Acceleration)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}