Linear regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cb69aeee-4251-4a25-9581-f7f0975b9d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install dask[complete] dask-ml\n",
    "# !pip install gensim[summarization]\n",
    "# !pip install gensim==3.8.3\n",
    "# !pip install sumy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb049138-5484-427c-8104-3d012763d28c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import pickle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ec71763-2af3-4301-93ff-e141f1d53df8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\templ\\OneDrive\\Desktop\\AI\\Hackathon\\Amazon\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "os.chdir('..') # Change to parent directory\n",
    "cwd = os.getcwd() # Get current working directory\n",
    "print(cwd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "acbf133b-e918-450f-85df-637614ed2555",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the dataset using Dask\n",
    "dataset = pd.read_csv('dataset/universal_train.csv', on_bad_lines='skip', encoding='utf-8', engine='python')\n",
    "# dataset =  dataset.sample(n=20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bab82042-aad6-4f91-936a-7e86245a4e34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10',\n",
       "       'PRODUCT_TYPE_ID', 'PRODUCT_LENGTH', 'PRODUCT_LENGTH_NONORM'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0b507960-9798-4994-9215-33ac2bfdc2b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>PRODUCT_TYPE_ID</th>\n",
       "      <th>PRODUCT_LENGTH</th>\n",
       "      <th>PRODUCT_LENGTH_NONORM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.033740</td>\n",
       "      <td>0.053593</td>\n",
       "      <td>0.119451</td>\n",
       "      <td>-0.002144</td>\n",
       "      <td>-0.023888</td>\n",
       "      <td>-0.009354</td>\n",
       "      <td>-0.013213</td>\n",
       "      <td>-0.011067</td>\n",
       "      <td>-0.052480</td>\n",
       "      <td>-0.005460</td>\n",
       "      <td>-0.048928</td>\n",
       "      <td>-0.592630</td>\n",
       "      <td>-0.001440</td>\n",
       "      <td>2125.980000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.021798</td>\n",
       "      <td>0.017311</td>\n",
       "      <td>-0.010543</td>\n",
       "      <td>0.044631</td>\n",
       "      <td>-0.013809</td>\n",
       "      <td>0.007838</td>\n",
       "      <td>-0.019112</td>\n",
       "      <td>0.012684</td>\n",
       "      <td>-0.002693</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>0.007681</td>\n",
       "      <td>-0.314022</td>\n",
       "      <td>-0.002721</td>\n",
       "      <td>393.700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.021713</td>\n",
       "      <td>0.010882</td>\n",
       "      <td>0.012488</td>\n",
       "      <td>-0.015654</td>\n",
       "      <td>0.011082</td>\n",
       "      <td>0.004986</td>\n",
       "      <td>-0.017478</td>\n",
       "      <td>-0.003682</td>\n",
       "      <td>-0.005930</td>\n",
       "      <td>-0.000192</td>\n",
       "      <td>-0.005423</td>\n",
       "      <td>0.891683</td>\n",
       "      <td>-0.002459</td>\n",
       "      <td>748.031495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.041322</td>\n",
       "      <td>0.102819</td>\n",
       "      <td>-0.029833</td>\n",
       "      <td>0.067824</td>\n",
       "      <td>0.101342</td>\n",
       "      <td>-0.023887</td>\n",
       "      <td>0.054705</td>\n",
       "      <td>-0.005680</td>\n",
       "      <td>-0.031651</td>\n",
       "      <td>-0.000720</td>\n",
       "      <td>-0.008348</td>\n",
       "      <td>-0.253258</td>\n",
       "      <td>-0.002430</td>\n",
       "      <td>787.401574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.004194</td>\n",
       "      <td>0.001742</td>\n",
       "      <td>-0.000216</td>\n",
       "      <td>-0.001976</td>\n",
       "      <td>0.006235</td>\n",
       "      <td>-0.000607</td>\n",
       "      <td>0.000235</td>\n",
       "      <td>-0.002081</td>\n",
       "      <td>0.000451</td>\n",
       "      <td>0.008637</td>\n",
       "      <td>0.532392</td>\n",
       "      <td>-0.002570</td>\n",
       "      <td>598.424000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249693</th>\n",
       "      <td>0.019544</td>\n",
       "      <td>0.061277</td>\n",
       "      <td>-0.024850</td>\n",
       "      <td>0.036257</td>\n",
       "      <td>0.073269</td>\n",
       "      <td>-0.017734</td>\n",
       "      <td>0.047925</td>\n",
       "      <td>0.000918</td>\n",
       "      <td>0.012451</td>\n",
       "      <td>-0.000118</td>\n",
       "      <td>-0.004641</td>\n",
       "      <td>-0.249980</td>\n",
       "      <td>-0.002139</td>\n",
       "      <td>1181.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249694</th>\n",
       "      <td>0.020147</td>\n",
       "      <td>0.015623</td>\n",
       "      <td>0.021427</td>\n",
       "      <td>-0.000404</td>\n",
       "      <td>0.003816</td>\n",
       "      <td>0.009220</td>\n",
       "      <td>-0.025823</td>\n",
       "      <td>-0.002782</td>\n",
       "      <td>-0.011322</td>\n",
       "      <td>0.000861</td>\n",
       "      <td>0.017507</td>\n",
       "      <td>-0.148118</td>\n",
       "      <td>-0.002919</td>\n",
       "      <td>125.984252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249695</th>\n",
       "      <td>0.063208</td>\n",
       "      <td>0.032880</td>\n",
       "      <td>0.119688</td>\n",
       "      <td>0.024788</td>\n",
       "      <td>-0.071651</td>\n",
       "      <td>-0.024792</td>\n",
       "      <td>0.045420</td>\n",
       "      <td>-0.009083</td>\n",
       "      <td>0.004295</td>\n",
       "      <td>-0.002458</td>\n",
       "      <td>-0.022181</td>\n",
       "      <td>-0.611792</td>\n",
       "      <td>-0.002125</td>\n",
       "      <td>1200.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249696</th>\n",
       "      <td>0.033887</td>\n",
       "      <td>0.054742</td>\n",
       "      <td>0.086165</td>\n",
       "      <td>-0.033040</td>\n",
       "      <td>-0.014690</td>\n",
       "      <td>-0.016088</td>\n",
       "      <td>0.023324</td>\n",
       "      <td>0.032273</td>\n",
       "      <td>0.099026</td>\n",
       "      <td>-0.000348</td>\n",
       "      <td>-0.022752</td>\n",
       "      <td>-0.859387</td>\n",
       "      <td>-0.000867</td>\n",
       "      <td>2900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249697</th>\n",
       "      <td>0.005274</td>\n",
       "      <td>0.007339</td>\n",
       "      <td>0.013561</td>\n",
       "      <td>-0.001855</td>\n",
       "      <td>0.002032</td>\n",
       "      <td>0.004187</td>\n",
       "      <td>-0.014808</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>-0.009082</td>\n",
       "      <td>-0.000322</td>\n",
       "      <td>-0.002877</td>\n",
       "      <td>0.848820</td>\n",
       "      <td>-0.001533</td>\n",
       "      <td>2000.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2249698 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                0         1         2         3         4         5         6  \\\n",
       "0        0.033740  0.053593  0.119451 -0.002144 -0.023888 -0.009354 -0.013213   \n",
       "1        0.021798  0.017311 -0.010543  0.044631 -0.013809  0.007838 -0.019112   \n",
       "2        0.021713  0.010882  0.012488 -0.015654  0.011082  0.004986 -0.017478   \n",
       "3        0.041322  0.102819 -0.029833  0.067824  0.101342 -0.023887  0.054705   \n",
       "4        0.001786  0.004194  0.001742 -0.000216 -0.001976  0.006235 -0.000607   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "2249693  0.019544  0.061277 -0.024850  0.036257  0.073269 -0.017734  0.047925   \n",
       "2249694  0.020147  0.015623  0.021427 -0.000404  0.003816  0.009220 -0.025823   \n",
       "2249695  0.063208  0.032880  0.119688  0.024788 -0.071651 -0.024792  0.045420   \n",
       "2249696  0.033887  0.054742  0.086165 -0.033040 -0.014690 -0.016088  0.023324   \n",
       "2249697  0.005274  0.007339  0.013561 -0.001855  0.002032  0.004187 -0.014808   \n",
       "\n",
       "                7         8         9        10  PRODUCT_TYPE_ID  \\\n",
       "0       -0.011067 -0.052480 -0.005460 -0.048928        -0.592630   \n",
       "1        0.012684 -0.002693  0.000218  0.007681        -0.314022   \n",
       "2       -0.003682 -0.005930 -0.000192 -0.005423         0.891683   \n",
       "3       -0.005680 -0.031651 -0.000720 -0.008348        -0.253258   \n",
       "4        0.000235 -0.002081  0.000451  0.008637         0.532392   \n",
       "...           ...       ...       ...       ...              ...   \n",
       "2249693  0.000918  0.012451 -0.000118 -0.004641        -0.249980   \n",
       "2249694 -0.002782 -0.011322  0.000861  0.017507        -0.148118   \n",
       "2249695 -0.009083  0.004295 -0.002458 -0.022181        -0.611792   \n",
       "2249696  0.032273  0.099026 -0.000348 -0.022752        -0.859387   \n",
       "2249697  0.000081 -0.009082 -0.000322 -0.002877         0.848820   \n",
       "\n",
       "         PRODUCT_LENGTH  PRODUCT_LENGTH_NONORM  \n",
       "0             -0.001440            2125.980000  \n",
       "1             -0.002721             393.700000  \n",
       "2             -0.002459             748.031495  \n",
       "3             -0.002430             787.401574  \n",
       "4             -0.002570             598.424000  \n",
       "...                 ...                    ...  \n",
       "2249693       -0.002139            1181.100000  \n",
       "2249694       -0.002919             125.984252  \n",
       "2249695       -0.002125            1200.000000  \n",
       "2249696       -0.000867            2900.000000  \n",
       "2249697       -0.001533            2000.000000  \n",
       "\n",
       "[2249698 rows x 14 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.drop(['Unnamed: 0'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "99f7615d-f840-4f6e-9a4c-0f8995348b1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "829d8df3-43b8-448b-bb69-f762808be301",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_selection import SelectKBest, f_regression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "from sklearn.metrics import mean_squared_error, accuracy_score\n",
    "from scipy.sparse import hstack\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86465fc1-20db-4dac-8fa3-92efa33d14e2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "622846fe-c1f4-470b-b856-e4fd69bca5c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into features (X) and target (y)\n",
    "X = df.drop(columns = ['Unnamed: 0','PRODUCT_TYPE_ID','PRODUCT_LENGTH','PRODUCT_LENGTH_NONORM'], axis=1)\n",
    "y = df['PRODUCT_LENGTH']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e9b32778-8add-4072-9a8e-cd5be1db3702",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into training and validation sets\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9ba0578f-d2a1-40cd-a979-53809680b6d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale the data\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_val_scaled = scaler.transform(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2a0eafa6-1f6d-410b-9155-67b20d3ed5df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert y_train to a 1D array\n",
    "y_train = y_train.values.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a074bb80-2836-40d4-a143-3677decf4d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select the most relevant features using SelectKBest\n",
    "selector = SelectKBest(f_regression, k=5)\n",
    "X_train_selected = selector.fit_transform(X_train_scaled, y_train)\n",
    "X_val_selected = selector.transform(X_val_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "356a5003-5271-464e-8013-b2e10e6d687b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-4.57684152e-01,  3.18890015e-02,  4.39176919e-03,\n",
       "        -1.65307492e-01, -4.19713330e-01],\n",
       "       [ 5.55979130e-01, -1.04476268e-01, -4.68948539e+00,\n",
       "        -2.69864890e+00,  1.88605766e-01],\n",
       "       [ 5.82221092e-02, -3.88482447e-01, -1.15624057e-01,\n",
       "        -1.50577259e-01, -4.27852246e-01],\n",
       "       ...,\n",
       "       [ 1.61940825e-01,  1.20136722e+00,  8.64063904e-01,\n",
       "        -1.69113464e+00,  3.63634005e-01],\n",
       "       [ 3.93410225e-01, -1.26274496e+00,  5.44963215e-01,\n",
       "        -9.59446095e-02, -3.46188115e-01],\n",
       "       [ 1.17519790e+00, -1.33161005e+00,  1.12092143e+00,\n",
       "         1.76205884e-01, -1.03333623e+00]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9e0c6c3c-3780-42b0-ac57-85e7f72680d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train a linear regression model on the selected features\n",
    "model = LinearRegression()\n",
    "model.fit(X_train_selected, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "514c76f2-9cc3-42b1-b674-c94d8f847588",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.47\n"
     ]
    }
   ],
   "source": [
    "# Predict on the validation set and calculate RMSE\n",
    "y_pred = model.predict(X_val_selected)\n",
    "rmse = mean_squared_error(y_val, y_pred, squared=False)\n",
    "print(f'RMSE: {rmse:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3a08b156-02a5-4f2f-9fca-669e754c2eae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "# Save the model to a file\n",
    "with open('AmazonLRmodel.pickle', 'wb') as f:\n",
    "    pickle.dump(model, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cd149ed3-1fb5-4204-bc0b-5f904b3fd418",
   "metadata": {},
   "outputs": [],
   "source": [
    "testdataset = pd.read_csv('dataset/universal_test.csv', on_bad_lines='skip', encoding='utf-8', engine='python')\n",
    "# dataset =  dataset.sample(n=20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2913926d-dc57-4ca2-ba80-1ca6a90464a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10',\n",
       "       'PRODUCT_TYPE_ID'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f2e34ce0-8fd0-437a-83ea-147908f9e690",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.002030</td>\n",
       "      <td>0.001105</td>\n",
       "      <td>0.004455</td>\n",
       "      <td>-0.002957</td>\n",
       "      <td>0.081613</td>\n",
       "      <td>-0.026087</td>\n",
       "      <td>-0.060922</td>\n",
       "      <td>-0.002510</td>\n",
       "      <td>-0.000102</td>\n",
       "      <td>0.002701</td>\n",
       "      <td>0.000682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.045924</td>\n",
       "      <td>0.004526</td>\n",
       "      <td>0.038838</td>\n",
       "      <td>0.000920</td>\n",
       "      <td>-0.004138</td>\n",
       "      <td>-0.004475</td>\n",
       "      <td>0.000287</td>\n",
       "      <td>-0.023393</td>\n",
       "      <td>-0.012571</td>\n",
       "      <td>-0.010897</td>\n",
       "      <td>-0.031862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.036662</td>\n",
       "      <td>0.006406</td>\n",
       "      <td>0.021295</td>\n",
       "      <td>-0.007004</td>\n",
       "      <td>0.011642</td>\n",
       "      <td>0.054653</td>\n",
       "      <td>-0.005702</td>\n",
       "      <td>0.005560</td>\n",
       "      <td>-0.000605</td>\n",
       "      <td>-0.006511</td>\n",
       "      <td>0.050248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.014183</td>\n",
       "      <td>0.006882</td>\n",
       "      <td>0.013506</td>\n",
       "      <td>0.006704</td>\n",
       "      <td>0.003102</td>\n",
       "      <td>0.003619</td>\n",
       "      <td>0.002005</td>\n",
       "      <td>-0.013562</td>\n",
       "      <td>-0.004010</td>\n",
       "      <td>0.003927</td>\n",
       "      <td>-0.002901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001468</td>\n",
       "      <td>0.000477</td>\n",
       "      <td>0.001944</td>\n",
       "      <td>-0.000566</td>\n",
       "      <td>0.002886</td>\n",
       "      <td>-0.000283</td>\n",
       "      <td>0.001136</td>\n",
       "      <td>-0.001220</td>\n",
       "      <td>0.000452</td>\n",
       "      <td>0.002904</td>\n",
       "      <td>0.000571</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  0.002030  0.001105  0.004455 -0.002957  0.081613 -0.026087 -0.060922   \n",
       "1  0.045924  0.004526  0.038838  0.000920 -0.004138 -0.004475  0.000287   \n",
       "2  0.036662  0.006406  0.021295 -0.007004  0.011642  0.054653 -0.005702   \n",
       "3  0.014183  0.006882  0.013506  0.006704  0.003102  0.003619  0.002005   \n",
       "4  0.001468  0.000477  0.001944 -0.000566  0.002886 -0.000283  0.001136   \n",
       "\n",
       "          7         8         9        10  \n",
       "0 -0.002510 -0.000102  0.002701  0.000682  \n",
       "1 -0.023393 -0.012571 -0.010897 -0.031862  \n",
       "2  0.005560 -0.000605 -0.006511  0.050248  \n",
       "3 -0.013562 -0.004010  0.003927 -0.002901  \n",
       "4 -0.001220  0.000452  0.002904  0.000571  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdataset = testdataset.drop(['Unnamed: 0',\"PRODUCT_TYPE_ID\"],axis=1)\n",
    "testdataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b05fbde9-85a0-4a06-9fd1-b260204dfd71",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\templ\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\base.py:413: UserWarning: X has feature names, but LinearRegression was fitted without feature names\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Select the relevant columns from the test dataset\n",
    "testdataset = testdataset.iloc[:, [0, 1, 2, 3, 4]]\n",
    "\n",
    "\n",
    "# Make predictions using the selected features\n",
    "y_pred = model.predict(testdataset)\n",
    "\n",
    "# Put the predictions in the PRODUCT_LENGTH column\n",
    "testdataset['PRODUCT_LENGTH'] = y_pred\n",
    "\n",
    "# # Select the relevant columns for output\n",
    "# output = testdataset[['PRODUCT_TYPE_ID', 'PRODUCT_LENGTH']]\n",
    "\n",
    "# Save the output as a CSV file\n",
    "testdataset.to_csv('output.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "89900863-0670-4736-8b09-7d5e204059c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6.98631843e-06, 5.76673436e-05, 4.68544215e-05, ...,\n",
       "       6.43533424e-05, 1.29886011e-04, 4.55691869e-05])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c7587e44-1756-4968-a067-8e3840fb064b",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred1 = y_pred*dataset[\"PRODUCT_LENGTH_NONORM\"].std()  + dataset[\"PRODUCT_LENGTH_NONORM\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "d8c21209-616b-4d11-bed7-795a6919487c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4081.28251722, 4149.78728606, 4135.1716242 , ..., 4158.82464867,\n",
       "       4247.40415763, 4133.4343922 ])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "12eb0208-ef18-4a52-9e17-c09af26eee20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PRODUCT_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>604373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1729783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1871949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1107571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>624253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734731</th>\n",
       "      <td>921419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734732</th>\n",
       "      <td>2456362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734733</th>\n",
       "      <td>841529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734734</th>\n",
       "      <td>1190194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734735</th>\n",
       "      <td>1040810</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>734736 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        PRODUCT_ID\n",
       "0           604373\n",
       "1          1729783\n",
       "2          1871949\n",
       "3          1107571\n",
       "4           624253\n",
       "...            ...\n",
       "734731      921419\n",
       "734732     2456362\n",
       "734733      841529\n",
       "734734     1190194\n",
       "734735     1040810\n",
       "\n",
       "[734736 rows x 1 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test = pd.read_csv('dataset/test.csv', on_bad_lines='skip', encoding='utf-8', engine='python')\n",
    "test = test.drop(columns = ['PRODUCT_TYPE_ID','TITLE','BULLET_POINTS','DESCRIPTION'],axis = 1)\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f50f77b0-acc4-46eb-854a-1cedb8503429",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PRODUCT_ID</th>\n",
       "      <th>PRODUCT_LENGTH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>604373</td>\n",
       "      <td>4081.282517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1729783</td>\n",
       "      <td>4149.787286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1871949</td>\n",
       "      <td>4135.171624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1107571</td>\n",
       "      <td>4149.773828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>624253</td>\n",
       "      <td>4153.515508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734731</th>\n",
       "      <td>921419</td>\n",
       "      <td>4132.525075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734732</th>\n",
       "      <td>2456362</td>\n",
       "      <td>4149.535864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734733</th>\n",
       "      <td>841529</td>\n",
       "      <td>4158.824649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734734</th>\n",
       "      <td>1190194</td>\n",
       "      <td>4247.404158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734735</th>\n",
       "      <td>1040810</td>\n",
       "      <td>4133.434392</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>734736 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        PRODUCT_ID  PRODUCT_LENGTH\n",
       "0           604373     4081.282517\n",
       "1          1729783     4149.787286\n",
       "2          1871949     4135.171624\n",
       "3          1107571     4149.773828\n",
       "4           624253     4153.515508\n",
       "...            ...             ...\n",
       "734731      921419     4132.525075\n",
       "734732     2456362     4149.535864\n",
       "734733      841529     4158.824649\n",
       "734734     1190194     4247.404158\n",
       "734735     1040810     4133.434392\n",
       "\n",
       "[734736 rows x 2 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[\"PRODUCT_LENGTH\"] = y_pred1\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "1f18ae71-8f70-47ef-a5dc-46027199da70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PRODUCT_ID</th>\n",
       "      <th>PRODUCT_LENGTH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>604373</td>\n",
       "      <td>481.282517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1729783</td>\n",
       "      <td>549.787286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1871949</td>\n",
       "      <td>535.171624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1107571</td>\n",
       "      <td>549.773828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>624253</td>\n",
       "      <td>553.515508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734731</th>\n",
       "      <td>921419</td>\n",
       "      <td>532.525075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734732</th>\n",
       "      <td>2456362</td>\n",
       "      <td>549.535864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734733</th>\n",
       "      <td>841529</td>\n",
       "      <td>558.824649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734734</th>\n",
       "      <td>1190194</td>\n",
       "      <td>647.404158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734735</th>\n",
       "      <td>1040810</td>\n",
       "      <td>533.434392</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>734736 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        PRODUCT_ID  PRODUCT_LENGTH\n",
       "0           604373      481.282517\n",
       "1          1729783      549.787286\n",
       "2          1871949      535.171624\n",
       "3          1107571      549.773828\n",
       "4           624253      553.515508\n",
       "...            ...             ...\n",
       "734731      921419      532.525075\n",
       "734732     2456362      549.535864\n",
       "734733      841529      558.824649\n",
       "734734     1190194      647.404158\n",
       "734735     1040810      533.434392\n",
       "\n",
       "[734736 rows x 2 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[\"PRODUCT_LENGTH\"] = test[\"PRODUCT_LENGTH\"] - 3600\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "e6969480-7662-4d27-b84a-f8ef600d439f",
   "metadata": {},
   "outputs": [],
   "source": [
    "test.to_csv(\"Namuna3600.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f041d0d-ed7f-4bb4-b951-144f553d0c71",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}