Mercari Price Suggestion Challenge/Mercari Price Suggestion Challenge.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Mercari Price Suggestion Challenge.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNKmaGfpJVAeHTy7N7++k6Y"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"rz2vh7PuQvnI"},"source":["# **데이터 및 라이브러리 로드**"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TmTrQxCROUkS","executionInfo":{"status":"ok","timestamp":1629291385950,"user_tz":-540,"elapsed":17423,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"99ab8405-76f2-4f3b-f939-bbe02ceac4f5"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":445},"id":"55c0PV9JOYej","executionInfo":{"status":"ok","timestamp":1629291398007,"user_tz":-540,"elapsed":9446,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"1c3b4828-f1a0-4e4e-eaf0-335bb4c9fe6c"},"source":["import pandas as pd\n","import os\n","import warnings\n","warnings.filterwarnings(action='ignore')\n","\n","path= '/content/drive/MyDrive/data/mercari-price-suggestion-challenge/'\n","\n","mercari_df = pd.read_csv(os.path.join(path, 'train.tsv'), sep='\\t')\n","mercari_df.head()"],"execution_count":2,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>train_id</th>\n","      <th>name</th>\n","      <th>item_condition_id</th>\n","      <th>category_name</th>\n","      <th>brand_name</th>\n","      <th>price</th>\n","      <th>shipping</th>\n","      <th>item_description</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>MLB Cincinnati Reds T Shirt Size XL</td>\n","      <td>3</td>\n","      <td>Men/Tops/T-shirts</td>\n","      <td>NaN</td>\n","      <td>10.0</td>\n","      <td>1</td>\n","      <td>No description yet</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>Razer BlackWidow Chroma Keyboard</td>\n","      <td>3</td>\n","      <td>Electronics/Computers &amp; Tablets/Components &amp; P...</td>\n","      <td>Razer</td>\n","      <td>52.0</td>\n","      <td>0</td>\n","      <td>This keyboard is in great condition and works ...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>AVA-VIV Blouse</td>\n","      <td>1</td>\n","      <td>Women/Tops &amp; Blouses/Blouse</td>\n","      <td>Target</td>\n","      <td>10.0</td>\n","      <td>1</td>\n","      <td>Adorable top with a hint of lace and a key hol...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>Leather Horse Statues</td>\n","      <td>1</td>\n","      <td>Home/Home Décor/Home Décor Accents</td>\n","      <td>NaN</td>\n","      <td>35.0</td>\n","      <td>1</td>\n","      <td>New with tags. Leather horses. Retail for [rm]...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>24K GOLD plated rose</td>\n","      <td>1</td>\n","      <td>Women/Jewelry/Necklaces</td>\n","      <td>NaN</td>\n","      <td>44.0</td>\n","      <td>0</td>\n","      <td>Complete with certificate of authenticity</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   train_id  ...                                   item_description\n","0         0  ...                                 No description yet\n","1         1  ...  This keyboard is in great condition and works ...\n","2         2  ...  Adorable top with a hint of lace and a key hol...\n","3         3  ...  New with tags. Leather horses. Retail for [rm]...\n","4         4  ...          Complete with certificate of authenticity\n","\n","[5 rows x 8 columns]"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"H5Dk4sxMcy5w","executionInfo":{"status":"ok","timestamp":1629293334682,"user_tz":-540,"elapsed":248,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"c9a87fd3-79dd-4e4c-d9c3-f9867bd1e7da"},"source":["mercari_df.shape"],"execution_count":33,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(1482535, 11)"]},"metadata":{"tags":[]},"execution_count":33}]},{"cell_type":"markdown","metadata":{"id":"RqDj3N2DQyG3"},"source":["# **데이터 전처리**"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vWzCtKnfWj_z","executionInfo":{"status":"ok","timestamp":1629291711081,"user_tz":-540,"elapsed":673,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"adc02a14-1902-4105-f9a2-18a6270545de"},"source":["mercari_df.info()"],"execution_count":9,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 1482535 entries, 0 to 1482534\n","Data columns (total 8 columns):\n"," #   Column             Non-Null Count    Dtype  \n","---  ------             --------------    -----  \n"," 0   train_id           1482535 non-null  int64  \n"," 1   name               1482535 non-null  object \n"," 2   item_condition_id  1482535 non-null  int64  \n"," 3   category_name      1476208 non-null  object \n"," 4   brand_name         849853 non-null   object \n"," 5   price              1482535 non-null  float64\n"," 6   shipping           1482535 non-null  int64  \n"," 7   item_description   1482531 non-null  object \n","dtypes: float64(1), int64(3), object(4)\n","memory usage: 90.5+ MB\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"ADx6tYB3WpSy"},"source":["brand_name column에 null값이 많음"]},{"cell_type":"markdown","metadata":{"id":"Ej8YwVF1Q6q_"},"source":["#### 타겟값의 분포도 확인"]},{"cell_type":"code","metadata":{"id":"n8606lTCPQpK","colab":{"base_uri":"https://localhost:8080/","height":363},"executionInfo":{"status":"ok","timestamp":1629291630053,"user_tz":-540,"elapsed":784,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"a474df5b-bd62-4a29-a9d9-c79f6b173d69"},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","%matplotlib inline\n","\n","y_train_df = mercari_df['price']\n","plt.figure(figsize=(5,5))\n","sns.distplot(y_train_df, kde=False)"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.axes._subplots.AxesSubplot at 0x7fe071463b50>"]},"metadata":{"tags":[]},"execution_count":7},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAATwAAAFICAYAAADNtkLRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATPUlEQVR4nO3da7BdZ13H8e/PhOII5Zoj02kaUzRFMyBSDqUqYpWKaUcbb0ALCmIh3srg4GXq4FSmvqqOvlALGKFTQdpS8JbRYEGsVoGUpKUtTUrKISBNrSSW0qqopfr3xV7B3ZNzcvY5e+2zkz7fz8yZrP2s5+zn/+yd/cu67LWSqkKSWvA10y5AklaLgSepGQaepGYYeJKaYeBJaoaBJ6kZUw28JFclOZTkzhH7vzzJviR7k1wz6fokPbZkmt/DS/Ji4N+Bd1XVs5fouwm4HvjeqnogyddX1aHVqFPSY8NUt/Cq6ibgi8NtSb4xyV8nuSXJPyT55m7V64Erq+qB7ncNO0nLcjwew9sOvKGqng/8EvDWrv0M4IwkH0myK8mWqVUo6YS0dtoFDEvyROA7gPclOdL8+O7PtcAm4BxgPXBTkudU1ZdWu05JJ6bjKvAYbHF+qaq+bYF1B4Gbq+orwGeT3M0gAHevZoGSTlzH1S5tVT3EIMxeBpCB53ar/5zB1h1J1jHYxT0wjTolnZim/bWUa4GPAc9KcjDJxcCrgIuT3A7sBbZ23W8A7k+yD7gR+OWqun8adUs6MU31aymStJqOq11aSZokA09SM6Z2lnbdunW1cePGaQ0v6THqlltu+deqmllo3dQCb+PGjezZs2daw0t6jEryT4utc5dWUjMMPEnNMPAkNcPAk9QMA09SMww8Sc0w8CQ1w8CT1AwDT1IzDDxJzTDwJDXjeLvF+zFdc/Pnj2p75Qs3TKESSScit/AkNcPAk9QMA09SMww8Sc0w8CQ1w8CT1AwDT1IzDDxJzVgy8JJcleRQkjsXWf+qJHck+WSSjyZ5bv9lStL4RtnCuxrYcoz1nwW+u6qeA/wGsL2HuiSpd0teWlZVNyXZeIz1Hx16uAtYP35ZktS/vo/hXQx8oOfnlKRe9HbzgCTfwyDwXnSMPtuAbQAbNnjRv6TV1csWXpJvBd4BbK2q+xfrV1Xbq2q2qmZnZmb6GFqSRjZ24CXZAPwp8BNVdff4JUnSZCy5S5vkWuAcYF2Sg8CvA48DqKq3A5cBTwfemgTgkaqanVTBkrRSo5ylvWiJ9a8DXtdbRZI0IV5pIakZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWrGkoGX5Kokh5Lcucj6JPndJHNJ7khyZv9lStL4RtnCuxrYcoz15wGbup9twNvGL0uS+rdk4FXVTcAXj9FlK/CuGtgFPCXJKX0VKEl96eMY3qnAPUOPD3ZtknRcWdWTFkm2JdmTZM/hw4dXc2hJ6iXw7gVOG3q8vms7SlVtr6rZqpqdmZnpYWhJGl0fgbcDeHV3tvZs4MGquq+H55WkXq1dqkOSa4FzgHVJDgK/DjwOoKreDuwEzgfmgC8Dr51UsZI0jiUDr6ouWmJ9AT/fW0WSNCFeaSGpGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqxkiBl2RLkv1J5pJcusD6DUluTPKJJHckOb//UiVpPEsGXpI1wJXAecBm4KIkm+d1+zXg+qp6HnAh8Na+C5WkcY2yhXcWMFdVB6rqYeA6YOu8PgU8qVt+MvDP/ZUoSf1YO0KfU4F7hh4fBF44r89bgA8meQPwBODcXqqTpB71ddLiIuDqqloPnA+8O8lRz51kW5I9SfYcPny4p6ElaTSjBN69wGlDj9d3bcMuBq4HqKqPAV8LrJv/RFW1vapmq2p2ZmZmZRVL0gqNEni7gU1JTk9yEoOTEjvm9fk88BKAJN/CIPDchJN0XFky8KrqEeAS4AbgLgZnY/cmuTzJBV23XwRen+R24FrgJ6uqJlW0JK3EKCctqKqdwM55bZcNLe8DvrPf0iSpX15pIakZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWrGSIGXZEuS/Unmkly6SJ+XJ9mXZG+Sa/otU5LGt3apDknWAFcC3wccBHYn2VFV+4b6bAJ+FfjOqnogyddPqmBJWqlRtvDOAuaq6kBVPQxcB2yd1+f1wJVV9QBAVR3qt0xJGt8ogXcqcM/Q44Nd27AzgDOSfCTJriRb+ipQkvqy5C7tMp5nE3AOsB64KclzqupLw52SbAO2AWzYsKGnoSVpNKNs4d0LnDb0eH3XNuwgsKOqvlJVnwXuZhCAj1JV26tqtqpmZ2ZmVlqzJK3IKIG3G9iU5PQkJwEXAjvm9flzBlt3JFnHYBf3QI91StLYlgy8qnoEuAS4AbgLuL6q9ia5PMkFXbcbgPuT7ANuBH65qu6fVNGStBIjHcOrqp3Aznltlw0tF/Cm7keSjkteaSGpGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqxkiBl2RLkv1J5pJceox+P5qkksz2V6Ik9WPJwEuyBrgSOA/YDFyUZPMC/U4G3gjc3HeRktSHUbbwzgLmqupAVT0MXAdsXaDfbwBXAP/VY32S1JtRAu9U4J6hxwe7tq9KciZwWlX9VY+1SVKvxj5pkeRrgN8BfnGEvtuS7Emy5/Dhw+MOLUnLMkrg3QucNvR4fdd2xMnAs4G/S/I54Gxgx0InLqpqe1XNVtXszMzMyquWpBUYJfB2A5uSnJ7kJOBCYMeRlVX1YFWtq6qNVbUR2AVcUFV7JlKxJK3QkoFXVY8AlwA3AHcB11fV3iSXJ7lg0gVKUl/WjtKpqnYCO+e1XbZI33PGL0uS+ueVFpKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmjBR4SbYk2Z9kLsmlC6x/U5J9Se5I8uEk39B/qZI0niUDL8ka4ErgPGAzcFGSzfO6fQKYrapvBd4P/GbfhUrSuEbZwjsLmKuqA1X1MHAdsHW4Q1XdWFVf7h7uAtb3W6YkjW+UwDsVuGfo8cGubTEXAx8YpyhJmoS1fT5Zkh8HZoHvXmT9NmAbwIYNG/ocWpKWNMoW3r3AaUOP13dtj5LkXODNwAVV9d8LPVFVba+q2aqanZmZWUm9krRiowTebmBTktOTnARcCOwY7pDkecAfMAi7Q/2XKUnjWzLwquoR4BLgBuAu4Pqq2pvk8iQXdN1+C3gi8L4ktyXZscjTSdLUjHQMr6p2AjvntV02tHxuz3VJUu+80kJSMww8Sc0w8CQ1w8CT1Ixev3g8Ddfc/Pmj2l75Qr/ULOlobuFJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaYeBJaoaBJ6kZBp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmmHgSWqGgSepGQaepGYYeJKaccL/R9wL8T/nlrQQt/AkNcPAk9SMkQIvyZYk+5PMJbl0gfWPT/Lebv3NSTb2XagkjWvJY3hJ1gBXAt8HHAR2J9lRVfuGul0MPFBV35TkQuAK4BWTKHilPK4naZQtvLOAuao6UFUPA9cBW+f12Qr8Ubf8fuAlSdJfmZI0vlHO0p4K3DP0+CDwwsX6VNUjSR4Eng78ax9FTspCW32jcutQOvGs6tdSkmwDtnUP/z3J/mU+xTqmF6KPGvtVUx5/Co6b176x8Z378n3DYitGCbx7gdOGHq/v2hbqczDJWuDJwP3zn6iqtgPbRxhzQUn2VNXsSn9/HNMcu/Xxnbtz78sox/B2A5uSnJ7kJOBCYMe8PjuA13TLPwb8bVVVf2VK0viW3MLrjsldAtwArAGuqqq9SS4H9lTVDuCdwLuTzAFfZBCKknRcGekYXlXtBHbOa7tsaPm/gJf1W9qCVrw7fIKP3fr4zr3N8XsfO+55SmqFl5ZJasYJEXhLXdrW0xinJbkxyb4ke5O8sWt/S5J7k9zW/Zw/9Du/2tW0P8n3jzn+55J8shtjT9f2tCQfSvLp7s+ndu1J8rvd2HckOXPMsZ81NL/bkjyU5BcmOfckVyU5lOTOobZlzzfJa7r+n07ymoXGGnHs30ryqe75/yzJU7r2jUn+c+g1ePvQ7zy/e8/muvpG+rL9IuMv+7VeyedikbHfOzTu55LcNsG5L/Y5W5X3nqo6rn8YnCj5DPBM4CTgdmDzBMY5BTizWz4ZuBvYDLwF+KUF+m/uank8cHpX45oxxv8csG5e228Cl3bLlwJXdMvnAx8AApwN3Nzz6/0vDL7LNLG5Ay8GzgTuXOl8gacBB7o/n9otP3WFY78UWNstXzE09sbhfvOe5+NdPenqO2+MuS/rtV7p52Khseet/23gsgnOfbHP2aq89yfCFt4ol7aNraruq6pbu+V/A+5icAXJYrYC11XVf1fVZ4G5rtY+DV+y90fADw21v6sGdgFPSXJKT2O+BPhMVf3TEnWNNfequonBGf35z7uc+X4/8KGq+mJVPQB8CNiykrGr6oNV9Uj3cBeD75suqhv/SVW1qwafwHcN1bvs8Y9hsdd6RZ+LY43dbaW9HLj2WM8x5twX+5ytynt/IgTeQpe2HSuIxpbB3V6eB9zcNV3SbU5fdWRTewJ1FfDBJLdkcEUKwDOq6r5u+V+AZ0xo7GEX8ui/8Ksx9yOWO99J1fFTDLYqjjg9ySeS/H2S7xqq6WDPYy/ntZ7E3L8L+EJVfXqobWJzn/c5W5X3/kQIvFWV5InAnwC/UFUPAW8DvhH4NuA+Bpv8k/CiqjoTOA/4+SQvHl7Z/Us60VPqGXyx/ALgfV3Tas39KKsx34UkeTPwCPCeruk+YENVPQ94E3BNkidNYOipvdZDLuLR/9hNbO4LfM6+apLv/YkQeKNc2taLJI9j8Ca8p6r+FKCqvlBV/1NV/wv8If+/69ZrXVV1b/fnIeDPunG+cGRXtfvz0CTGHnIecGtVfaGrZVXmPmS58+21jiQ/CfwA8KruQ0e3K3l/t3wLg+NmZ3TjDO/2jvv+L/e17nvua4EfAd47VNNE5r7Q54xVeu9PhMAb5dK2sXXHL94J3FVVvzPUPnxs7IeBI2e3dgAXZnDz09OBTQwO5K5k7CckOfnIMoMD6Hfy6Ev2XgP8xdDYr+7OYJ0NPDi0OzCOR/0Lvxpzn2e5870BeGmSp3a7gC/t2pYtyRbgV4ALqurLQ+0zGdwTkiTPZDDXA934DyU5u/u78+qhelcy/nJf674/F+cCn6qqr+6qTmLui33OWK33fpQzK9P+YXCm5m4G/8K8eUJjvIjBZvQdwG3dz/nAu4FPdu07gFOGfufNXU37GfEs1SJjP5PBWbbbgb1H5sjgFlsfBj4N/A3wtK49DG7K+pmuttke5v8EBjd8ePJQ28TmziBY7wO+wuD4y8UrmS+D421z3c9rxxh7jsExoSPv/du7vj/avSe3AbcCPzj0PLMMgukzwO/TfZF/heMv+7VeyediobG79quBn5nXdxJzX+xztirvvVdaSGrGibBLK0m9MPAkNcPAk9QMA09SMww8Sc0w8HRcS3J5knOnXYceG/xaio5bSdZU1f9Muw49driFp6nI4F5rn0ryniR3JXl/kq/L4H5sVyS5FXhZkquT/Fj3Oy9I8tEktyf5eJKTk6zJ4F52u7sL7396ylPTcczA0zQ9C3hrVX0L8BDwc137/VV1ZlVdd6Rjd/nUe4E3VtVzGVwK9Z8MrlJ4sKpeALwAeH13CZZ0FANP03RPVX2kW/5jBpcdwdAF7EOeBdxXVbsBquqhGty/7qUMrrW8jcFthp7O4JpP6Sgj/a9l0oTMP4B85PF/LOM5AryhqlZ00wC1xS08TdOGJN/eLb8S+Mdj9N0PnJLkBQDd8bu1DO6Q8bPdLYdIckZ3xxnpKAaepmk/g5ud3sXg/yV422Ida3Ab81cAv5fkdga39P5a4B3APuDWDP5jmj/APRctwq+laCq623v/ZVU9e8qlqCFu4Ulqhlt4kprhFp6kZhh4kpph4ElqhoEnqRkGnqRmGHiSmvF/8ZhzR3egr1oAAAAASUVORK5CYII=\n","text/plain":["<Figure size 360x360 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"HF2pbsxkV-o9"},"source":["#### 타겟값 로그 변환"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":297},"id":"kzxvIhb0V4xx","executionInfo":{"status":"ok","timestamp":1629291632232,"user_tz":-540,"elapsed":629,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"5d69e1b8-0835-4bdf-f8ca-c66e8025c2c1"},"source":["y_train_df = np.log1p(y_train_df)\n","sns.distplot(y_train_df, kde=False)"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.axes._subplots.AxesSubplot at 0x7fe064c94f10>"]},"metadata":{"tags":[]},"execution_count":8},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYkAAAEGCAYAAACQO2mwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZDUlEQVR4nO3df7BfdZ3f8edrE8Efu/xOKU3CJq2RFpla8fKj43aHNYrBdQx/oBvclegypl3R1XZnFGynTFVmcHZnWeko05RkCS4SKGrJuGGzGX7UblsCF1D5JctdVLgZMJEgrFqlwXf/+H6iXy735Mf93nu/N+H5mPnOPed9Pud8P98M3Nf9nPP5npOqQpKkyfzKsDsgSZq7DAlJUidDQpLUyZCQJHUyJCRJneYPuwPT7bjjjqslS5YMuxuSdFC55557flBVCybWD7mQWLJkCaOjo8PuhiQdVJJ8b7K6p5skSZ0MCUlSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUyZCQJHUyJCRJnfb5jesk64F3Ajuq6pS++keAi4AXgL+sqo+3+iXAha3+h1W1pdVXAJ8D5gFXV9Xlrb4U2AgcC9wDvK+qnk9yOHAt8CbgaeB3quq70/GhNXxf2vZ457b3nnHiLPZE0t7sz0jiGmBFfyHJbwErgTdU1euBP2n1k4FVwOvbPl9IMi/JPODzwDnAycD5rS3AZ4Erquq1wDP0Aob285lWv6K1kyTNon2GRFV9Hdg1ofwHwOVV9bPWZkerrwQ2VtXPquo7wBhwenuNVdVjVfU8vZHDyiQB3gLc1PbfAJzbd6wNbfkmYHlrL0maJVO9JvE64F8l2ZbkfyQ5rdUXAk/0tRtvta76scAPq2r3hPqLjtW2P9vaS5JmyVTvAjsfOAY4EzgNuDHJP562Xh2gJGuANQAnnuj5bEmaLlMdSYwDX6meu4CfA8cB24HFfe0WtVpX/WngqCTzJ9Tp36dtP7K1f4mqWltVI1U1smDBS26HLkmaoqmGxH8HfgsgyeuAw4AfAJuAVUkOb7OWlgF3AXcDy5IsTXIYvYvbm6qqgNuB89pxVwM3t+VNbZ22/bbWXpI0S/ZnCuz1wFnAcUnGgUuB9cD6JA8AzwOr2y/wB5PcCDwE7AYuqqoX2nE+DGyhNwV2fVU92N7iE8DGJJ8B7gPWtfo64ItJxuhdOF81DZ9XknQA9hkSVXV+x6bf62h/GXDZJPXNwOZJ6o/Rm/00sf5T4N376p8kaeb4jWtJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0KS1MmQkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVKnfYZEkvVJdrRHlU7c9kdJKslxbT1JrkwyluRbSU7ta7s6yaPttbqv/qYk97d9rkySVj8mydbWfmuSo6fnI0uS9tf+jCSuAVZMLCZZDJwNPN5XPgdY1l5rgKta22PoPRv7DHqPKr2075f+VcAH+/bb814XA7dW1TLg1rYuSZpF+wyJqvo6sGuSTVcAHweqr7YSuLZ67gSOSnIC8HZga1XtqqpngK3AirbtiKq6s6oKuBY4t+9YG9ryhr66JGmWTOmaRJKVwPaq+uaETQuBJ/rWx1ttb/XxSeoAx1fVk235KeD4vfRnTZLRJKM7d+480I8jSepwwCGR5NXAJ4H/OP3dmVwbZdRetq+tqpGqGlmwYMFsdUuSDnlTGUn8E2Ap8M0k3wUWAfcm+YfAdmBxX9tFrba3+qJJ6gDfb6ejaD93TKGvkqQBHHBIVNX9VfUPqmpJVS2hd4ro1Kp6CtgEXNBmOZ0JPNtOGW0Bzk5ydLtgfTawpW17LsmZbVbTBcDN7a02AXtmQa3uq0uSZsn+TIG9Hvg/wElJxpNcuJfmm4HHgDHgvwIfAqiqXcCngbvb61OtRmtzddvn74BbWv1y4G1JHgXe2tYlSbNo/r4aVNX5+9i+pG+5gIs62q0H1k9SHwVOmaT+NLB8X/2TJM0cv3EtSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI67fO2HNKh5kvbHp+0/t4zTpzlnkhznyMJSVInQ0KS1MmQkCR18pqEDnpeY5BmjiMJSVKn/Xky3fokO5I80Ff74yTfTvKtJF9NclTftkuSjCV5JMnb++orWm0sycV99aVJtrX6DUkOa/XD2/pY275kuj60JGn/7M9I4hpgxYTaVuCUqvrnwN8ClwAkORlYBby+7fOFJPOSzAM+D5wDnAyc39oCfBa4oqpeCzwD7Hk86oXAM61+RWsnSZpF+wyJqvo6sGtC7a+randbvRNY1JZXAhur6mdV9R16z60+vb3Gquqxqnoe2AisTBLgLcBNbf8NwLl9x9rQlm8Clrf2kqRZMh3XJH4fuKUtLwSe6Ns23mpd9WOBH/YFzp76i47Vtj/b2r9EkjVJRpOM7ty5c+APJEnqGSgkkvx7YDdw3fR0Z2qqam1VjVTVyIIFC4bZFUk6pEx5CmyS9wPvBJZXVbXydmBxX7NFrUZH/WngqCTz22ihv/2eY40nmQ8c2drrEOeUVmnumNJIIskK4OPAu6rqJ32bNgGr2sykpcAy4C7gbmBZm8l0GL2L25tauNwOnNf2Xw3c3Hes1W35POC2vjCSJM2CfY4kklwPnAUcl2QcuJTebKbDga3tWvKdVfVvqurBJDcCD9E7DXVRVb3QjvNhYAswD1hfVQ+2t/gEsDHJZ4D7gHWtvg74YpIxehfOV03D55U6OYKRXmqfIVFV509SXjdJbU/7y4DLJqlvBjZPUn+M3uynifWfAu/eV/8kSTPHb1xLkjoZEpKkToaEJKmTd4HVQaPrwrKkmeNIQpLUyZCQJHUyJCRJnQwJSVInQ0KS1MmQkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUaZ8hkWR9kh1JHuirHZNka5JH28+jWz1JrkwyluRbSU7t22d1a/9oktV99Tclub/tc2Xa81C73kOSNHv2ZyRxDbBiQu1i4NaqWgbc2tYBzgGWtdca4Cro/cKn92zsM+g9qvTSvl/6VwEf7NtvxT7eQ5I0S/bnGddfT7JkQnklcFZb3gDcAXyi1a+tqgLuTHJUkhNa261VtQsgyVZgRZI7gCOq6s5WvxY4F7hlL+8h7RefPyENbqrXJI6vqifb8lPA8W15IfBEX7vxVttbfXyS+t7e4yWSrEkymmR0586dU/g4kqTJDHzhuo0aahr6MuX3qKq1VTVSVSMLFiyYya5I0svKVEPi++00Eu3njlbfDizua7eo1fZWXzRJfW/vIUmaJVMNiU3AnhlKq4Gb++oXtFlOZwLPtlNGW4CzkxzdLlifDWxp255Lcmab1XTBhGNN9h6SpFmyzwvXSa6ndwH5uCTj9GYpXQ7cmORC4HvAe1rzzcA7gDHgJ8AHAKpqV5JPA3e3dp/acxEb+BC9GVSvonfB+pZW73oPSdIs2Z/ZTed3bFo+SdsCLuo4znpg/ST1UeCUSepPT/YekqTZ4zeuJUmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0KS1Gmf35OQXu667ib73jNOnOWeSLPPkYQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6GRKSpE4DhUSSf5vkwSQPJLk+ySuTLE2yLclYkhuSHNbaHt7Wx9r2JX3HuaTVH0ny9r76ilYbS3LxIH2VJB24KYdEkoXAHwIjVXUKMA9YBXwWuKKqXgs8A1zYdrkQeKbVr2jtSHJy2+/1wArgC0nmJZkHfB44BzgZOL+1lSTNkkFPN80HXpVkPvBq4EngLcBNbfsG4Ny2vLKt07YvT5JW31hVP6uq79B7Pvbp7TVWVY9V1fPAxtZWkjRLphwSVbUd+BPgcXrh8CxwD/DDqtrdmo0DC9vyQuCJtu/u1v7Y/vqEfbrqL5FkTZLRJKM7d+6c6keSJE0wyOmmo+n9Zb8U+EfAa+idLpp1VbW2qkaqamTBggXD6IIkHZIGOd30VuA7VbWzqv4f8BXgzcBR7fQTwCJge1veDiwGaNuPBJ7ur0/Yp6suSZolg4TE48CZSV7dri0sBx4CbgfOa21WAze35U1tnbb9tqqqVl/VZj8tBZYBdwF3A8vabKnD6F3c3jRAfyVJB2jKz5Ooqm1JbgLuBXYD9wFrgb8ENib5TKuta7usA76YZAzYRe+XPlX1YJIb6QXMbuCiqnoBIMmHgS30Zk6tr6oHp9pfSdKBS++P+UPHyMhIjY6ODrsb2oeuB/kcTHzokA4lSe6pqpGJdb9xLUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKnTlJ9MB5DkKOBq4BSggN8HHgFuAJYA3wXeU1XPtEecfg54B/AT4P1VdW87zmrgP7TDfqaqNrT6m4BrgFcBm4GP1qH2lCQdtLoenOTDiHQoGXQk8Tngr6rqnwJvAB4GLgZuraplwK1tHeAces+vXgasAa4CSHIMcClwBnA6cGmSo9s+VwEf7NtvxYD9lSQdgCmPJJIcCfwm8H6AqnoeeD7JSuCs1mwDcAfwCWAlcG0bCdyZ5KgkJ7S2W6tqVzvuVmBFkjuAI6rqzla/FjgXuGWqfdbsOxQeUyq9nA0yklgK7AT+PMl9Sa5O8hrg+Kp6srV5Cji+LS8Enujbf7zV9lYfn6QuSZolg4TEfOBU4KqqeiPwY355agmANmqY8WsISdYkGU0yunPnzpl+O0l62RjkwvU4MF5V29r6TfRC4vtJTqiqJ9vppB1t+3Zgcd/+i1ptO788PbWnfkerL5qk/UtU1VpgLcDIyIgXtjUneaFbB6MpjySq6ingiSQntdJy4CFgE7C61VYDN7flTcAF6TkTeLadltoCnJ3k6HbB+mxgS9v2XJIz28yoC/qOJUmaBQNNgQU+AlyX5DDgMeAD9ILnxiQXAt8D3tPabqY3/XWM3hTYDwBU1a4knwbubu0+teciNvAhfjkF9ha8aC1Js2qgkKiqbwAjk2xaPknbAi7qOM56YP0k9VF638GQJA2B37iWJHUyJCRJnQwJSVInQ0KS1GnQ2U0S4O03pEOVIwlJUidHEtI0c1SlQ4kjCUlSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0KS1GngkEgyL8l9Sb7W1pcm2ZZkLMkN7dGmJDm8rY+17Uv6jnFJqz+S5O199RWtNpbk4kH7Kkk6MNMxkvgo8HDf+meBK6rqtcAzwIWtfiHwTKtf0dqR5GRgFfB6YAXwhRY884DPA+cAJwPnt7aSpFkyUEgkWQT8NnB1Ww/wFuCm1mQDcG5bXtnWaduXt/YrgY1V9bOq+g4wBpzeXmNV9VhVPQ9sbG0lSbNk0JHEnwEfB37e1o8FflhVu9v6OLCwLS8EngBo259t7X9Rn7BPV/0lkqxJMppkdOfOnQN+JEnSHlMOiSTvBHZU1T3T2J8pqaq1VTVSVSMLFiwYdnck6ZAxyPMk3gy8K8k7gFcCRwCfA45KMr+NFhYB21v77cBiYDzJfOBI4Om++h79+3TVJUmzYMojiaq6pKoWVdUSeheeb6uq3wVuB85rzVYDN7flTW2dtv22qqpWX9VmPy0FlgF3AXcDy9psqcPae2yaan8lSQduJp5M9wlgY5LPAPcB61p9HfDFJGPALnq/9KmqB5PcCDwE7AYuqqoXAJJ8GNgCzAPWV9WDM9BfSVKHaQmJqroDuKMtP0ZvZtLENj8F3t2x/2XAZZPUNwObp6OPkqQD5zeuJUmdDAlJUidDQpLUyZCQJHWaidlNOoR9advjw+6CpFlkSGhShoEk8HSTJGkvDAlJUidPN0lDNpVTe+8948QZ6In0Uo4kJEmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVInp8BKB6GuabNOjdV0m/JIIsniJLcneSjJg0k+2urHJNma5NH28+hWT5Irk4wl+VaSU/uOtbq1fzTJ6r76m5Lc3/a5MkkG+bCSpAMzyOmm3cAfVdXJwJnARUlOBi4Gbq2qZcCtbR3gHHrPr14GrAGugl6oAJcCZ9B7ot2le4Kltflg334rBuivJOkATTkkqurJqrq3Lf898DCwEFgJbGjNNgDntuWVwLXVcydwVJITgLcDW6tqV1U9A2wFVrRtR1TVnVVVwLV9x5IkzYJpuXCdZAnwRmAbcHxVPdk2PQUc35YXAk/07Tbeanurj09Sn+z91yQZTTK6c+fOgT6LJOmXBg6JJL8KfBn4WFU917+tjQBq0PfYl6paW1UjVTWyYMGCmX47SXrZGGh2U5JX0AuI66rqK638/SQnVNWT7ZTRjlbfDizu231Rq20HzppQv6PVF03SXlPgbBhJUzHI7KYA64CHq+pP+zZtAvbMUFoN3NxXv6DNcjoTeLadltoCnJ3k6HbB+mxgS9v2XJIz23td0HcsSdIsGGQk8WbgfcD9Sb7Rap8ELgduTHIh8D3gPW3bZuAdwBjwE+ADAFW1K8mngbtbu09V1a62/CHgGuBVwC3tJUmaJVMOiar6G6DrewvLJ2lfwEUdx1oPrJ+kPgqcMtU+at98TKmkvfG2HJKkToaEJKmT926SDiHOYtN0MySklwHDQ1Pl6SZJUidDQpLUyZCQJHUyJCRJnQwJSVInZzdJL2POetK+OJKQJHVyJCHpJRxhaA9HEpKkTo4kJO03RxgvP44kJEmdHElIGpgjjEOXISFpxhgeB785HxJJVgCfA+YBV1fV5UPukqQBGR4HjzkdEknmAZ8H3gaMA3cn2VRVDw23Z5JmwlQep2uwzKw5HRLA6cBYVT0GkGQjsBKYkZDwrxvp4DPTz2l/uf//P9dDYiHwRN/6OHDGxEZJ1gBr2uqPkjwyxfc7DvjBxOLvTvFgM2DS/s0h9m8w9m8wM9K/afz/f67/+/36ZMW5HhL7parWAmsHPU6S0aoamYYuzQj7Nxj7Nxj7N5i53r8uc/17EtuBxX3ri1pNkjQL5npI3A0sS7I0yWHAKmDTkPskSS8bc/p0U1XtTvJhYAu9KbDrq+rBGXzLgU9ZzTD7Nxj7Nxj7N5i53r9JpaqG3QdJ0hw11083SZKGyJCQJHUyJJokK5I8kmQsycXD7k+/JOuT7EjywLD7Mpkki5PcnuShJA8m+eiw+9QvySuT3JXkm61//2nYfZpMknlJ7kvytWH3ZaIk301yf5JvJBkddn8mSnJUkpuSfDvJw0n+5bD7tEeSk9q/257Xc0k+Nux+7S+vSfCL23/8LX23/wDOnyu3/0jym8CPgGur6pRh92eiJCcAJ1TVvUl+DbgHOHcO/fsFeE1V/SjJK4C/AT5aVXcOuWsvkuTfASPAEVX1zmH3p1+S7wIjVTUnvwyWZAPwP6vq6jYT8tVV9cNh92ui9rtmO3BGVX1v2P3ZH44ken5x+4+qeh7Yc/uPOaGqvg7sGnY/ulTVk1V1b1v+e+Bhet+WnxOq50dt9RXtNaf+OkqyCPht4Oph9+Vgk+RI4DeBdQBV9fxcDIhmOfB3B0tAgCGxx2S3/5gzv+QOJkmWAG8Etg23Jy/WTuV8A9gBbK2qOdU/4M+AjwM/H3ZHOhTw10nuabfBmUuWAjuBP2+n665O8pphd6rDKuD6YXfiQBgSmjZJfhX4MvCxqnpu2P3pV1UvVNW/oPet/dOTzJnTdkneCeyoqnuG3Ze9+I2qOhU4B7ionQKdK+YDpwJXVdUbgR8Dc+q6IkA7DfYu4L8Nuy8HwpDo8fYfA2rn+r8MXFdVXxl2f7q00xC3AyuG3Zc+bwbe1c77bwTekuQvhtulF6uq7e3nDuCr9E7RzhXjwHjf6PAmeqEx15wD3FtV3x92Rw6EIdHj7T8G0C4MrwMerqo/HXZ/JkqyIMlRbflV9CYofHu4vfqlqrqkqhZV1RJ6/+3dVlW/N+Ru/UKS17QJCbTTOGcDc2amXVU9BTyR5KRWWs4MPU5gQOdzkJ1qgjl+W47ZMoTbfxyQJNcDZwHHJRkHLq2qdcPt1Yu8GXgfcH877w/wyaraPMQ+9TsB2NBmlvwKcGNVzblppnPY8cBXe38LMB/4UlX91XC79BIfAa5rf+Q9BnxgyP15kRaubwP+9bD7cqCcAitJ6uTpJklSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpoFST6V5K3D7od0oJwCK82wJPOq6oVh90OaCkcS0gCSLGnPMLiuPcfgpiSvbs9f+GySe4F3J7kmyXltn9OS/O/2fIu7kvxauwHhHye5O8m3khx0X7rSocmQkAZ3EvCFqvpnwHPAh1r96ao6tao27mnYvhF8A73nWbwBeCvwf4ELgWer6jTgNOCDSZbO5oeQJmNISIN7oqr+V1v+C+A32vINk7Q9CXiyqu4GqKrnqmo3vfshXdBua7INOBZYNrPdlvbNezdJg5t4YW/P+o8P4BgBPlJVW6anS9L0cCQhDe7Evmcqv5fe41G7PAKckOQ0gHY9Yj69m0v+QbvlOkleN4cfnKOXEUNCGtwj9B7E8zBwNHBVV8P2eNzfAf5zkm8CW4FX0nts6UPAvUkeAP4LjvQ1BzgFVhpAe1zr16pqzjzpTppOjiQkSZ0cSUiSOjmSkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdfr/w6wCcgK7/qwAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"VreeqDEtW47R"},"source":["타겟값이 정규분포형으로 변환"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"k9Rfx21oWMY9","executionInfo":{"status":"ok","timestamp":1629291869724,"user_tz":-540,"elapsed":248,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"59dc38c2-381e-4a4d-8f5f-c38cdb4ebbb9"},"source":["mercari_df['price'] = np.log1p(mercari_df['price'])\n","mercari_df['price'].head(3)"],"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0    2.397895\n","1    3.970292\n","2    2.397895\n","Name: price, dtype: float64"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"markdown","metadata":{"id":"exbgjTNdXTn2"},"source":["#### 각 피쳐 유형"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bXbAmiKZXPOo","executionInfo":{"status":"ok","timestamp":1629292208825,"user_tz":-540,"elapsed":247,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"4a7160ba-b1c3-4073-c778-cf82e4d27d64"},"source":["from termcolor import colored\n","\n","print(colored(\"item_condition_id 피쳐 유형: \\n\", 'blue'), mercari_df['item_condition_id'].value_counts())\n","print('\\n')\n","print(colored(\"shipping 피쳐 유형: \\n\", 'blue'), mercari_df['shipping'].value_counts())"],"execution_count":17,"outputs":[{"output_type":"stream","text":["\u001b[34mitem_condition_id 피쳐 유형: \n","\u001b[0m 1    640549\n","3    432161\n","2    375479\n","4     31962\n","5      2384\n","Name: item_condition_id, dtype: int64\n","\n","\n","\u001b[34mshipping 피쳐 유형: \n","\u001b[0m 0    819435\n","1    663100\n","Name: shipping, dtype: int64\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JhERlh1XX5W5","executionInfo":{"status":"ok","timestamp":1629292293977,"user_tz":-540,"elapsed":11,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"dc3ba177-0536-4493-d6bd-cf91db314de1"},"source":["# item description에 설명이 없는 것 count\n","boolean_cond = (mercari_df['item_description'] == 'No description yet')\n","mercari_df[boolean_cond]['item_description'].count()"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["82489"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"markdown","metadata":{"id":"tSXhzX9ZZAUo"},"source":["#### category name 피쳐 '/'기준 분리하여 개별 컬럼 재생성"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tSHU0FI_ZaSt","executionInfo":{"status":"ok","timestamp":1629292449896,"user_tz":-540,"elapsed":254,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"8a70347b-88aa-4738-c28c-d73fe443791e"},"source":["mercari_df['category_name'].value_counts()"],"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Women/Athletic Apparel/Pants, Tights, Leggings    60177\n","Women/Tops & Blouses/T-Shirts                     46380\n","Beauty/Makeup/Face                                34335\n","Beauty/Makeup/Lips                                29910\n","Electronics/Video Games & Consoles/Games          26557\n","                                                  ...  \n","Handmade/Bags and Purses/Diaper Bag                   1\n","Handmade/Pets/Blanket                                 1\n","Handmade/Holidays/Patriotic                           1\n","Handmade/Housewares/Bathroom                          1\n","Handmade/Furniture/Shelf                              1\n","Name: category_name, Length: 1287, dtype: int64"]},"metadata":{"tags":[]},"execution_count":19}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Pv2mvRseY2td","executionInfo":{"status":"ok","timestamp":1629292894144,"user_tz":-540,"elapsed":10529,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"21cb8c51-daa2-4818-9df9-dcc8e21a4b04"},"source":["# category '/' 분할 함수\n","def split_cat(category_name):\n","  try: \n","    return category_name.split('/')\n","  except:\n","    return ['Other_Null', 'Other_Null', 'Other_Null']\n","  \n","# 대분류, 중분류, 소분류로 분류\n","mercari_df['cat_1'], mercari_df['cat_2'], mercari_df['cat_3'] = zip(*mercari_df['category_name'].apply(lambda x: split_cat(x)))\n","\n","# 대분류, 중분류, 소분류 갯수\n","print('대분류 유형: ', mercari_df['cat_1'].value_counts())\n","print('중분류 갯수: ', mercari_df['cat_2'].nunique())\n","print('소분류 갯수: ', mercari_df['cat_3'].nunique())"],"execution_count":24,"outputs":[{"output_type":"stream","text":["대분류 유형:  Women                     664385\n","Beauty                    207828\n","Kids                      171689\n","Electronics               122690\n","Men                        93680\n","Home                       67871\n","Vintage & Collectibles     46530\n","Other                      45351\n","Handmade                   30842\n","Sports & Outdoors          25342\n","Other_Null                  6327\n","Name: cat_1, dtype: int64\n","중분류 갯수:  114\n","소분류 갯수:  871\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"uYB0EYQ8bbYC"},"source":["#### Null 값 처리"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uVS8EQOgbsDn","executionInfo":{"status":"ok","timestamp":1629293049576,"user_tz":-540,"elapsed":1331,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"055ba55a-12d6-446a-b370-eb4e76fa5759"},"source":["mercari_df.isnull().sum()"],"execution_count":25,"outputs":[{"output_type":"execute_result","data":{"text/plain":["train_id                  0\n","name                      0\n","item_condition_id         0\n","category_name          6327\n","brand_name           632682\n","price                     0\n","shipping                  0\n","item_description          4\n","cat_1                     0\n","cat_2                     0\n","cat_3                     0\n","dtype: int64"]},"metadata":{"tags":[]},"execution_count":25}]},{"cell_type":"code","metadata":{"id":"q56SRvlXapRP","executionInfo":{"status":"ok","timestamp":1629293055309,"user_tz":-540,"elapsed":855,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}}},"source":["mercari_df['brand_name'] = mercari_df['brand_name'].fillna(value='Other_Null')\n","mercari_df['category_name'] = mercari_df['item_description'].fillna(value='Other_Null')\n","mercari_df['item_description'] = mercari_df['item_description'].fillna(value='Other_Null')"],"execution_count":26,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"6yM5-ddTbybw"},"source":["# **피쳐 인코딩, 벡터화**"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"d_Jhze8Zb2za","executionInfo":{"status":"ok","timestamp":1629293091608,"user_tz":-540,"elapsed":1684,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"dd6ba678-4bf9-4e76-8368-c3b5b8ee5575"},"source":["mercari_df.info()"],"execution_count":27,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 1482535 entries, 0 to 1482534\n","Data columns (total 11 columns):\n"," #   Column             Non-Null Count    Dtype  \n","---  ------             --------------    -----  \n"," 0   train_id           1482535 non-null  int64  \n"," 1   name               1482535 non-null  object \n"," 2   item_condition_id  1482535 non-null  int64  \n"," 3   category_name      1482535 non-null  object \n"," 4   brand_name         1482535 non-null  object \n"," 5   price              1482535 non-null  float64\n"," 6   shipping           1482535 non-null  int64  \n"," 7   item_description   1482535 non-null  object \n"," 8   cat_1              1482535 non-null  object \n"," 9   cat_2              1482535 non-null  object \n"," 10  cat_3              1482535 non-null  object \n","dtypes: float64(1), int64(3), object(7)\n","memory usage: 124.4+ MB\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"ZU0XhG3vb-Qn"},"source":["##### brand_name 피쳐"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rn9vJsmQbwiE","executionInfo":{"status":"ok","timestamp":1629293222644,"user_tz":-540,"elapsed":249,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"2ccccaa9-9eab-43c5-d817-2f2031541245"},"source":["print('brand_name 유형 건수 :', mercari_df['brand_name'].nunique())"],"execution_count":30,"outputs":[{"output_type":"stream","text":["brand_name 유형 건수 : 4810\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"poWORNUUcIES","executionInfo":{"status":"ok","timestamp":1629293287389,"user_tz":-540,"elapsed":750,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"274a4f8b-a530-4586-a704-2aae428a4a63"},"source":["print('name 유형 건수 :', mercari_df['name'].nunique())"],"execution_count":32,"outputs":[{"output_type":"stream","text":["name 유형 건수 : 1225273\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"g1s6yJ90cqOk"},"source":["##### item_description 문자열 갯수"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"orUfTbM1cleM","executionInfo":{"status":"ok","timestamp":1629293448346,"user_tz":-540,"elapsed":1159,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"b4bdebd1-1ea3-404e-e93f-3f753d9f18f4"},"source":["pd.set_option('max_colwidth', 200)\n","\n","print('item_description 평균 문자열 갯수:', mercari_df['item_description'].str.len().mean())"],"execution_count":34,"outputs":[{"output_type":"stream","text":["item_description 평균 문자열 갯수: 145.7113889385411\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"-jM5vtXYdaee"},"source":["#### **피쳐 벡터화**\n","- name : count \n","- item_description : TF-IDF"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"sUDRq0SrdQXd","executionInfo":{"status":"ok","timestamp":1629294046096,"user_tz":-540,"elapsed":276409,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"abb9c5cc-451b-4339-ce99-46a4017c7952"},"source":["from sklearn.feature_extraction.text import CountVectorizer , TfidfVectorizer\n","\n","# name 피쳐\n","cnt_vec = CountVectorizer(max_features=30000)\n","X_name = cnt_vec.fit_transform(mercari_df['name'])\n","\n","# item_description 피쳐\n","tfidf_vec = TfidfVectorizer(max_features=50000, ngram_range=(1,3), stop_words='english')\n","X_descript = tfidf_vec.fit_transform(mercari_df['item_description'])\n","\n","print(\"name vecterization shape: \", X_name.shape)\n","print(\"item_description vecterization shape: \", X_descript.shape)"],"execution_count":35,"outputs":[{"output_type":"stream","text":["name vecterization shape:  (1482535, 30000)\n","item_description vecterization shape:  (1482535, 50000)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"30UWkQEuejSs"},"source":["#### **원핫인코딩 변환 후 희소행렬 최적화 형태로 저장**"]},{"cell_type":"code","metadata":{"id":"WbF08ALNefJk","executionInfo":{"status":"ok","timestamp":1629294456990,"user_tz":-540,"elapsed":179246,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}}},"source":["from sklearn.preprocessing import LabelBinarizer\n","\n","# brand_name 피쳐\n","lb_brand_name = LabelBinarizer(sparse_output=True)\n","X_brand = lb_brand_name.fit_transform(mercari_df['brand_name'])\n","\n","# item_condition_id 피쳐\n","lb_item_condition_id = LabelBinarizer(sparse_output=True)\n","X_item_condition_id = lb_item_condition_id.fit_transform(mercari_df['item_condition_id'])\n","\n","# shipping 피쳐\n","lb_shipping = LabelBinarizer(sparse_output=True)\n","X_shipping = lb_shipping.fit_transform(mercari_df['shipping'])\n","\n","# cat_1, cat_2, cat_3 피쳐\n","lb_cat_1 = LabelBinarizer(sparse_output=True)\n","X_cat_1 = lb_cat_1.fit_transform(mercari_df['cat_1'])\n","\n","lb_cat_2 = LabelBinarizer(sparse_output=True)\n","X_cat_2 = lb_cat_2.fit_transform(mercari_df['cat_1'])\n","\n","lb_cat_3 = LabelBinarizer(sparse_output=True)\n","X_cat_3 = lb_cat_3.fit_transform(mercari_df['cat_1'])"],"execution_count":37,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MUkaUb2Lf0ps","executionInfo":{"status":"ok","timestamp":1629294494932,"user_tz":-540,"elapsed":251,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"4573d33e-9f79-426e-f6e6-976686611a92"},"source":["print(type(X_brand), type(X_item_condition_id), type(X_shipping))\n","print('X_brand_shape:{0}, X_item_condition_id shape:{1}'.format(X_brand.shape, X_item_condition_id.shape))\n","print('X_shipping shape:{0}, X_cat_1 shape:{1}'.format(X_shipping.shape, X_cat_1.shape))\n","print('X_cat_2 shape:{0}, X_cat_3 shape:{1}'.format(X_cat_2.shape, X_cat_3.shape))"],"execution_count":39,"outputs":[{"output_type":"stream","text":["<class 'scipy.sparse.csr.csr_matrix'> <class 'scipy.sparse.csr.csr_matrix'> <class 'scipy.sparse.csr.csr_matrix'>\n","X_brand_shape:(1482535, 4810), X_item_condition_id shape:(1482535, 5)\n","X_shipping shape:(1482535, 1), X_cat_1 shape:(1482535, 11)\n","X_cat_2 shape:(1482535, 11), X_cat_3 shape:(1482535, 11)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"n_J7NSVagBb7","executionInfo":{"status":"ok","timestamp":1629294504061,"user_tz":-540,"elapsed":254,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"9f989861-aa65-466f-9543-9efae74e1939"},"source":["import gc\n","gc.collect()"],"execution_count":40,"outputs":[{"output_type":"execute_result","data":{"text/plain":["1117"]},"metadata":{"tags":[]},"execution_count":40}]},{"cell_type":"markdown","metadata":{"id":"Ip_Cxv8ugM40"},"source":["#### 피쳐벡터화 피쳐와 원핫인코딩화 피쳐 희소행렬들 결합"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RWOhEOY5gJxa","executionInfo":{"status":"ok","timestamp":1629294574946,"user_tz":-540,"elapsed":4680,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"04428211-29f1-46f0-a46c-e9959da77e14"},"source":["from scipy.sparse import hstack\n","\n","sparse_matrix_list = (X_name, X_descript, X_brand, X_item_condition_id, X_shipping,\n","                      X_cat_1, X_cat_2, X_cat_3)\n","\n","X_features_sparse = hstack(sparse_matrix_list).tocsr()\n","print(type(X_features_sparse))\n","print(X_features_sparse.shape)"],"execution_count":41,"outputs":[{"output_type":"stream","text":["<class 'scipy.sparse.csr.csr_matrix'>\n","(1482535, 84849)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Ad_t1xF1hjhU"},"source":["# **모델 구축**"]},{"cell_type":"markdown","metadata":{"id":"F2kYubsJjuwf"},"source":["#### rmsle 정의\n"]},{"cell_type":"code","metadata":{"id":"72CQNpzchimp","executionInfo":{"status":"ok","timestamp":1629295374269,"user_tz":-540,"elapsed":232,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}}},"source":["def rmsle(y, y_pred):\n","  return np.sqrt(np.mean(np.power(np.log1p(y) - np.log1p(y_pred), 2)))\n","\n","def evaluate_org_price(y_test, preds):\n","  # 로그변환이 되어있으므로 복구\n","  preds_exmpm = np.expm1(preds)\n","  y_test_exmpm = np.expm1(y_test)\n","\n","  # rmsle 값\n","  rmsle_result = rmsle(y_test_exmpm, preds_exmpm)\n","  return rmsle_result"],"execution_count":42,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Tay2bDrPkp0g"},"source":["#### 여러 모델 학습/예측을 위한 함수 생성\n","* 여러 희소 행렬을 hstack()으로 결합한 뒤 학습과 테스트 데이터 세트로 분할 후 모델 학습 및 예측을 수행"]},{"cell_type":"code","metadata":{"id":"iwlVF7-Qkm1i","executionInfo":{"status":"ok","timestamp":1629295592487,"user_tz":-540,"elapsed":253,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}}},"source":["import gc \n","from  scipy.sparse import hstack\n","from sklearn.model_selection import train_test_split , cross_val_score\n","\n","def model_train_predict(model,matrix_list):\n","    # sparse matrix 결합\n","    X= hstack(matrix_list).tocsr()     \n","    \n","    # 학습/테스트 데이터 분할\n","    X_train, X_test, y_train, y_test=train_test_split(X, mercari_df['price'], \n","                                                      test_size=0.2, random_state=216)\n","    \n","    # 모델 학습 및 예측\n","    model.fit(X_train , y_train)\n","    preds = model.predict(X_test)\n","    \n","    del X , X_train , X_test , y_train \n","    gc.collect()\n","    \n","    return preds , y_test"],"execution_count":43,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"nqDFCEWzlkfR"},"source":["#### Lidge 선형 회귀 모델"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JixXnmGClTBg","executionInfo":{"status":"ok","timestamp":1629295832148,"user_tz":-540,"elapsed":33223,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"69dd0111-6622-4716-d7d6-f34187297087"},"source":["from sklearn.linear_model import Ridge , LogisticRegression\n","\n","linear_model = Ridge(solver = 'lsqr', fit_intercept=False)\n","\n","sparse_matrix_list = (X_name, X_descript, X_brand, X_item_condition_id, X_shipping,\n","                      X_cat_1, X_cat_2, X_cat_3)\n","linear_preds, y_test = model_train_predict(model=linear_model, matrix_list=sparse_matrix_list)\n","print(\"rmsle: \", evaluate_org_price(y_test, linear_preds))"],"execution_count":44,"outputs":[{"output_type":"stream","text":["rmsle:  0.48703297925833866\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"umsErfp7mOkQ","executionInfo":{"status":"ok","timestamp":1629295842666,"user_tz":-540,"elapsed":9317,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"2634ccdc-c852-4acf-a6a6-5c5fc721d956"},"source":["# Item description 피쳐가 없을 때\n","sparse_matrix_list = (X_name, X_brand, X_item_condition_id, X_shipping,\n","                      X_cat_1, X_cat_2, X_cat_3)\n","linear_preds, y_test = model_train_predict(model=linear_model, matrix_list=sparse_matrix_list)\n","print(\"rmsle: \", evaluate_org_price(y_test, linear_preds))"],"execution_count":45,"outputs":[{"output_type":"stream","text":["rmsle:  0.5276743826600357\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"RJXcirBUmXI5"},"source":["# LightGBM 회귀 모델"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_6rTO5o2mVWI","executionInfo":{"status":"ok","timestamp":1629298289926,"user_tz":-540,"elapsed":1649423,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"36002dde-4b92-4ebe-9059-b5c7627f1157"},"source":["from lightgbm import LGBMRegressor\n","\n","sparse_matrix_list = (X_name, X_descript, X_brand, X_item_condition_id, X_shipping,\n","                      X_cat_1, X_cat_2, X_cat_3)\n","\n","lgbm_model = LGBMRegressor(n_estimators=200, learning_rate=0.5, num_leaves=125, random_state=216)\n","lgbm_preds , y_test = model_train_predict(model=lgbm_model, matrix_list=sparse_matrix_list)\n","print(\"rmsle: \", evaluate_org_price(y_test, lgbm_preds))"],"execution_count":47,"outputs":[{"output_type":"stream","text":["rmsle:  0.46973196295270087\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QD7COyVNpsPA","executionInfo":{"status":"ok","timestamp":1629298293742,"user_tz":-540,"elapsed":266,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"e23beaa7-a695-400e-fa67-926a252179e8"},"source":["import gc\n","gc.collect()"],"execution_count":48,"outputs":[{"output_type":"execute_result","data":{"text/plain":["335"]},"metadata":{"tags":[]},"execution_count":48}]},{"cell_type":"markdown","metadata":{"id":"58zDtKr4penF"},"source":["#### lgbm + Ridge ensemble"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rfR-sGP0pWwZ","executionInfo":{"status":"ok","timestamp":1629299449761,"user_tz":-540,"elapsed":374,"user":{"displayName":"김별희","photoUrl":"","userId":"06602448826503759202"}},"outputId":"e9c860ec-8678-48b8-9677-0774ea9bd632"},"source":["preds = lgbm_preds * 0.48 + linear_preds * 0.52\n","print('LightGBM과 Ridge를 ensemble한 최종 rmsle 값:',  evaluate_org_price(y_test , preds))"],"execution_count":50,"outputs":[{"output_type":"stream","text":["LightGBM과 Ridge를 ensemble한 최종 rmsle 값: 0.4777924822201176\n"],"name":"stdout"}]}]}