MInkunKim_datacleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('/home/minkun/Desktop/BigCloud/assignment1/data/1st.csv')\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50000 entries, 0 to 49999\n",
      "Data columns (total 22 columns):\n",
      "Id                       50000 non-null int64\n",
      "PostTypeId               50000 non-null int64\n",
      "AcceptedAnswerId         42335 non-null float64\n",
      "ParentId                 0 non-null float64\n",
      "CreationDate             50000 non-null object\n",
      "DeletionDate             0 non-null float64\n",
      "Score                    50000 non-null int64\n",
      "ViewCount                50000 non-null int64\n",
      "Body                     50000 non-null object\n",
      "OwnerUserId              48549 non-null float64\n",
      "OwnerDisplayName         6035 non-null object\n",
      "LastEditorUserId         42141 non-null float64\n",
      "LastEditorDisplayName    3580 non-null object\n",
      "LastEditDate             42576 non-null object\n",
      "LastActivityDate         50000 non-null object\n",
      "Title                    50000 non-null object\n",
      "Tags                     50000 non-null object\n",
      "AnswerCount              50000 non-null int64\n",
      "CommentCount             50000 non-null int64\n",
      "FavoriteCount            49758 non-null float64\n",
      "ClosedDate               4571 non-null object\n",
      "CommunityOwnedDate       579 non-null object\n",
      "dtypes: float64(6), int64(6), object(10)\n",
      "memory usage: 8.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df1.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    50000\n",
       "Name: PostTypeId, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.PostTypeId.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    <p>I accidentally committed the wrong files to...\n",
       "1    <p>I want to delete a branch both locally and ...\n",
       "2    <p>How can I convert a <code>String</code> to ...\n",
       "3    <p>I have an array of Numbers, and I'm using t...\n",
       "4    <p>How can I redirect the user from one page t...\n",
       "Name: Body, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.Body.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<p>I accidentally committed the wrong files to <a href=\"https://en.wikipedia.org/wiki/Git_software\" rel=\"noreferrer\">Git</a>, but I haven\\'t pushed the commit to the server yet.</p>\\n\\n<p><strong>How can I undo those commits from the local repository?</strong> </p>\\n'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.Body[0] #df1.Body[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<p>I want to delete a branch both locally and on my remote project fork on <a href=\"http://en.wikipedia.org/wiki/GitHub\" rel=\"noreferrer\">GitHub</a>.</p>\\n\\n<h3>Failed Attempts to Delete Remote Branch</h3>\\n\\n<pre><code>$ git branch -d remotes/origin/bugfix\\nerror: branch \\'remotes/origin/bugfix\\' not found.\\n\\n$ git branch -d origin/bugfix\\nerror: branch \\'origin/bugfix\\' not found.\\n\\n$ git branch -rd origin/bugfix\\nDeleted remote branch origin/bugfix (was 2a14ef7).\\n\\n$ git push\\nEverything up-to-date\\n\\n$ git pull\\nFrom github.com:gituser/gitproject\\n* [new branch] bugfix -&gt; origin/bugfix\\nAlready up-to-date.\\n</code></pre>\\n\\n<p>What do I need to do differently to successfully delete the\\n<code>remotes/origin/bugfix</code> branch both locally and on GitHub?</p>\\n'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.Body[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    <git><git-bash><git-commit><git-reset><git-rev...\n",
       "1                        <git><git-branch><git-remote>\n",
       "2                 <java><string><int><type-conversion>\n",
       "3                                 <javascript><arrays>\n",
       "4                       <javascript><jquery><redirect>\n",
       "Name: Tags, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.Tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<git><git-bash><git-commit><git-reset><git-revert>'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.Tags[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<git><git-branch><git-remote>'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.Tags[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df1.Tags[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>PostTypeId</th>\n",
       "      <th>AcceptedAnswerId</th>\n",
       "      <th>ParentId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>DeletionDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>ViewCount</th>\n",
       "      <th>Body</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>...</th>\n",
       "      <th>LastEditorDisplayName</th>\n",
       "      <th>LastEditDate</th>\n",
       "      <th>LastActivityDate</th>\n",
       "      <th>Title</th>\n",
       "      <th>Tags</th>\n",
       "      <th>AnswerCount</th>\n",
       "      <th>CommentCount</th>\n",
       "      <th>FavoriteCount</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>CommunityOwnedDate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5558</th>\n",
       "      <td>19943766</td>\n",
       "      <td>1</td>\n",
       "      <td>19993403.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-11-13 01:53:24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>233</td>\n",
       "      <td>350803</td>\n",
       "      <td>&lt;p&gt;I'm currently configuring hadoop on a serve...</td>\n",
       "      <td>768439.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-09-20 13:14:58</td>\n",
       "      <td>2018-04-16 12:09:55</td>\n",
       "      <td>Hadoop \"Unable to load native-hadoop library f...</td>\n",
       "      <td>&lt;java&gt;&lt;linux&gt;&lt;hadoop&gt;&lt;hadoop2&gt;&lt;nativelibrary&gt;</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>103.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13614</th>\n",
       "      <td>29704333</td>\n",
       "      <td>1</td>\n",
       "      <td>29705881.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-17 16:10:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96</td>\n",
       "      <td>208600</td>\n",
       "      <td>&lt;p&gt;I would like to read a CSV in spark and con...</td>\n",
       "      <td>2411173.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-07-24 14:00:35</td>\n",
       "      <td>2019-02-14 07:01:32</td>\n",
       "      <td>Spark - load CSV file as DataFrame?</td>\n",
       "      <td>&lt;scala&gt;&lt;apache-spark&gt;&lt;hadoop&gt;&lt;apache-spark-sql...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>48.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id  PostTypeId  AcceptedAnswerId  ParentId         CreationDate  \\\n",
       "5558   19943766           1        19993403.0       NaN  2013-11-13 01:53:24   \n",
       "13614  29704333           1        29705881.0       NaN  2015-04-17 16:10:21   \n",
       "\n",
       "       DeletionDate  Score  ViewCount  \\\n",
       "5558            NaN    233     350803   \n",
       "13614           NaN     96     208600   \n",
       "\n",
       "                                                    Body  OwnerUserId  \\\n",
       "5558   <p>I'm currently configuring hadoop on a serve...     768439.0   \n",
       "13614  <p>I would like to read a CSV in spark and con...    2411173.0   \n",
       "\n",
       "              ...         LastEditorDisplayName         LastEditDate  \\\n",
       "5558          ...                           NaN  2015-09-20 13:14:58   \n",
       "13614         ...                           NaN  2018-07-24 14:00:35   \n",
       "\n",
       "          LastActivityDate                                              Title  \\\n",
       "5558   2018-04-16 12:09:55  Hadoop \"Unable to load native-hadoop library f...   \n",
       "13614  2019-02-14 07:01:32                Spark - load CSV file as DataFrame?   \n",
       "\n",
       "                                                    Tags AnswerCount  \\\n",
       "5558       <java><linux><hadoop><hadoop2><nativelibrary>          20   \n",
       "13614  <scala><apache-spark><hadoop><apache-spark-sql...           9   \n",
       "\n",
       "      CommentCount  FavoriteCount  ClosedDate  CommunityOwnedDate  \n",
       "5558             2          103.0         NaN                 NaN  \n",
       "13614            1           48.0         NaN                 NaN  \n",
       "\n",
       "[2 rows x 22 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[df1['Body'].str.contains(\"hadoop\")==True].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>PostTypeId</th>\n",
       "      <th>AcceptedAnswerId</th>\n",
       "      <th>ParentId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>DeletionDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>ViewCount</th>\n",
       "      <th>Body</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>...</th>\n",
       "      <th>LastEditorDisplayName</th>\n",
       "      <th>LastEditDate</th>\n",
       "      <th>LastActivityDate</th>\n",
       "      <th>Title</th>\n",
       "      <th>Tags</th>\n",
       "      <th>AnswerCount</th>\n",
       "      <th>CommentCount</th>\n",
       "      <th>FavoriteCount</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>CommunityOwnedDate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5558</th>\n",
       "      <td>19943766</td>\n",
       "      <td>1</td>\n",
       "      <td>19993403.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-11-13 01:53:24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>233</td>\n",
       "      <td>350803</td>\n",
       "      <td>&lt;p&gt;I'm currently configuring hadoop on a serve...</td>\n",
       "      <td>768439.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-09-20 13:14:58</td>\n",
       "      <td>2018-04-16 12:09:55</td>\n",
       "      <td>Hadoop \"Unable to load native-hadoop library f...</td>\n",
       "      <td>&lt;java&gt;&lt;linux&gt;&lt;hadoop&gt;&lt;hadoop2&gt;&lt;nativelibrary&gt;</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>103.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13105</th>\n",
       "      <td>17837871</td>\n",
       "      <td>1</td>\n",
       "      <td>17838068.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-07-24 15:03:27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>106</td>\n",
       "      <td>213501</td>\n",
       "      <td>&lt;p&gt;How to copy file from HDFS to the local fil...</td>\n",
       "      <td>2499617.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-21 11:50:46</td>\n",
       "      <td>2018-03-06 22:32:11</td>\n",
       "      <td>How to copy file from HDFS to the local file s...</td>\n",
       "      <td>&lt;hadoop&gt;&lt;copy&gt;&lt;hdfs&gt;</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id  PostTypeId  AcceptedAnswerId  ParentId         CreationDate  \\\n",
       "5558   19943766           1        19993403.0       NaN  2013-11-13 01:53:24   \n",
       "13105  17837871           1        17838068.0       NaN  2013-07-24 15:03:27   \n",
       "\n",
       "       DeletionDate  Score  ViewCount  \\\n",
       "5558            NaN    233     350803   \n",
       "13105           NaN    106     213501   \n",
       "\n",
       "                                                    Body  OwnerUserId  \\\n",
       "5558   <p>I'm currently configuring hadoop on a serve...     768439.0   \n",
       "13105  <p>How to copy file from HDFS to the local fil...    2499617.0   \n",
       "\n",
       "              ...         LastEditorDisplayName         LastEditDate  \\\n",
       "5558          ...                           NaN  2015-09-20 13:14:58   \n",
       "13105         ...                           NaN  2015-04-21 11:50:46   \n",
       "\n",
       "          LastActivityDate                                              Title  \\\n",
       "5558   2018-04-16 12:09:55  Hadoop \"Unable to load native-hadoop library f...   \n",
       "13105  2018-03-06 22:32:11  How to copy file from HDFS to the local file s...   \n",
       "\n",
       "                                                Tags AnswerCount CommentCount  \\\n",
       "5558   <java><linux><hadoop><hadoop2><nativelibrary>          20            2   \n",
       "13105                           <hadoop><copy><hdfs>           6            0   \n",
       "\n",
       "       FavoriteCount  ClosedDate  CommunityOwnedDate  \n",
       "5558           103.0         NaN                 NaN  \n",
       "13105           33.0         NaN                 NaN  \n",
       "\n",
       "[2 rows x 22 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[df1['Tags'].str.contains(\"hadoop\")==True].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1.drop(['Body'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv('/home/minkun/Desktop/BigCloud/assignment1/data/2nd.csv')\n",
    "df3 = pd.read_csv('/home/minkun/Desktop/BigCloud/assignment1/data/3rd.csv')\n",
    "df4 = pd.read_csv('/home/minkun/Desktop/BigCloud/assignment1/data/4th.csv')\n",
    "df5 = pd.read_csv('/home/minkun/Desktop/BigCloud/assignment1/data/extra50.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.drop(['Body'], axis=1, inplace=True)\n",
    "df3.drop(['Body'], axis=1, inplace=True)\n",
    "df4.drop(['Body'], axis=1, inplace=True)\n",
    "df5.drop(['Body'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine = df1.append(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine = df_fine.append(df3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine = df_fine.append(df4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine = df_fine.append(df5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200050 entries, 0 to 200049\n",
      "Data columns (total 21 columns):\n",
      "Id                       200050 non-null int64\n",
      "PostTypeId               200050 non-null int64\n",
      "AcceptedAnswerId         164823 non-null float64\n",
      "ParentId                 0 non-null float64\n",
      "CreationDate             200050 non-null object\n",
      "DeletionDate             0 non-null float64\n",
      "Score                    200050 non-null int64\n",
      "ViewCount                200050 non-null int64\n",
      "OwnerUserId              195070 non-null float64\n",
      "OwnerDisplayName         16860 non-null object\n",
      "LastEditorUserId         149856 non-null float64\n",
      "LastEditorDisplayName    9737 non-null object\n",
      "LastEditDate             151552 non-null object\n",
      "LastActivityDate         200050 non-null object\n",
      "Title                    200050 non-null object\n",
      "Tags                     200050 non-null object\n",
      "AnswerCount              200050 non-null int64\n",
      "CommentCount             200050 non-null int64\n",
      "FavoriteCount            191268 non-null float64\n",
      "ClosedDate               15900 non-null object\n",
      "CommunityOwnedDate       1172 non-null object\n",
      "dtypes: float64(6), int64(6), object(9)\n",
      "memory usage: 32.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_fine.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df_fine.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200035"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fine.Id.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>PostTypeId</th>\n",
       "      <th>AcceptedAnswerId</th>\n",
       "      <th>ParentId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>DeletionDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>ViewCount</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>OwnerDisplayName</th>\n",
       "      <th>...</th>\n",
       "      <th>LastEditorDisplayName</th>\n",
       "      <th>LastEditDate</th>\n",
       "      <th>LastActivityDate</th>\n",
       "      <th>Title</th>\n",
       "      <th>Tags</th>\n",
       "      <th>AnswerCount</th>\n",
       "      <th>CommentCount</th>\n",
       "      <th>FavoriteCount</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>CommunityOwnedDate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50000</th>\n",
       "      <td>9647561</td>\n",
       "      <td>1</td>\n",
       "      <td>9648264.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-03-10 15:22:05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>34</td>\n",
       "      <td>87000</td>\n",
       "      <td>219167.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-08-25 04:07:03</td>\n",
       "      <td>2017-11-15 18:57:14</td>\n",
       "      <td>How to connect to MongoDB in Windows?</td>\n",
       "      <td>&lt;windows&gt;&lt;mongodb&gt;</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50001</th>\n",
       "      <td>36985112</td>\n",
       "      <td>1</td>\n",
       "      <td>36987506.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-02 14:38:22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49</td>\n",
       "      <td>87000</td>\n",
       "      <td>2410379.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-03 15:45:46</td>\n",
       "      <td>2018-07-19 09:16:26</td>\n",
       "      <td>Angular2, what is the correct way to disable a...</td>\n",
       "      <td>&lt;html&gt;&lt;dom&gt;&lt;typescript&gt;&lt;angular&gt;&lt;angular2-temp...</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50002</th>\n",
       "      <td>57484</td>\n",
       "      <td>1</td>\n",
       "      <td>57526.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-09-11 20:04:17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>75</td>\n",
       "      <td>87000</td>\n",
       "      <td>1310.0</td>\n",
       "      <td>ScArcher2</td>\n",
       "      <td>...</td>\n",
       "      <td>ScArcher2</td>\n",
       "      <td>2008-09-11 20:10:16</td>\n",
       "      <td>2017-03-21 23:30:10</td>\n",
       "      <td>How do you \"OR\" criteria together when using a...</td>\n",
       "      <td>&lt;java&gt;&lt;hibernate&gt;</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Id  PostTypeId  AcceptedAnswerId  ParentId         CreationDate  \\\n",
       "50000   9647561           1         9648264.0       NaN  2012-03-10 15:22:05   \n",
       "50001  36985112           1        36987506.0       NaN  2016-05-02 14:38:22   \n",
       "50002     57484           1           57526.0       NaN  2008-09-11 20:04:17   \n",
       "\n",
       "       DeletionDate  Score  ViewCount  OwnerUserId OwnerDisplayName  \\\n",
       "50000           NaN     34      87000     219167.0              NaN   \n",
       "50001           NaN     49      87000    2410379.0              NaN   \n",
       "50002           NaN     75      87000       1310.0        ScArcher2   \n",
       "\n",
       "             ...          LastEditorDisplayName         LastEditDate  \\\n",
       "50000        ...                            NaN  2015-08-25 04:07:03   \n",
       "50001        ...                            NaN  2016-05-03 15:45:46   \n",
       "50002        ...                      ScArcher2  2008-09-11 20:10:16   \n",
       "\n",
       "          LastActivityDate                                              Title  \\\n",
       "50000  2017-11-15 18:57:14              How to connect to MongoDB in Windows?   \n",
       "50001  2018-07-19 09:16:26  Angular2, what is the correct way to disable a...   \n",
       "50002  2017-03-21 23:30:10  How do you \"OR\" criteria together when using a...   \n",
       "\n",
       "                                                    Tags AnswerCount  \\\n",
       "50000                                 <windows><mongodb>          10   \n",
       "50001  <html><dom><typescript><angular><angular2-temp...           7   \n",
       "50002                                  <java><hibernate>           6   \n",
       "\n",
       "       CommentCount  FavoriteCount  ClosedDate CommunityOwnedDate  \n",
       "50000             5           10.0         NaN                NaN  \n",
       "50001             0            7.0         NaN                NaN  \n",
       "50002             0           28.0         NaN                NaN  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fine[df_fine.duplicated()==True].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 200035 entries, 0 to 200049\n",
      "Data columns (total 21 columns):\n",
      "Id                       200035 non-null int64\n",
      "PostTypeId               200035 non-null int64\n",
      "AcceptedAnswerId         164813 non-null float64\n",
      "ParentId                 0 non-null float64\n",
      "CreationDate             200035 non-null object\n",
      "DeletionDate             0 non-null float64\n",
      "Score                    200035 non-null int64\n",
      "ViewCount                200035 non-null int64\n",
      "OwnerUserId              195055 non-null float64\n",
      "OwnerDisplayName         16858 non-null object\n",
      "LastEditorUserId         149845 non-null float64\n",
      "LastEditorDisplayName    9736 non-null object\n",
      "LastEditDate             151541 non-null object\n",
      "LastActivityDate         200035 non-null object\n",
      "Title                    200035 non-null object\n",
      "Tags                     200035 non-null object\n",
      "AnswerCount              200035 non-null int64\n",
      "CommentCount             200035 non-null int64\n",
      "FavoriteCount            191253 non-null float64\n",
      "ClosedDate               15899 non-null object\n",
      "CommunityOwnedDate       1172 non-null object\n",
      "dtypes: float64(6), int64(6), object(9)\n",
      "memory usage: 33.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df_fine.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fine.to_csv('/home/minkun/Desktop/BigCloud/assignment1/data/final_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}