4_language_filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import pandas as pd\n",
    "from langdetect import detect_langs\n",
    "import multiprocessing as mp\n",
    "import tqdm\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>score</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CryptoHODLer101</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1513292107</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No BTC is not fiat. I get paid in Bitcoin. As ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nappiestapparatus</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1395194709</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I think you're overestimating your hashing pow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Skating2Death</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1390016247</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I've heard of it, but with the high volatility...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OmarJunkman</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1528300155</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Do you have any bitcoin?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>evanc1411</td>\n",
       "      <td>ASU</td>\n",
       "      <td>1525380807</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>BTC  -?  Damn, when you buy a Bitcoin you're ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              author subreddit  created_utc  score  controversiality  \\\n",
       "0    CryptoHODLer101       AMA   1513292107      0                 0   \n",
       "1  nappiestapparatus       AMA   1395194709      0                 0   \n",
       "2      Skating2Death       AMA   1390016247      0                 0   \n",
       "3        OmarJunkman       AMA   1528300155      0                 0   \n",
       "4          evanc1411       ASU   1525380807      0                 1   \n",
       "\n",
       "                                                body  \n",
       "0  No BTC is not fiat. I get paid in Bitcoin. As ...  \n",
       "1  I think you're overestimating your hashing pow...  \n",
       "2  I've heard of it, but with the high volatility...  \n",
       "3                           Do you have any bitcoin?  \n",
       "4   BTC  -?  Damn, when you buy a Bitcoin you're ...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import data\n",
    "data = pd.read_csv('clean_data_odd.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>score</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>body</th>\n",
       "      <th>has_char</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CryptoHODLer101</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1513292107</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No BTC is not fiat. I get paid in Bitcoin. As ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nappiestapparatus</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1395194709</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I think you're overestimating your hashing pow...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Skating2Death</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1390016247</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I've heard of it, but with the high volatility...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OmarJunkman</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1528300155</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Do you have any bitcoin?</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>evanc1411</td>\n",
       "      <td>ASU</td>\n",
       "      <td>1525380807</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>BTC  -?  Damn, when you buy a Bitcoin you're ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              author subreddit  created_utc  score  controversiality  \\\n",
       "0    CryptoHODLer101       AMA   1513292107      0                 0   \n",
       "1  nappiestapparatus       AMA   1395194709      0                 0   \n",
       "2      Skating2Death       AMA   1390016247      0                 0   \n",
       "3        OmarJunkman       AMA   1528300155      0                 0   \n",
       "4          evanc1411       ASU   1525380807      0                 1   \n",
       "\n",
       "                                                body  has_char  \n",
       "0  No BTC is not fiat. I get paid in Bitcoin. As ...      True  \n",
       "1  I think you're overestimating your hashing pow...      True  \n",
       "2  I've heard of it, but with the high volatility...      True  \n",
       "3                           Do you have any bitcoin?      True  \n",
       "4   BTC  -?  Damn, when you buy a Bitcoin you're ...      True  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter out coments where there are no alphabetical characters\n",
    "# will mess with language detection if left in!\n",
    "data['has_char'] = data['body'].apply(str).str.contains('[a-zA-Z]+')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     1607781\n",
       "False          1\n",
       "Name: has_char, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see how many returned\n",
    "data['has_char'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True    1607781\n",
       "Name: has_char, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter out the flase results\n",
    "data = data.loc[data['has_char'] == True]\n",
    "\n",
    "#check to make sure that they were filtered out\n",
    "data['has_char'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reindex since data has been filtered out\n",
    "data = data.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## method 1\n",
    "- #### filter via subreddit using R\n",
    "- #### go to part 2_clean_reddit_comments.md "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## > method 2\n",
    "- #### filter via language detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```Python\n",
    "# create new ['language'] column\n",
    "data['language'] = data['body'].apply(detect_langs)\n",
    "\n",
    "# new column ['english'] returns 'True' if english has any probability of being the language\n",
    "data['english'] = data['language'].apply(str).str.contains('en')\n",
    "\n",
    "# see how many were classified as each\n",
    "data['english'].value_counts()\n",
    "\n",
    "# see what was classified as false\n",
    "data[data['english'] == False].head()\n",
    "\n",
    "# parse data to only return values where ['english'] is True\n",
    "data = data[(data['english'] == True)]\n",
    "\n",
    "#reset index as indexes of filtered data are deleted\n",
    "data = data.reset_index(drop = True)\n",
    "\n",
    "# drop data that we don't need anymore\n",
    "del data['language']\n",
    "del data['english']\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## > method 3\n",
    "- #### method 2 but as a function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```Python\n",
    "# Arguments: (df = your dataframe), (series = name of column in df as string), (language_select = two letter string of language code that you want)\n",
    "\n",
    "def language_filter(df, series = str, language_select = str):\n",
    "    \n",
    "    # create copied df\n",
    "    df_copy = df.copy()\n",
    "    \n",
    "    # create ['languague'] from output of detect_langs\n",
    "    df_copy['language'] = df_copy[series].apply(detect_langs)\n",
    "    \n",
    "    # new column ['contains_your_language'] returns 'True' if ['language'] contains any probability of your language\n",
    "    df_copy['contains_your_language'] = df_copy['language'].apply(str).str.contains(language_select)\n",
    "    \n",
    "    # parse data to only return values where ['contains_your_language'] is True\n",
    "    df_copy = df_copy.loc[df_copy['contains_your_language'] == True]\n",
    "    \n",
    "    # remove ['language'] and ['contains_your_language'] as they are no longer needed\n",
    "    del df_copy['language']\n",
    "    del df_copy['contains_your_language']\n",
    "    \n",
    "    # reindex df\n",
    "    df_copy = df_copy.reset_index(drop = True)\n",
    "    \n",
    "   # return your new filtered dataframe\n",
    "    return df_copy\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```Python\n",
    "# run function\n",
    "data = language_filter(df = data, series = 'body', language_select = 'en')\n",
    "data.head()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## > method 4\n",
    "- #### parallel processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The parallel processing package doesn't have a version of pandas `.apply`, it has a<br>\n",
    "version of `map` (which is a list's version of apply). This means we need to transform <br>\n",
    "our data into a list and then use the map function. below are some examples:\n",
    "\n",
    "```Python\n",
    "# list comprehension = this is the function that is going to be parallel processed\n",
    "test_list = [i for i in map(detect_langs, data['body'])]\n",
    "\n",
    "# same thing as list comprehension above\n",
    "test_list = []\n",
    "for i in data['body']:\n",
    "    results = detect_langs(i)\n",
    "    test_list.append(results)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of parallel processes to create\n",
    "# should be number of cpu cores that your computer has\n",
    "num_processes = mp.cpu_count()\n",
    "num_processes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below, `pool.map` works just the same as `map`. Chunksize is the amount of data<br>\n",
    "that each process will work on at a time. To get the progress bar, simply wrap<br>\n",
    "`tqdm.tqdm` around the object that you are iterating over, which in this case<br>\n",
    "is `data['body']`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1607781/1607781 [33:00<00:00, 811.75it/s]\n"
     ]
    }
   ],
   "source": [
    "# parallel process with progress bar\n",
    "if __name__ == '__main__':\n",
    "    with mp.Pool(num_processes) as pool:\n",
    "       data['language'] = [i for i in pool.map(detect_langs, tqdm.tqdm(data['body']), chunksize = 10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>score</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>body</th>\n",
       "      <th>has_char</th>\n",
       "      <th>language</th>\n",
       "      <th>english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CryptoHODLer101</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1513292107</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No BTC is not fiat. I get paid in Bitcoin. As ...</td>\n",
       "      <td>True</td>\n",
       "      <td>[en:0.999994581371662]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nappiestapparatus</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1395194709</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I think you're overestimating your hashing pow...</td>\n",
       "      <td>True</td>\n",
       "      <td>[en:0.9999973707254454]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Skating2Death</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1390016247</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I've heard of it, but with the high volatility...</td>\n",
       "      <td>True</td>\n",
       "      <td>[en:0.999997406214342]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OmarJunkman</td>\n",
       "      <td>AMA</td>\n",
       "      <td>1528300155</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Do you have any bitcoin?</td>\n",
       "      <td>True</td>\n",
       "      <td>[en:0.5714282300865294, tr:0.4285707383592642]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>evanc1411</td>\n",
       "      <td>ASU</td>\n",
       "      <td>1525380807</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>BTC  -?  Damn, when you buy a Bitcoin you're ...</td>\n",
       "      <td>True</td>\n",
       "      <td>[en:0.8571382001868462, cy:0.1428609631258628]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              author subreddit  created_utc  score  controversiality  \\\n",
       "0    CryptoHODLer101       AMA   1513292107      0                 0   \n",
       "1  nappiestapparatus       AMA   1395194709      0                 0   \n",
       "2      Skating2Death       AMA   1390016247      0                 0   \n",
       "3        OmarJunkman       AMA   1528300155      0                 0   \n",
       "4          evanc1411       ASU   1525380807      0                 1   \n",
       "\n",
       "                                                body  has_char  \\\n",
       "0  No BTC is not fiat. I get paid in Bitcoin. As ...      True   \n",
       "1  I think you're overestimating your hashing pow...      True   \n",
       "2  I've heard of it, but with the high volatility...      True   \n",
       "3                           Do you have any bitcoin?      True   \n",
       "4   BTC  -?  Damn, when you buy a Bitcoin you're ...      True   \n",
       "\n",
       "                                         language  english  \n",
       "0                          [en:0.999994581371662]     True  \n",
       "1                         [en:0.9999973707254454]     True  \n",
       "2                          [en:0.999997406214342]     True  \n",
       "3  [en:0.5714282300865294, tr:0.4285707383592642]     True  \n",
       "4  [en:0.8571382001868462, cy:0.1428609631258628]     True  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# new column ['english'] returns 'True' if english has any probability of being the language\n",
    "data['english'] = data['language'].apply(str).str.contains('en')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     1603318\n",
       "False       4463\n",
       "Name: english, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see how many were classified as each\n",
    "data['english'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parse data to only return values where ['english'] is True\n",
    "data = data[(data['english'] == True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reset index as indexes of filtered data are deleted\n",
    "data = data.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop data that we don't need anymore\n",
    "del data['language']\n",
    "del data['english']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.write_csv('filtered_data_odd.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}