Job_title_clean_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named 'fuzzywuzzy'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-4a30ef58a4eb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mfuzzywuzzy\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfuzz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfuzzywuzzy\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprocess\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0menchant\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named 'fuzzywuzzy'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from fuzzywuzzy import fuzz\n",
    "from fuzzywuzzy import process\n",
    "import enchant \n",
    "import nltk\n",
    "from nltk.corpus import words\n",
    "import re\n",
    "import html\n",
    "import re\n",
    "dict_en = enchant.Dict(\"en_US\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# job_titles = pd.read_csv(\"job_title_need_clean_2.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Lower case and remove unicode job title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# job_titles['lower_case'] = ''\n",
    "# job_titles['remove_unicode'] = ''\n",
    "# for index, row in job_titles.iterrows():\n",
    "#     job_title_lower_case = str.lower(row['c_employment_title'])\n",
    "#     job_title_remove_unicode = html.unescape(job_title_lower_case)\n",
    "    \n",
    "#     job_titles.set_value(index, 'lower_case', job_title_lower_case)\n",
    "#     job_titles.set_value(index, 'remove_unicode', job_title_remove_unicode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# job_titles.to_csv(\"job_title_lower_case_and_remove_unicode.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Fixed spelling case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>c_employment_title</th>\n",
       "      <th>lower_case</th>\n",
       "      <th>remove_unicode</th>\n",
       "      <th>fix_spelling</th>\n",
       "      <th>fix_spelling_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Web Developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Web Developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Web Developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Web Master</td>\n",
       "      <td>web master</td>\n",
       "      <td>web master</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Web Developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td>web developer</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  c_employment_title     lower_case remove_unicode fix_spelling  \\\n",
       "0      Web Developer  web developer  web developer                \n",
       "1      Web Developer  web developer  web developer                \n",
       "2      Web Developer  web developer  web developer                \n",
       "3         Web Master     web master     web master                \n",
       "4      Web Developer  web developer  web developer                \n",
       "\n",
       "   fix_spelling_score  \n",
       "0                   0  \n",
       "1                   0  \n",
       "2                   0  \n",
       "3                   0  \n",
       "4                   0  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_titles_step_3 = pd.read_csv(\"job_title_lower_case_and_remove_unicode.csv\")\n",
    "job_titles_step_3['fix_spelling'] = ''\n",
    "job_titles_step_3['fix_spelling_score'] = 0\n",
    "job_titles_step_3 = job_titles_step_3.drop('Unnamed: 0', 1)\n",
    "words_not_in_en = pd.read_csv(\"words_not_in_en.csv\")\n",
    "words_not_in_en = words_not_in_en.drop('Unnamed: 0', 1)\n",
    "job_titles_step_3.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Check the title with all words are correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def remove_special_character(word):\n",
    "    result = ''.join(e for e in word if e.isalnum() or e == ' ')\n",
    "    result = re.sub('[!@#$%^&*()]', '', result)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / 199665\n",
      "10000 / 199665\n",
      "20000 / 199665\n",
      "30000 / 199665\n",
      "40000 / 199665\n",
      "50000 / 199665\n",
      "60000 / 199665\n",
      "70000 / 199665\n",
      "80000 / 199665\n",
      "90000 / 199665\n",
      "100000 / 199665\n",
      "110000 / 199665\n",
      "120000 / 199665\n",
      "130000 / 199665\n",
      "140000 / 199665\n",
      "150000 / 199665\n",
      "160000 / 199665\n",
      "170000 / 199665\n",
      "180000 / 199665\n",
      "190000 / 199665\n"
     ]
    }
   ],
   "source": [
    "total = len(job_titles_step_3)\n",
    "for index, row in job_titles_step_3.iterrows():\n",
    "        \n",
    "    if index % 10000 == 0:\n",
    "        print(index, '/', total)\n",
    "    \n",
    "    job_title = remove_special_character(row['remove_unicode'])\n",
    "    \n",
    "    is_correct = True\n",
    "    for word in job_title.split():\n",
    "        if not dict_en.check(word):\n",
    "            is_correct = False\n",
    "            break\n",
    "            \n",
    "    for word in remove_special_character(row['c_employment_title']).split():\n",
    "        if word.isupper() == True:\n",
    "            is_correct = False\n",
    "            # Run only 1 time for create new word in dictionary\n",
    "#             word = str.lower(word)\n",
    "#             new_words = words_not_in_en[words_not_in_en.word == word].head(1)\n",
    "            \n",
    "#             if len(new_words) == 0:\n",
    "#                 suggest_word = str.lower(dict_en.suggest(word)[0])\n",
    "#                 score = fuzz.ratio(word, suggest_word)\n",
    "#                 words_not_in_en.loc[len(words_not_in_en)] = [word, 1, suggest_word, score]\n",
    "#             else:\n",
    "#                 frequency = new_words.get_value(new_words.index[0], 'frequency') + 1 \n",
    "#                 words_not_in_en.set_value(new_words.index[0], 'frequency', frequency)\n",
    "            break\n",
    "            \n",
    "            \n",
    "    if is_correct:\n",
    "        # All words in job_title are correctly spelling\n",
    "        job_titles_step_3.set_value(index, 'fix_spelling', job_title)\n",
    "        job_titles_step_3.set_value(index, 'fix_spelling_score', 1000) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / 199665\n",
      "10000 / 199665\n",
      "20000 / 199665\n",
      "30000 / 199665\n",
      "40000 / 199665\n",
      "50000 / 199665\n",
      "60000 / 199665\n",
      "70000 / 199665\n",
      "80000 / 199665\n",
      "90000 / 199665\n",
      "100000 / 199665\n",
      "110000 / 199665\n",
      "120000 / 199665\n",
      "130000 / 199665\n",
      "140000 / 199665\n",
      "150000 / 199665\n",
      "160000 / 199665\n",
      "170000 / 199665\n",
      "180000 / 199665\n",
      "190000 / 199665\n"
     ]
    }
   ],
   "source": [
    "total = len(job_titles_step_3)\n",
    "for index, row in job_titles_step_3.iterrows():\n",
    "    if index % 10000 == 0:\n",
    "        print(index, '/', total)\n",
    "\n",
    "        \n",
    "    suggest_title = row['remove_unicode']\n",
    "    original_title = remove_special_character(row['remove_unicode'])\n",
    "    \n",
    "    # Remove all word between \"(\" and \")\"\n",
    "    open_bracket = suggest_title.find(\"(\")\n",
    "    close_bracket = suggest_title.find(\")\") \n",
    "    if open_bracket != -1 and close_bracket != -1:\n",
    "        suggest_title = suggest_title[:open_bracket] + suggest_title[(close_bracket + 1):]\n",
    "    \n",
    "    # Remove all word after \"/\"\n",
    "    below_charactor = suggest_title.find(\"/\")\n",
    "    if below_charactor != -1:\n",
    "        suggest_title = suggest_title[:below_charactor]\n",
    "        \n",
    "    # Remove dot \".\" charactor\n",
    "    suggest_title = suggest_title.replace(\".\", \"\")\n",
    "    \n",
    "\n",
    "    for word in original_title.split():\n",
    "        wrong_words = words_not_in_en[words_not_in_en.word == word].head(1)\n",
    "        if (len(wrong_words) == 1):\n",
    "\n",
    "            suggest_word = wrong_words.get_value(wrong_words.index[0], 'suggest_word')\n",
    "            wrong_word = wrong_words.get_value(wrong_words.index[0], 'word')\n",
    "            score = wrong_words.get_value(wrong_words.index[0], 'score')\n",
    "\n",
    "\n",
    "            if score >= 95:\n",
    "                suggest_title = suggest_title.replace(wrong_word, suggest_word)\n",
    "        \n",
    "        score = fuzz.ratio(original_title,suggest_title)\n",
    "        job_titles_step_3.set_value(index, 'fix_spelling', suggest_title)\n",
    "        job_titles_step_3.set_value(index, 'fix_spelling_score', score) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "job_titles_step_3.to_csv(\"job_titles_step_3.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Create evalution file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "test_row_indexs = np.random.randint(0, high=200000, size= 200)\n",
    "evaluation = ['', '', ''] * 200\n",
    "i = 0\n",
    "for index, row in job_titles_step_3.iterrows():\n",
    "    if index in test_row_indexs:\n",
    "        row_i = [row['c_employment_title'],row['fix_spelling'],row['fix_spelling_score']]\n",
    "        evaluation[i] = row_i\n",
    "        i = i + 1\n",
    "\n",
    "evaluation_file = open('evaluation_2.csv', 'w')\n",
    "for item in evaluation:\n",
    "  evaluation_file.write(\"%s\\n\" % item)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}