Jobzilla_SpaCy_KWs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "automotive-slide",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install spacy-model-en_core_web_lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "rubber-coverage",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import textract\n",
    "from spacy.pipeline import EntityRuler\n",
    "from spacy import displacy\n",
    "import jsonlines\n",
    "from spacy.lang.en import English\n",
    "from spacy.tokens import Doc\n",
    "import spacy\n",
    "\n",
    "\n",
    "nlp = spacy.load('en')\n",
    "skill_pattern_path = \"jz_skill_patterns.jsonl\"\n",
    "with jsonlines.open(\"jz_skill_patterns.jsonl\") as f:\n",
    "    created_entities = [line['label'].upper() for line in f.iter()]\n",
    "\n",
    "ruler = EntityRuler(nlp).from_disk(skill_pattern_path)\n",
    "nlp.add_pipe(ruler, after='parser')\n",
    "\n",
    "def get_skills(text):\n",
    "    doc = nlp(text)\n",
    "    myset = []\n",
    "    subset = []\n",
    "    for ent in doc.ents:\n",
    "        if ent.label_==\"SKILL\":\n",
    "             subset.append(ent.text)\n",
    "    myset.append(subset)\n",
    "    return subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "ancient-brass",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['skills'] = df['body'].apply(get_skills)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "lesbian-lending",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "      <th>skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Application Engineer</td>\n",
       "      <td>Dear Mr. Thompson:\\nI am interested in being c...</td>\n",
       "      <td>[engineering, software, support, design, suppo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Business Development Manager</td>\n",
       "      <td>Dear Ms. Lordes:\\nUpon review of your posting ...</td>\n",
       "      <td>[Business, business, business, business, busin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Physical Therapist Assistant</td>\n",
       "      <td>Dear Ms. McCabe:\\nWhen I learned of your need ...</td>\n",
       "      <td>[collaboration, Monitoring]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Child And Youth Worker</td>\n",
       "      <td>Dear Mr. Perez:\\nAs a skilled and enthusiastic...</td>\n",
       "      <td>[collaboration]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Teacher Aide</td>\n",
       "      <td>Dear Ms. Bernadette Craig:\\nIt is my objective...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          title  \\\n",
       "0          Application Engineer   \n",
       "1  Business Development Manager   \n",
       "2  Physical Therapist Assistant   \n",
       "3        Child And Youth Worker   \n",
       "4                  Teacher Aide   \n",
       "\n",
       "                                                body  \\\n",
       "0  Dear Mr. Thompson:\\nI am interested in being c...   \n",
       "1  Dear Ms. Lordes:\\nUpon review of your posting ...   \n",
       "2  Dear Ms. McCabe:\\nWhen I learned of your need ...   \n",
       "3  Dear Mr. Perez:\\nAs a skilled and enthusiastic...   \n",
       "4  Dear Ms. Bernadette Craig:\\nIt is my objective...   \n",
       "\n",
       "                                              skills  \n",
       "0  [engineering, software, support, design, suppo...  \n",
       "1  [Business, business, business, business, busin...  \n",
       "2                        [collaboration, Monitoring]  \n",
       "3                                    [collaboration]  \n",
       "4                                                 []  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}