ngrams/Ngram_creator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.linear_model import RandomizedLogisticRegression, LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and process data\n",
    "\n",
    "`data/combined_new.csv` contains:\n",
    "  - company names (labelled `1`)\n",
    "  - phrases of length (1-4) constructed randomly from english scrabble words (labelled `0`).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                              word  label\n",
      "0  technology management solutions      1\n",
      "1             unsublimed bergstedt      0\n",
      "2                       sartor nyc      1\n",
      "3                     invulnerably      0\n",
      "4      harmonies spahn overteeming      0\n",
      " \n",
      "we have 36265 company examples out of 78303 total examples\n"
     ]
    }
   ],
   "source": [
    "# load into pandas dataframe and separate\n",
    "df = pd.read_csv('data/combined_new.csv', sep=',', names=['word','label'])\n",
    "total_examples = len(df)\n",
    "company_examples = len(df[df.label==1])\n",
    "print(df.head())\n",
    "print(' ')\n",
    "print('we have {} company examples out of {} total examples'.format(company_examples, total_examples))\n",
    "\n",
    "df_x=df['word'].values.astype('U')\n",
    "df_y=df['label'].values.astype('U')\n",
    "x_train, x_test, y_train, y_test = train_test_split(df_x, df_y, test_size=0.2, random_state=1110)\n",
    "y_train = [int(y) for y in y_train]\n",
    "y_test = [int(y) for y in y_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making character ngrams from the words\n",
    "\n",
    "Now, I create a vectorizer that takes words and counts their character-level ngrams with length 3-6 characters (inclusive).  \n",
    "\n",
    "With this vectorizer I transform my word inputs and get a lookup table for the corresponding ngram names. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define range of ngram length to look at\n",
    "ngram_range = (1,5)\n",
    "\n",
    "# transform the phrases into their ngram counts\n",
    "vectorizer = CountVectorizer(ngram_range=ngram_range, analyzer='char')\n",
    "\n",
    "X = vectorizer.fit_transform(df_x)\n",
    "X_train = vectorizer.transform(x_train)\n",
    "X_test  = vectorizer.transform(x_test)\n",
    "\n",
    "names = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning the most relevant features\n",
    "\n",
    "I use a randomized logistic regression to find the ngrams that are most relevant to classifying the elelemnts of my training set.\n",
    "\n",
    "To do this, I fit the character ngram inputs to the corresponding labels with a selection threshold for retaining each ngram in the model.\n",
    "\n",
    "Then, I can use the returned mask of important features to get the best ngrams."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedLogisticRegression(C=1, fit_intercept=True, memory=None, n_jobs=1,\n",
       "               n_resampling=200, normalize=True, pre_dispatch='3*n_jobs',\n",
       "               random_state=None, sample_fraction=0.75, scaling=0.5,\n",
       "               selection_threshold=0.5, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# run a logistic regression to return ngrams that are above 'selection threshold' of significant in the model\n",
    "selection_threshold = 0.5\n",
    "randomized_logistic = RandomizedLogisticRegression(selection_threshold=selection_threshold)\n",
    "randomized_logistic.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 265 ngrams:\n",
      "[' ', ' and ', ' ange', ' app', ' ass', ' by ', ' capi', ' com', ' comm', ' deve', ' engi', ' esta', ' for ', ' fund', ' i', ' in', ' lab', ' ll', ' medi', ' of', ' of ', ' secu', ' sys', ' syst', ' tech', 'a', 'abl', 'ably', 'ae', 'agazi', 'ah', 'al ', 'ali', 'ameri', 'an', 'ange', 'anti', 'app', 'apps', 'are', 'asia', 'at', 'au', 'bever', 'box', 'brand', 'care', 'cc', 'che', 'click', 'club', 'code', 'cot', 'creat', 'crui', 'cz', 'd', 'dail', 'dat', 'data', 'dc', 'desk', 'dv', 'e', 'e ', 'ealt', 'easy', 'ed', 'edg', 'edi', 'edia', 'edu', 'eet', 'ei', 'em', 'emm', 'er', 'ergy', 'es', 'esear', 'ete', 'ett', 'ey', 'feed', 'ff', 'film', 'films', 'find', 'fit', 'fle', 'fly', 'food', 'for ', 'fund', 'g', 'game', 'games', 'geniu', 'gg', 'gree', 'group', 'h', 'ham', 'hl', 'hnolo', 'holdi', 'home', 'hub', 'hy', 'i', 'ias', 'ical', 'ics', 'ie', 'ier', 'iest', 'ight', 'imite', 'ing', 'insi', 'inst', 'is', 'ism', 'ix', 'k', 'l', 'l ', 'lab', 'labs', 'lass', 'lb', 'learn', 'lert', 'ley', 'life', 'ligen', 'link', 'littl', 'lla', 'llc', 'llege', 'loca', 'local', 'loud', 'ly', 'm', 'man', 'mc', 'medi', 'mi', 'mind', 'mis', 'mob', 'mol', 'mpany', 'my', 'n', 'nagem', 'nd ', 'netwo', 'new ', 'nnect', 'nnov', 'non', 'note', 'nsult', 'ntern', 'nture', 'nves', 'o', 'oe', 'oid', 'ol', 'ollec', 'onlin', 'open', 'oph', 'os', 'over', 'pay', 'pel', 'ph', 'pharm', 'platf', 'power', 'pri', 'pub', 'quare', 'r', 'rban ', 'rer', 'rk', 'rket', 'rnati', 'rr', 's', 's ', 'sale', 'sch', 'searc', 'senge', 'shar', 'share', 'sho', 'shop', 'sig', 'simpl', 'skill', 'smart', 'snap', 'soci', 'socia', 'softw', 'solut', 'sport', 'spot', 'sta', 'start', 'stor', 'stud', 'style', 'sub', 'sulti', 'sys', 't ', 'talen', 'task', 'td', 'tec', 'the ', 'thin', 'to', 'ton', 'trade', 'trave', 'trea', 'trend', 'tt', 'tures', 'tz', 'u', 'udio', 'ug', 'ul', 'um', 'unive', 'ustr', 'v', 'vent', 'vices', 'vid', 'view', 'vis', 'visi', 'web', 'where', 'wood', 'work', 'works', 'wort', 'x', 'you', 'your', 'ytics', 'z']\n"
     ]
    }
   ],
   "source": [
    "# get the final ngrams\n",
    "mask = randomized_logistic.get_support()\n",
    "features = np.array(names)[mask]\n",
    "print('found {} ngrams:'.format(len([f for f in features])))\n",
    "print([f for f in features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy : 0.8133999553015548\n",
      "testing accuracy : 0.8072281463508078\n"
     ]
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "  ('feature_selection', RandomizedLogisticRegression(selection_threshold=selection_threshold)),\n",
    "  ('classification', LogisticRegression())\n",
    "])\n",
    "pipe.fit(X_train, y_train)\n",
    "print('training accuracy : {}'.format(pipe.score(X_train, y_train)))\n",
    "print('testing accuracy : {}'.format(pipe.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive features : 141\n",
      "[' ange', ' ass', ' by ', ' capi', ' com', ' comm', ' deve', ' engi', ' for ', ' i', ' in', ' lab', ' ll', ' medi', ' of', ' of ', ' secu', ' tech', 'agazi', 'al ', 'at', 'au', 'bever', 'care', 'club', 'code', 'creat', 'crui', 'cz', 'desk', 'e', 'e ', 'edia', 'ei', 'em', 'emm', 'er', 'ergy', 'esear', 'ete', 'ett', 'feed', 'film', 'films', 'find', 'fit', 'fle', 'fly', 'food', 'fund', 'g', 'game', 'games', 'geniu', 'ham', 'hnolo', 'hub', 'hy', 'i', 'ical', 'ics', 'ie', 'ing', 'is', 'ix', 'lab', 'labs', 'learn', 'lert', 'ley', 'life', 'ligen', 'link', 'littl', 'lla', 'llc', 'm', 'man', 'mi', 'mind', 'mol', 'my', 'n', 'nagem', 'nd ', 'new ', 'nnect', 'nture', 'nves', 'o', 'oe', 'oid', 'onlin', 'oph', 'os', 'pay', 'pel', 'ph', 'pharm', 'power', 'r', 'rban ', 'rer', 'rk', 'rket', 'rr', 'sale', 'sch', 'searc', 'senge', 'shar', 'share', 'sho', 'shop', 'sig', 'simpl', 'skill', 'smart', 'soci', 'softw', 'sport', 'sta', 'start', 'stor', 'stud', 'style', 'sulti', 'sys', 't ', 'task', 'the ', 'ton', 'trade', 'trave', 'trea', 'trend', 'tt', 'tures', 'tz', 'ul', 'um']\n",
      "\n",
      "negative features : 24\n",
      "['abl', 'ah', 'are', 'asia', 'dv', 'easy', 'es', 'gg', 'group', 'h', 'holdi', 'home', 'imite', 'lb', 'loud', 'netwo', 'non', 'note', 'ol', 'socia', 'solut', 'td', 'u', 'udio']\n"
     ]
    }
   ],
   "source": [
    "# sort into positive and negative ngrams over some threshold\n",
    "\n",
    "# minimum absolute value of the coefficient needed to save the ngram\n",
    "threshold = 1\n",
    "\n",
    "params = pipe.get_params()\n",
    "logistic = params['classification']\n",
    "coeffs = logistic.coef_[0]\n",
    "coef_dict = {f:c for f,c in zip(features, coeffs)}\n",
    "\n",
    "positive_features = [f for f, c in coef_dict.items() if abs(c) > threshold and c > 0]\n",
    "negative_features = [f for f, c in coef_dict.items() if abs(c) > threshold and c < 0]\n",
    "\n",
    "print('positive features : {}\\n{}\\n'.format(len(positive_features), positive_features))\n",
    "print('negative features : {}\\n{}'.format(len(negative_features), negative_features))\n",
    "with open('./data/pos_ngrams.txt', 'w') as f:\n",
    "    f.write(','.join([str(feat) for feat in positive_features]))\n",
    "with open('./data/neg_ngrams.txt', 'w') as f:\n",
    "    f.write(','.join([str(feat) for feat in negative_features]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}