vw_ner_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Named Entity Recognition (NER) - ISSUES/PROBLEMS\n",
    "\n",
    "Building 3 NER classifiers to detect locations, organisations, and people respectively from short sentences using Vowpal Wabbit's Python API (pyvw).\n",
    "\n",
    "Results -\n",
    " - NER classifiers do not appear to generalise and learn sentence structure well. \n",
    " - Overfit to the training data, \n",
    " - Don't do well on examples with same sentence structure but different named entities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pyvw\n",
    "import joblib\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vowpal Wabbit class instantation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SequenceLabeler(pyvw.SearchTask):\n",
    "    def __init__(self, vw, sch, num_actions):\n",
    "        # you must must must initialize the parent class\n",
    "        # this will automatically store self.sch <- sch, self.vw <- vw\n",
    "        pyvw.SearchTask.__init__(self, vw, sch, num_actions)\n",
    "        \n",
    "        # set whatever options you want\n",
    "        sch.set_options( sch.AUTO_HAMMING_LOSS | sch.AUTO_CONDITION_FEATURES )\n",
    "\n",
    "    def _run(self, sentence):   # it's called _run to remind you that you shouldn't call it directly!\n",
    "        output = []\n",
    "        for n in range(len(sentence)):\n",
    "            pos,word = sentence[n]\n",
    "            # use \"with...as...\" to guarantee that the example is finished properly\n",
    "            with self.vw.example({'w': [word]}) as ex:\n",
    "                pred = self.sch.predict(examples=ex, my_tag=n+1, oracle=pos, condition=[(n,'p'), (n-1, 'q')])\n",
    "                output.append(pred)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utility Functions\n",
    "Functions to train models, extract entities, and evaluate test sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_model(training_set, num_labels=3):\n",
    "    '''\n",
    "    Train NER models for each category (location, organisation, and person). \n",
    "    The number of labels is 3 using the BIO encoding scheme\n",
    "    '''\n",
    "    vw = pyvw.vw(search=num_labels, search_task='hook', ring_size=1024)\n",
    "    sequenceLabeler = vw.init_search_task(SequenceLabeler)    \n",
    "    sequenceLabeler.learn(training_set)\n",
    "    \n",
    "    return sequenceLabeler\n",
    "\n",
    "def extract_entities(sentence, label_list):\n",
    "    '''\n",
    "    Takes in a sentence and a sequence of labels (from the Vowpal Wabbit NER classifier) as input.\n",
    "    Identifies subsequences in sentence where the classifier believes named entities are present. \n",
    "    It returns identified entities as a list of strings.\n",
    "    '''\n",
    "    entities = []\n",
    "    for pos, label in enumerate(label_list):\n",
    "        if label == 1:\n",
    "            pos2 = pos+1\n",
    "            current_entity = sentence[pos]\n",
    "            while pos2 < len(label_list) and label_list[pos2] == 2:\n",
    "                current_entity = current_entity + ' ' + sentence[pos2]\n",
    "                pos2 += 1\n",
    "            entities.append(current_entity)\n",
    "    return entities\n",
    "\n",
    "def eval_sentences(sentences):\n",
    "    '''\n",
    "    Takes in a list of sentences.\n",
    "    Runs each model's predict() method and extracts predicted entities from the sentence.\n",
    "    Returns these identified entities\n",
    "    '''\n",
    "    models = {'locations': loc_model, 'organisations': org_model, 'people': per_model}\n",
    "    for sentence in sentences:\n",
    "        print sentence\n",
    "        for key, model in models.iteritems():\n",
    "            split_sentence = sentence.split()\n",
    "            vw_sentence = [(0, word) for word in split_sentence]\n",
    "            predicted_labels = model.predict(vw_sentence)\n",
    "            predicted_entities = extract_entities(sentence=split_sentence, label_list=predicted_labels)\n",
    "            print key, '-', predicted_entities\n",
    "        print ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data and train models\n",
    "Each dataset contains only one type of entity - one of locations, persons, or organisations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Location data and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loc_train = joblib.load('conll_atis_merged_loc_train.pkl') #locations from CONLL 2003 and ATIS\n",
    "loc_model = train_model(loc_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(1, 'new'),\n",
      "  (2, 'york'),\n",
      "  (3, 'to'),\n",
      "  (1, 'las'),\n",
      "  (2, 'vegas'),\n",
      "  (3, 'on'),\n",
      "  (3, 'sunday'),\n",
      "  (3, 'afternoon')],\n",
      " [(1, 'new'),\n",
      "  (2, 'york'),\n",
      "  (3, 'to'),\n",
      "  (1, 'las'),\n",
      "  (2, 'vegas'),\n",
      "  (3, 'on'),\n",
      "  (3, 'sunday'),\n",
      "  (3, 'afternoon')],\n",
      " [(1, 'st'), (2, 'helens'), (3, ','), (1, 'england'), (3, '1996-08-26')],\n",
      " [(3, 'what'),\n",
      "  (3, 'flights'),\n",
      "  (3, 'from'),\n",
      "  (1, 'denver'),\n",
      "  (3, 'to'),\n",
      "  (1, 'salt'),\n",
      "  (2, 'lake'),\n",
      "  (2, 'city')]]\n"
     ]
    }
   ],
   "source": [
    "pprint([sentence for sentence in loc_train for label, word in sentence if label == 2 and len(sentence) <= 8][8:12])\n",
    "# {'B-LOC': 1, 'I-LOC': 2, 'Other': 3}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Organisation data and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "org_train = joblib.load('conll_org_train.pkl') #Organisations from CONLL 2003\n",
    "org_model = train_model(org_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(1, 'chernomorets'),\n",
      "  (2, 'novorossiisk'),\n",
      "  (3, '2'),\n",
      "  (1, 'rostselmash'),\n",
      "  (2, 'rostov'),\n",
      "  (3, '1')],\n",
      " [(3, '--'),\n",
      "  (1, 'sydney'),\n",
      "  (2, 'newsroom'),\n",
      "  (3, '61-2'),\n",
      "  (3, '9373'),\n",
      "  (3, '1800')],\n",
      " [(1, 'financial'), (2, 'kathimerini')],\n",
      " [(1, 'nec'),\n",
      "  (2, 'nijmegen'),\n",
      "  (3, '1'),\n",
      "  (1, 'psv'),\n",
      "  (2, 'eindhoven'),\n",
      "  (3, '4')]]\n"
     ]
    }
   ],
   "source": [
    "pprint([sentence for sentence in org_train for label, word in sentence if label == 2 and len(sentence) <= 6][12:16])\n",
    "# {'B-ORG': 1, 'I-ORG': 2, 'Other': 3}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## People data and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "per_train = joblib.load('conll_per_train.pkl') #Persons from CONLL 2003\n",
    "per_model = train_model(per_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(3, '1.'),\n",
      "  (1, 'tom'),\n",
      "  (2, 'pukstys'),\n",
      "  (3, '('),\n",
      "  (3, 'u.s.'),\n",
      "  (3, ')'),\n",
      "  (3, '86.82')],\n",
      " [(3, 'shelbourne'),\n",
      "  (3, '-'),\n",
      "  (1, 'mark'),\n",
      "  (2, 'rutherford'),\n",
      "  (3, '('),\n",
      "  (3, '5th'),\n",
      "  (3, ')')],\n",
      " [(1, 'fred'),\n",
      "  (2, 'trueman'),\n",
      "  (3, '('),\n",
      "  (3, 'england'),\n",
      "  (3, ')'),\n",
      "  (3, '307'),\n",
      "  (3, ','),\n",
      "  (3, '67')],\n",
      " [(3, '5.'),\n",
      "  (1, 'jamie'),\n",
      "  (2, 'baulch'),\n",
      "  (3, '('),\n",
      "  (3, 'britain'),\n",
      "  (3, ')'),\n",
      "  (3, '45.08')]]\n"
     ]
    }
   ],
   "source": [
    "pprint([sentence for sentence in per_train for label, word in sentence if label == 2 and len(sentence) <= 8][10:14])\n",
    "# {'B-PER': 1, 'I-PER': 2, 'Other': 3}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate models on test sentences\n",
    "\n",
    " - We test the NER models on sentences that are based on training data. \n",
    " - In each set of sentences, we modify only the named entities while keeping the sentence structure the same. \n",
    " - Results indicate that the model is unable to generalise to new locations, people, and organisations with ease."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Locations\n",
    "In the examples below, only the location names change, but the `location` NER classifier is unable to recognise them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new york to las vegas on sunday afternoon\n",
      "organisations - []\n",
      "locations - ['new york', 'las vegas']\n",
      "people - []\n",
      "\n",
      "chennai to mumbai on sunday afternoon\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n",
      "lima to ascuncion on a sunday afternoon\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#the first sentence comes directly from the training data, the other two use the same structure with different locations\n",
    "sample_sentences = ['new york to las vegas on sunday afternoon', #flight tickets\n",
    "                    'chennai to mumbai on sunday afternoon',\n",
    "                    'lima to ascuncion on a sunday afternoon']\n",
    "eval_sentences(sample_sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Organisations\n",
    "In the examples below, only the organisation names change, but the `organisation` NER classifier is unable to recognise them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neuchatel 3 st gallen 0\n",
      "organisations - ['neuchatel', 'st gallen']\n",
      "locations - []\n",
      "people - []\n",
      "\n",
      "universidad 3 river plate 0\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n",
      "osasuna 3 real zaragoza 0\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#the first sentence comes directly from the training data, the other two use the same structure with different orgs\n",
    "sample_sentences2 = ['neuchatel 3 st gallen 0', #football clubs\n",
    "                     'universidad 3 river plate 0',\n",
    "                     'osasuna 3 real zaragoza 0']\n",
    "eval_sentences(sample_sentences2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## People\n",
    "In the examples below, the person names change, but the `person` NER classifier is unable to recognise them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bill gates is the ceo\n",
      "organisations - []\n",
      "locations - []\n",
      "people - ['bill gates']\n",
      "\n",
      "ma huateng is the ceo\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n",
      "narayana moorthy is the ceo\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n",
      "elon musk is the ceo\n",
      "organisations - []\n",
      "locations - []\n",
      "people - []\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#the first sentence is based on training data, the other two use the same structure with different persons\n",
    "sample_sentences3 = ['bill gates is the ceo', \n",
    "                     'ma huateng is the ceo', #CEO of Tencent\n",
    "                     'narayana moorthy is the ceo', #CEO of Infosys\n",
    "                     'elon musk is the ceo'] #CEO of Tesla\n",
    "eval_sentences(sample_sentences3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}