tagger.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensorflow based morphological tagger experiment for Latvian\n",
    "Repository at https://github.com/PeterisP/tf-morphotagger  \n",
    "Appendix to the paper \"Deep neural learning approaches for Latvian morphological tagging\" at Baltic HLT 2016 (http://hlt2016.tilde.eu/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Required libraries\n",
    "import json, os, pickle, datetime, collections, itertools\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from gensim.models import Word2Vec\n",
    "from tensorflow.contrib.learn.python.learn.preprocessing import CategoricalVocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Experiment parametrization\n",
    "default_epochs = 20\n",
    "development = False\n",
    "remove_rare_words = True\n",
    "max_ngrams = 4\n",
    "input_features = {\n",
    "    'wordform_onehot' : True,\n",
    "    'wordform_embeddings' : True,\n",
    "    'analyzer_nhot' : True,\n",
    "    'ngrams' : True,\n",
    "    'wordshape' : True    \n",
    "}\n",
    "output_features = {\n",
    "    'pos_onehot' : True,\n",
    "    'tag_onehot' : True,\n",
    "    'attribute_nhot' : True\n",
    "}\n",
    "\n",
    "if development:\n",
    "    train_data_filename = 'data/train.json'\n",
    "    eval_data_filename = 'data/dev.json'\n",
    "else:\n",
    "    train_data_filename = 'data/train_dev.json'  # When training the final system, add the development set to the training set\n",
    "    eval_data_filename = 'data/test.json'\n",
    "embeddings_filename = 'embeddings/lvnews2.c0p0d0.shuf.txt.we216-200-ssg-w5-m10.20160516233643.bin'\n",
    "# This embeddings file can be downloaded from https://dl.dropboxusercontent.com/u/9455117/lvnews2.c0p0d0.shuf.txt.we216-200-ssg-w5-m10.20160516233643.bin\n",
    "\n",
    "wordform_key = 'wordform'\n",
    "wordform_original_key = 'wordform_original'\n",
    "tag_key = 'gold_tag_simple'\n",
    "attribute_key = 'gold_attributes'\n",
    "pos_key = 'pos'\n",
    "options_key = 'options'\n",
    "unk = '_UNK_'\n",
    "output_dir = 'output'\n",
    "os.makedirs(output_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Vocabulary management - mapping between words and numeric identifiers/positions in vectors\n",
    "# Currently implements 5 different vocabularies and can populate them from a document (training data)\n",
    "class Vocabularies(object):\n",
    "    def __init__(self, document=None, folder=None):\n",
    "        if folder:\n",
    "            with open(folder + '/voc_wordforms.p', 'rb') as f:\n",
    "                self.voc_wordforms = pickle.load(f)\n",
    "            with open(folder + '/voc_tags.p', 'rb') as f:\n",
    "                self.voc_tags = pickle.load(f)\n",
    "            with open(folder + '/voc_pos.p', 'rb') as f:\n",
    "                self.voc_pos = pickle.load(f)\n",
    "            with open(folder + '/voc_ngrams.p', 'rb') as f:\n",
    "                self.voc_ngrams = pickle.load(f)\n",
    "            with open(folder + '/voc_attributes.p', 'rb') as f:\n",
    "                self.voc_attributes = pickle.load(f)\n",
    "        else:\n",
    "            self.voc_wordforms = CategoricalVocabulary(unk)\n",
    "            self.voc_tags = CategoricalVocabulary(unk)\n",
    "            self.voc_pos = CategoricalVocabulary(unk)\n",
    "            self.voc_ngrams = CategoricalVocabulary(unk)\n",
    "            self.voc_attributes = CategoricalVocabulary(unk)\n",
    "\n",
    "        if document:\n",
    "            self.add_document(document)\n",
    "        \n",
    "    def dump(self, folder):\n",
    "        with open(folder + '/voc_wordforms.p', 'wb') as f:\n",
    "            pickle.dump(self.voc_wordforms, f)\n",
    "        with open(folder + '/voc_tags.p', 'wb') as f:\n",
    "            pickle.dump(self.voc_tags, f)\n",
    "        with open(folder + '/voc_pos.p', 'wb') as f:\n",
    "            pickle.dump(self.voc_pos, f)\n",
    "        with open(folder + '/voc_ngrams.p', 'wb') as f:\n",
    "            pickle.dump(self.voc_ngrams, f)\n",
    "        with open(folder + '/voc_attributes.p', 'wb') as f:\n",
    "            pickle.dump(self.voc_attributes, f)\n",
    "            \n",
    "    def add_document(self, document):\n",
    "        self.voc_wordforms.freeze(False)\n",
    "        self.voc_tags.freeze(False)\n",
    "        for sentence in document.sentences:\n",
    "            for token in sentence:\n",
    "                wordform = token[wordform_key]\n",
    "                self.voc_wordforms.add(wordform)\n",
    "                self.voc_tags.add(token[tag_key])\n",
    "                self.voc_pos.add(token[pos_key])\n",
    "                for option in token[options_key]:\n",
    "                    self.voc_tags.add(option) \n",
    "                    # We assume that analyzer options have the same set of possible values as tags\n",
    "                    # If that's not the case, modifications will be required\n",
    "                for i in range(1, max_ngrams+1):\n",
    "                    self.voc_ngrams.add(wordform[-i:])\n",
    "                for key, value in token[attribute_key].items():\n",
    "                    self.voc_attributes.add('{}:{}'.format(key, value))\n",
    "        if remove_rare_words:\n",
    "            self.voc_wordforms.trim(2) # Remove rare wordforms - treat once-seen words as OOV\n",
    "        self.voc_wordforms.freeze()\n",
    "        self.voc_tags.freeze()\n",
    "        self.voc_pos.freeze()\n",
    "        self.voc_ngrams.trim(2) # Ignore suffix ngrams that appear only once\n",
    "        self.voc_ngrams.freeze()\n",
    "        self.voc_attributes.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# A helper class for accuracy evaluations\n",
    "class AccuracyCounter():\n",
    "    def __init__(self):\n",
    "        self.c = collections.Counter()\n",
    "    def add(self, gold, silver):\n",
    "        self.c[gold == silver] += 1\n",
    "    def add_b(self, boolean):\n",
    "        self.c[boolean] += 1\n",
    "    def average(self):\n",
    "        total = sum(self.c.values()) \n",
    "        if not total:\n",
    "            return 0\n",
    "        return self.c[True] / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# A wrapper for JSON document used for input of training and test data. Includes also the evaluation measurement functionality\n",
    "# The data needs to be sentence-split, tokenized and (for best results) with data from the lexicon/paradigm based morphological analyzer\n",
    "# File format is generated by https://github.com/PeterisP/LVTagger/blob/master/src/main/java/lv/lumii/morphotagger/MorphoConverter.java\n",
    "class Document(object):\n",
    "    # self.sentences - list of sentences, each sentence is a list of tokens, each token is a dict of fields with key names defined in the configuration cell above.\n",
    "    def __init__(self, filename, limit=None):\n",
    "        with open(filename, 'r') as f:\n",
    "            self.sentences = json.load(f)\n",
    "        if limit and limit < len(self.sentences):\n",
    "            self.sentences = self.sentences[:limit]\n",
    "        self._preprocess()\n",
    "\n",
    "    def _simplify_wordform(word):\n",
    "        return word.lower()\n",
    "    \n",
    "    def _preprocess(self):\n",
    "        for sentence in self.sentences:\n",
    "            for token in sentence:\n",
    "                if not token.get(tag_key) or not token.get(wordform_key):\n",
    "                    print(json.dumps(token))    \n",
    "                    assert False\n",
    "                token[wordform_original_key] = token[wordform_key]\n",
    "                token[wordform_key] = Document._simplify_wordform(token[wordform_key])                    \n",
    "                token[pos_key] = token[tag_key][0]\n",
    "\n",
    "    # Outputs the tagged results and also calculates the accuracy metrics\n",
    "    def output_tagged(self, silver_tags, silver_poses, silver_attributes, filename, evaluate = True, vocabularies = None):\n",
    "        tag = AccuracyCounter()\n",
    "        oov_tag = AccuracyCounter()\n",
    "        tag_pos = AccuracyCounter()\n",
    "        oov_tag_pos = AccuracyCounter()\n",
    "        direct_pos = AccuracyCounter()\n",
    "        oov_direct_pos = AccuracyCounter()\n",
    "        attributes = AccuracyCounter()\n",
    "        oov_attributes = AccuracyCounter()\n",
    "        per_attribute = collections.defaultdict(AccuracyCounter)\n",
    "        attribute_errors = collections.Counter()\n",
    "        if not silver_tags:\n",
    "            silver_tags = []\n",
    "        if not silver_poses:\n",
    "            silver_poses = []\n",
    "        if not silver_attributes:\n",
    "            silver_attributes = []\n",
    "            \n",
    "        with open(filename, 'w') as f:\n",
    "            for sentence, sentence_tags, sentence_poses, sentence_attributes in itertools.zip_longest(self.sentences, silver_tags, silver_poses, silver_attributes):\n",
    "                if not sentence_tags:\n",
    "                    sentence_tags = []\n",
    "                if not sentence_poses:\n",
    "                    sentence_poses = []\n",
    "                if not sentence_attributes:\n",
    "                    sentence_attributes = []\n",
    "                for token, silver_tag, silver_pos, silver_token_attributes in itertools.zip_longest(sentence, sentence_tags, sentence_poses, sentence_attributes):\n",
    "                    gold_tag = token.get(tag_key)\n",
    "                    silver_tag_pos = silver_tag[0] if silver_tag else None\n",
    "                    tag.add(gold_tag, silver_tag)\n",
    "                    tag_pos.add(gold_tag[0], silver_tag_pos)\n",
    "                    direct_pos.add(gold_tag[0], silver_pos)\n",
    "                    if vocabularies and not vocabularies.voc_wordforms.get(token.get(wordform_key)):\n",
    "                        oov_tag.add(gold_tag, silver_tag)\n",
    "                        oov_tag_pos.add(gold_tag[0], silver_tag_pos)\n",
    "                        oov_direct_pos.add(gold_tag[0], silver_pos)\n",
    "                    \n",
    "                    gold_attrs = ','.join('{}:{}'.format(key, value) for key, value in token.get(attribute_key).items())\n",
    "                    silver_attrs = ''                    \n",
    "                    # Check the accuracy of predicted attributes\n",
    "                    if silver_token_attributes:\n",
    "                        errors = []                        \n",
    "                        for key, gold_value in token.get(attribute_key).items():\n",
    "                            if key in ['Skaitlis 2', 'Locījums 2', 'Rekcija']: # Lexical properties that shouldn't be tagged\n",
    "                                continue\n",
    "                            silver_value = ''\n",
    "                            best_confidence = 0\n",
    "                            for silver_attribute, confidence in silver_token_attributes.items():\n",
    "                                if silver_attribute.split(':')[0] != key:\n",
    "                                    continue  # NB! we simply ignore the tagger's opinion on any attributes that are not relevant for this POS\n",
    "                                if confidence > best_confidence:\n",
    "                                    best_confidence = confidence\n",
    "                                    silver_value = silver_attribute.split(':', maxsplit=1)[1]\n",
    "#                             print(\"{}: gold -'{}', silver -'{}' @ {}\".format(key, gold_value, silver_value, best_confidence))\n",
    "                            per_attribute[key].add(gold_value, silver_value)\n",
    "                            if gold_value != silver_value:\n",
    "                                errors.append('{}:{} nevis {}'.format(key, silver_value, gold_value))\n",
    "                                attribute_errors['{}:{} nevis {}'.format(key, silver_value, gold_value)] += 1\n",
    "                                \n",
    "                        attributes.add_b(not errors)\n",
    "                        if vocabularies and not vocabularies.voc_wordforms.get(token.get(wordform_key)):\n",
    "                            oov_attributes.add_b(not errors)     \n",
    "                        silver_attrs = '\\t'.join(errors)\n",
    "                                            \n",
    "                    if not silver_tag:\n",
    "                        silver_tag = silver_pos\n",
    "                    if not silver_tag:\n",
    "                        silver_tag = ''\n",
    "                    f.write('\\t'.join([token.get(wordform_original_key), gold_tag, silver_tag, gold_attrs, silver_attrs]) + '\\n')\n",
    "        print('Test set tag accuracy:        {:.2%} ({:.2%})'.format(tag.average(), oov_tag.average()))\n",
    "        print('Attribute accuracy:           {:.2%} ({:.2%})'.format(attributes.average(), oov_attributes.average()))\n",
    "        print('Test set tag POS accuracy:    {:.2%} ({:.2%})'.format(tag_pos.average(), oov_tag_pos.average()))\n",
    "        print('Test set direct POS accuracy: {:.2%} ({:.2%})'.format(direct_pos.average(), oov_direct_pos.average()))\n",
    "        for key, counter in per_attribute.items():\n",
    "            print('    {}: {:.2%})'.format(key, counter.average()))\n",
    "        print(attribute_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A helper method to allow altering word embedding models loaded in gensim library, so that we can add an all-zero vector for OOV words\n",
    "# Vocab class copied from gensim project\n",
    "class Vocab(object):\n",
    "    \"\"\"\n",
    "    A single vocabulary item, used internally for collecting per-word frequency/sampling info,\n",
    "    and for constructing binary trees (incl. both word leaves and inner nodes).\n",
    "    \"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        self.count = 0\n",
    "        self.__dict__.update(kwargs)\n",
    "\n",
    "    def __lt__(self, other):  # used for sorting in a priority queue\n",
    "        return self.count < other.count\n",
    "\n",
    "    def __str__(self):\n",
    "        vals = ['%s:%r' % (key, self.__dict__[key]) for key in sorted(self.__dict__) if not key.startswith('_')]\n",
    "        return \"%s(%s)\" % (self.__class__.__name__, ', '.join(vals))\n",
    "    \n",
    "def gensim_add_zero_word(model, word):\n",
    "    word_id = len(model.vocab)\n",
    "    model.vocab[word] = Vocab(index=word_id, count=1)\n",
    "    model.syn0 = np.append(model.syn0, [np.zeros(model.vector_size, dtype=np.float32)], 0)\n",
    "    model.index2word.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# class to build input/output vectors for each sentence from the raw data.\n",
    "FeatureVectors = collections.namedtuple('FeatureVectors', \n",
    "    ['wordform_ids','wordshape', 'wordform_embeddings','analyzer_options','ngrams','tag_ids','pos_ids','attribute_ids'])\n",
    "class FeatureFactory(object):\n",
    "    def __init__(self, vocabularies, embeddings):\n",
    "        self._vocabularies = vocabularies\n",
    "        self._embeddings = embeddings\n",
    "        # Tā kā gensim modeļi mirst, ja prasam OOV vārdus, tad mums vajag pievienot tur nulles vektoru ja tāds jau nav\n",
    "        if self._embeddings and unk not in self._embeddings:\n",
    "            gensim_add_zero_word(self._embeddings, unk)\n",
    "    \n",
    "    def wordform_vector_size(self):\n",
    "        return len(self._vocabularies.voc_wordforms) # one-hot vector describing the wordform\n",
    "    \n",
    "    def embedding_vector_size(self):\n",
    "        return self._embeddings.vector_size\n",
    "    \n",
    "    def tag_vector_size(self):\n",
    "        return len(self._vocabularies.voc_tags) # one-hot vector mapping to individual tags\n",
    "\n",
    "    def pos_vector_size(self):\n",
    "        return len(self._vocabularies.voc_pos) # one-hot vector mapping to pos tags\n",
    "\n",
    "    def ngram_vector_size(self):\n",
    "        return len(self._vocabularies.voc_ngrams) # n-hot vector mapping to ngrams\n",
    "    \n",
    "    def attribute_vector_size(self):\n",
    "        return len(self._vocabularies.voc_attributes) # n-hot vector mapping to attribute-value pairs\n",
    "\n",
    "    def wordshape_vector_size(self):\n",
    "        return 1\n",
    "    \n",
    "    # convert a document to the appropriate numeric input and output vectors suitable for tagging\n",
    "    def vectorize(self, document):\n",
    "        return [self._vectorize_sentence(sentence) for sentence in document.sentences]\n",
    "               \n",
    "    def _vectorize_sentence(self, sentence):\n",
    "        wordform_ids = np.array([self._vocabularies.voc_wordforms.get(token[wordform_key]) for token in sentence])\n",
    "        if input_features.get('wordform_embeddings'):\n",
    "            wordforms_filtered = [token[wordform_key] if token[wordform_key] in self._embeddings else unk for token in sentence]\n",
    "            wordform_embeddings = np.tanh(self._embeddings[wordforms_filtered]) # normalizējam uz [0..1] diapazonu no [-x..+x]\n",
    "        else:\n",
    "            wordform_embeddings = None\n",
    "        \n",
    "        if input_features.get('wordshape'):\n",
    "            wordshape = np.zeros([len(sentence), 1], dtype = np.float32)\n",
    "            for tok_id, token in enumerate(sentence):\n",
    "                if token[wordform_original_key][0].isupper():  # Pagaidām tikai pārbaudam vai sākas ar lielo burtu.\n",
    "                    wordshape[tok_id, 0] = 1\n",
    "        else:\n",
    "            wordshape = None\n",
    "            \n",
    "        if input_features.get('analyzer_nhot'):\n",
    "            # TODO - šo noteikti var kautkā optimizēt, ja datu preparēšana kļūst par lēnu\n",
    "            analyzer_options = np.zeros([len(sentence),len(self._vocabularies.voc_tags)], dtype = np.float32)\n",
    "            for tok_id, token in enumerate(sentence):\n",
    "                for option in token[options_key]:\n",
    "                    analyzer_options[tok_id, self._vocabularies.voc_tags.get(option)] = 1\n",
    "        else:\n",
    "            analyzer_options = None\n",
    "            \n",
    "        if input_features.get('ngrams'):\n",
    "            ngrams = np.zeros([len(sentence),len(self._vocabularies.voc_ngrams)], dtype = np.float32)\n",
    "            for tok_id, token in enumerate(sentence):\n",
    "                wordform = token.get(wordform_key)\n",
    "                for i in range(1, max_ngrams+1):\n",
    "                    ngrams[tok_id, self._vocabularies.voc_ngrams.get(wordform[-i:])] = 1\n",
    "        else:\n",
    "            ngrams = None            \n",
    "        \n",
    "        tag_ids = np.array([self._vocabularies.voc_tags.get(token[tag_key]) for token in sentence], dtype = np.float32)\n",
    "        pos_ids = np.array([self._vocabularies.voc_pos.get(token[pos_key]) for token in sentence], dtype = np.float32)\n",
    "        if output_features.get('attribute_nhot'):\n",
    "            attribute_ids = np.zeros([len(sentence),len(self._vocabularies.voc_attributes)], dtype = np.float32)\n",
    "            for tok_id, token in enumerate(sentence):\n",
    "                for key, value in token[attribute_key].items():\n",
    "                    attribute_ids[tok_id, self._vocabularies.voc_attributes.get('{}:{}'.format(key, value))] = 1\n",
    "        else:\n",
    "            attribute_ids = None\n",
    "        return FeatureVectors(wordform_ids, wordshape, wordform_embeddings, analyzer_options, ngrams, tag_ids, pos_ids, attribute_ids)\n",
    "    \n",
    "    def dump(self, folder):\n",
    "        self._vocabularies.dump(folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# NN layer helper functions\n",
    "\n",
    "# A layer for a bidirectional RNN (currently LSTM cells)\n",
    "def recurrent_layer(input_layer, sequence_lengths, dropout_keep_prob, name_scope='recurrent', rnn_hidden=100, num_layers=1):\n",
    "    with tf.name_scope(name_scope):\n",
    "        # Transform from \"num_words x data_width\" uz \"batch_size x num_words x data_width\"\n",
    "        batched_input_layer = tf.expand_dims(input_layer,0)\n",
    "        with tf.name_scope('fw'):\n",
    "            cell_fw = tf.nn.rnn_cell.LSTMCell(rnn_hidden, state_is_tuple=True)\n",
    "            cell_with_dropout_fw = tf.nn.rnn_cell.DropoutWrapper(cell_fw, output_keep_prob=dropout_keep_prob)                \n",
    "            if num_layers > 1:\n",
    "                multicell_fw = tf.nn.rnn_cell.MultiRNNCell([cell_with_dropout_fw] * num_layers, state_is_tuple=True)\n",
    "            else:\n",
    "                multicell_fw = cell_fw\n",
    "            \n",
    "        with tf.name_scope('bw'):\n",
    "            cell_bw = tf.nn.rnn_cell.LSTMCell(rnn_hidden, state_is_tuple=True)\n",
    "            cell_with_dropout_bw = tf.nn.rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=dropout_keep_prob)\n",
    "            if num_layers > 1:\n",
    "                multicell_bw = tf.nn.rnn_cell.MultiRNNCell([cell_with_dropout_bw] * num_layers, state_is_tuple=True)\n",
    "            else:\n",
    "                multicell_bw = cell_bw\n",
    "        rnn_outputs, _ = tf.nn.bidirectional_dynamic_rnn(multicell_fw, multicell_bw, batched_input_layer,\n",
    "                                                         sequence_lengths, scope=name_scope+\"BiRNN\", dtype=tf.float32)\n",
    "        rnn_output = tf.concat(2, rnn_outputs)\n",
    "        return tf.reshape(rnn_output, [-1, rnn_hidden*2]) # flatten to return to an unbatched representation\n",
    "    \n",
    "# standard fully connected ReLU layer\n",
    "def fully_connected_layer(input_layer, dropout_keep_prob, hidden_units, name_scope='fully_connected'):\n",
    "    input_vector_size = input_layer.get_shape().as_list()[1]\n",
    "    with tf.name_scope(name_scope):\n",
    "        weights = tf.Variable(tf.truncated_normal([input_vector_size, hidden_units], stddev=0.1), name='weights')\n",
    "        bias = tf.Variable(tf.zeros([hidden_units]), name='bias')\n",
    "        fc = tf.nn.relu(tf.matmul(input_layer, weights) + bias, name=name_scope) \n",
    "        return tf.nn.dropout(fc, dropout_keep_prob)\n",
    "    \n",
    "# one dimensional convolution over words in sentence; from a vector in shape [num_words x in_data_width], return [num_words x out_data_width]\n",
    "# For each word's output, it will take input data from 'window' of surrounding words\n",
    "def convolution_layer(input_layer, window = 3, hidden_units = 400, name_scope='convolution'):\n",
    "    input_vector_size = input_layer.get_shape().as_list()[1]\n",
    "    # Transform from [num_words x data_width] to [1 x 1 x num_words x data_width] - \"[batch x height x num_words x data_width]\"\n",
    "    batched_layer = tf.expand_dims( tf.expand_dims(input_layer, 0), 0)\n",
    "    with tf.name_scope(name_scope):\n",
    "        convolution = tf.Variable(tf.truncated_normal([1, window, input_vector_size, hidden_units], stddev=0.1) , name='convolution_filter')\n",
    "        result = tf.nn.conv2d(batched_layer, convolution, [1,1,1,1], padding='SAME')        \n",
    "        unbatched = tf.reshape(result, [-1, hidden_units])\n",
    "    return unbatched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The actual tagger class - construction of the network graph, training and tagging\n",
    "class Tagger(object):\n",
    "    def __init__(self, featurefactory = None):\n",
    "        self.session = None\n",
    "        self._featurefactory = featurefactory # necessary because model structure depends on vocabulary sizes\n",
    "        self._prepare_graph()\n",
    "        \n",
    "    def __del__(self):\n",
    "        if self.session:\n",
    "            self.session.close()\n",
    "        \n",
    "    def _feed_dict(self, sentence, train=False, feed_output=True):\n",
    "        feed_dict = dict()\n",
    "        feed_dict[self.sentence_length] = [len(sentence.tag_ids)]\n",
    "        if train:\n",
    "            feed_dict[self.dropout_keep_prob] = 0.5\n",
    "        else:\n",
    "            feed_dict[self.dropout_keep_prob] = 1.0\n",
    "        if input_features.get('wordform_onehot'):\n",
    "            feed_dict[self.wordform_ids] = sentence.wordform_ids\n",
    "        if input_features.get('wordform_embeddings'):\n",
    "            feed_dict[self.wordform_embeddings] = sentence.wordform_embeddings\n",
    "        if input_features.get('analyzer_nhot'):\n",
    "            feed_dict[self.analyzer_options] = sentence.analyzer_options\n",
    "        if input_features.get('ngrams'):\n",
    "            feed_dict[self.ngrams] = sentence.ngrams\n",
    "        if input_features.get('wordshape'):\n",
    "            feed_dict[self.wordshape] = sentence.wordshape\n",
    "        if feed_output:\n",
    "            if output_features.get('pos_onehot'):\n",
    "                feed_dict[self.pos_ids] = sentence.pos_ids\n",
    "            if output_features.get('tag_onehot'):\n",
    "                feed_dict[self.tag_ids] = sentence.tag_ids\n",
    "            if output_features.get('attribute_nhot'):\n",
    "                feed_dict[self.attribute_ids] = sentence.attribute_ids\n",
    "        return feed_dict\n",
    "            \n",
    "    def _prepare_graph(self):\n",
    "        tf.reset_default_graph()\n",
    "        \n",
    "        self.dropout_keep_prob = tf.placeholder(tf.float32, name = 'dropout_keep_prob')\n",
    "        self.sentence_length = tf.placeholder(tf.int64, name = 'sentence_length')\n",
    "                \n",
    "        with tf.name_scope('input'):\n",
    "            input_vectors = []\n",
    "            compressed_input_vectors = []\n",
    "            if input_features.get('wordform_onehot'):\n",
    "                wordform_vector_size = self._featurefactory.wordform_vector_size()\n",
    "                self.wordform_ids = tf.placeholder(tf.int64, [None], name = 'wordform_ids')\n",
    "                wordform_onehot = tf.one_hot(self.wordform_ids, wordform_vector_size, 1.0, 0.0, name = 'wordform_onehot')\n",
    "                compressed_wordform = fully_connected_layer(wordform_onehot, self.dropout_keep_prob, 400, 'compressed_wordform')\n",
    "                input_vectors.append(wordform_onehot)\n",
    "                compressed_input_vectors.append(compressed_wordform)\n",
    "            if input_features.get('wordshape'):\n",
    "                wordshape_vector_size = self._featurefactory.wordshape_vector_size()\n",
    "                self.wordshape = tf.placeholder(tf.float32, [None, wordshape_vector_size], name = 'wordshape')\n",
    "                input_vectors.append(self.wordshape)\n",
    "                compressed_input_vectors.append(self.wordshape)\n",
    "            if input_features.get('wordform_embeddings'):\n",
    "                embedding_vector_size = self._featurefactory.embedding_vector_size()\n",
    "                self.wordform_embeddings = tf.placeholder(tf.float32, [None, embedding_vector_size], name = 'wordform_embeddings')\n",
    "#                 normalized_embeddings = tf.sigmoid(self.wordform_embeddings)\n",
    "                input_vectors.append(self.wordform_embeddings)\n",
    "                compressed_input_vectors.append(self.wordform_embeddings)\n",
    "            if input_features.get('analyzer_nhot'):\n",
    "                tag_vector_size = self._featurefactory.tag_vector_size()\n",
    "                self.analyzer_options = tf.placeholder(tf.float32, [None, tag_vector_size], name='analyzer_options_nhot')\n",
    "                input_vectors.append(self.analyzer_options)\n",
    "                compressed_input_vectors.append(self.analyzer_options)\n",
    "            if input_features.get('ngrams'):\n",
    "                ngram_vector_size = self._featurefactory.ngram_vector_size()\n",
    "                self.ngrams = tf.placeholder(tf.float32, [None, ngram_vector_size], name='ngram_nhot')\n",
    "                input_vectors.append(self.ngrams)\n",
    "                compressed_ngrams = fully_connected_layer(self.ngrams, self.dropout_keep_prob, 400, 'compressed_ngrams')\n",
    "                compressed_input_vectors.append(compressed_ngrams)\n",
    "            input_vector = tf.concat(1, input_vectors)\n",
    "            compressed_input_vector = tf.concat(1, compressed_input_vectors)\n",
    "            del input_vectors\n",
    "            del compressed_input_vectors\n",
    "\n",
    "        with tf.name_scope('gold_output'):\n",
    "            gold_output_vectors = []\n",
    "            if output_features.get('pos_onehot'):\n",
    "                pos_vector_size = self._featurefactory.pos_vector_size()\n",
    "                self.pos_ids = tf.placeholder(tf.int64, [None], name = 'pos_ids')\n",
    "                gold_pos_onehot = tf.one_hot(self.pos_ids, pos_vector_size, 1.0, 0.0, name = 'gold_pos_onehot')\n",
    "                gold_output_vectors.append(gold_pos_onehot)\n",
    "            if output_features.get('tag_onehot'):\n",
    "                tag_vector_size = self._featurefactory.tag_vector_size()\n",
    "                self.tag_ids = tf.placeholder(tf.int64, [None], name = 'tag_ids')\n",
    "                gold_tag_onehot = tf.one_hot(self.tag_ids, tag_vector_size, 1.0, 0.0, name = 'gold_tag_onehot')\n",
    "                gold_output_vectors.append(gold_tag_onehot)\n",
    "            if output_features.get('attribute_nhot'):\n",
    "                attribute_vector_size = self._featurefactory.attribute_vector_size()\n",
    "                self.attribute_ids = tf.placeholder(tf.float32, [None, attribute_vector_size], name = 'attribute_nhot')\n",
    "                gold_output_vectors.append(self.attribute_ids)\n",
    "            gold_output_vector = tf.concat(1,gold_output_vectors)\n",
    "            del gold_output_vectors\n",
    "            output_vector_size = gold_output_vector.get_shape().as_list()[1]\n",
    "\n",
    "        # NB! the following lines define the main layout of the nework\n",
    "#         layer = input_vector\n",
    "        layer = compressed_input_vector\n",
    "        layer = convolution_layer(layer, window=3, hidden_units=500, name_scope = 'convolution1')\n",
    "#         layer = fully_connected_layer(layer, self.dropout_keep_prob, 400)\n",
    "        layer = recurrent_layer(layer, self.sentence_length, self.dropout_keep_prob, 'recurrent1', rnn_hidden=400)\n",
    "#         layer = recurrent_layer(layer, self.sentence_length, self.dropout_keep_prob, 'recurrent2', rnn_hidden=300)\n",
    "#         layer = recurrent_layer(layer, self.sentence_length, self.dropout_keep_prob, 'recurrent3', rnn_hidden=300)         \n",
    "        layer = tf.concat(1, [input_vector, layer]) # wide and deep\n",
    "        final_layer = layer\n",
    "        final_layer_size = final_layer.get_shape().as_list()[1]\n",
    "                \n",
    "        with tf.name_scope('softmax'):\n",
    "            weights = tf.Variable(tf.truncated_normal([final_layer_size, output_vector_size], stddev=0.1), name='weights')  # simple mapping from all words to all tags\n",
    "            bias = tf.Variable(tf.zeros([output_vector_size]), name='bias')\n",
    "            output_vector = tf.nn.softmax(tf.matmul(final_layer, weights) + bias, name='output_vector')\n",
    "        \n",
    "        regularization_coeff = 1e-7\n",
    "#         regularization_coeff = 0\n",
    "        with tf.name_scope('train'):\n",
    "            cross_entropy = tf.reduce_mean(-tf.reduce_sum(gold_output_vector * tf.log(output_vector), reduction_indices=[1]), name='cross_entropy')                    \n",
    "            if regularization_coeff > 0:\n",
    "                loss = cross_entropy + regularization_coeff * (tf.nn.l2_loss(weights) + tf.nn.l2_loss(bias))\n",
    "            else:\n",
    "                loss = cross_entropy\n",
    "            cross_entropy_summary = tf.scalar_summary('cross entropy', cross_entropy)\n",
    "            loss_summary = tf.scalar_summary('loss', loss)\n",
    "            self.train_step = tf.train.AdamOptimizer(1e-4).minimize(loss, name='train_step')\n",
    "        \n",
    "        with tf.name_scope('evaluate'):\n",
    "            start_index = 0\n",
    "            if output_features.get('pos_onehot'):\n",
    "                pos_onehot = tf.slice(output_vector, [0, start_index], [-1, pos_vector_size])\n",
    "                start_index = start_index + pos_vector_size\n",
    "                self.pos_answers = tf.argmax(pos_onehot, 1, name='pos_answers')\n",
    "                pos_accuracy = tf.equal(tf.argmax(gold_pos_onehot, 1), tf.argmax(pos_onehot, 1), name='pos_accuracy')\n",
    "                self.pos_accuracy_measure = tf.reduce_mean(tf.cast(pos_accuracy, tf.float32))\n",
    "                self.pos_accuracy_summary = tf.scalar_summary('pos_accuracy', self.pos_accuracy_measure)\n",
    "            if output_features.get('tag_onehot'):                \n",
    "                tag_onehot = tf.slice(output_vector, [0, start_index], [-1, tag_vector_size])\n",
    "                start_index = start_index + tag_vector_size\n",
    "                self.tag_answers = tf.argmax(tag_onehot, 1, name='tag_answers')\n",
    "                tag_accuracy = tf.equal(tf.argmax(gold_tag_onehot, 1), tf.argmax(tag_onehot, 1), name='tag_accuracy')\n",
    "                self.tag_accuracy_measure = tf.reduce_mean(tf.cast(tag_accuracy, tf.float32))\n",
    "                self.tag_accuracy_summary = tf.scalar_summary('tag_accuracy', self.tag_accuracy_measure)\n",
    "            if output_features.get('attribute_nhot'):\n",
    "                attributes_nhot = tf.slice(output_vector, [0, start_index], [-1, attribute_vector_size])\n",
    "                self.attribute_answers = attributes_nhot\n",
    "                start_index = start_index + attribute_vector_size\n",
    "            assert start_index == output_vector_size  # check that all output has been processed            \n",
    "            \n",
    "        init = tf.initialize_all_variables()\n",
    "        self.merged = tf.merge_all_summaries()\n",
    "        self.session = tf.Session()\n",
    "        self.saver = tf.train.Saver()\n",
    "        self.session.run(init)\n",
    "        \n",
    "    def train(self, train_data, epochs, test_data = None):\n",
    "        sentence_count = 0\n",
    "        self.run_id = datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\n",
    "        self.train_writer = tf.train.SummaryWriter(output_dir + '/tb/train' + self.run_id, self.session.graph)\n",
    "        self.test_writer = tf.train.SummaryWriter(output_dir + '/tb/test' + self.run_id)\n",
    "\n",
    "        for epoch in range(1, epochs+1):\n",
    "            for sentence in train_data:\n",
    "                # If we don't want to check process in tensorboard, the following one line would suffice\n",
    "                # self.session.run(self.train_step, feed_dict = {self.token_ids : sentence_wordforms, self.tag_ids : sentence_tags})                \n",
    "                sentence_count = sentence_count + 1\n",
    "                if sentence_count % 1000 == 0:\n",
    "#                     run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "#                     run_metadata = tf.RunMetadata()\n",
    "#                     summary, _ = self.session.run([self.merged, self.train_step], feed_dict = self._feed_dict(sentence, train=True),\n",
    "#                                                  options = run_options, run_metadata = run_metadata)\n",
    "#                     self.train_writer.add_run_metadata(run_metadata, 'sentences_{}k'.format(sentence_count // 1000), sentence_count)\n",
    "                    summary, _ = self.session.run([self.merged, self.train_step], feed_dict = self._feed_dict(sentence, train=True))\n",
    "                    self.train_writer.add_summary(summary, sentence_count)\n",
    "                    if test_data:\n",
    "                        #TODO - calculate all results in a single parse\n",
    "                        if output_features.get('pos_onehot'):\n",
    "                            acc = self.evaluate_accuracy(test_data, self.pos_accuracy_measure)\n",
    "                            test_summary = tf.Summary(value=[tf.Summary.Value(tag=\"pos_accuracy\", simple_value=acc)])\n",
    "                            self.test_writer.add_summary(test_summary, sentence_count)\n",
    "                        if output_features.get('tag_onehot'):\n",
    "                            acc = self.evaluate_accuracy(test_data, self.tag_accuracy_measure)\n",
    "                            test_summary = tf.Summary(value=[tf.Summary.Value(tag=\"tag_accuracy\", simple_value=acc)])\n",
    "                            self.test_writer.add_summary(test_summary, sentence_count)\n",
    "                        if output_features.get('attributes_nhot'):\n",
    "                            # TODO - a simple metric for attribute accuracy \n",
    "                            pass\n",
    "                elif sentence_count % 10 == 9:\n",
    "                    summary, _ = self.session.run([self.merged, self.train_step], feed_dict = self._feed_dict(sentence, train=True))\n",
    "                    self.train_writer.add_summary(summary, sentence_count)\n",
    "                else:\n",
    "                    self.session.run(self.train_step, feed_dict = self._feed_dict(sentence, train=True))\n",
    "            print('Epoch {} done'.format(epoch))\n",
    "            \n",
    "    def evaluate_accuracy(self, document_vectors, measure):\n",
    "        acc = 0.0\n",
    "        tokens = 0.0\n",
    "        for sentence in document_vectors:\n",
    "            tokens = tokens + len(sentence.tag_ids)\n",
    "            acc = acc + len(sentence.tag_ids) * self.session.run(measure, feed_dict = self._feed_dict(sentence))\n",
    "        acc = acc / tokens\n",
    "        return acc\n",
    "                \n",
    "    \n",
    "    #TODO - calculate all outputs from a single run over testdata\n",
    "    def _parse_sentences_tags(self, document_vectors, vocabularies):\n",
    "        for sentence in document_vectors:\n",
    "            silver_tag_ids = self.session.run(self.tag_answers, feed_dict = self._feed_dict(sentence, feed_output=False))\n",
    "            yield [vocabularies.voc_tags.reverse(tag_id) for tag_id in silver_tag_ids]\n",
    "    def _parse_sentences_pos(self, document_vectors, vocabularies):\n",
    "        for sentence in document_vectors:\n",
    "            silver_pos_ids = self.session.run(self.pos_answers, feed_dict = self._feed_dict(sentence, feed_output=False))\n",
    "            yield [vocabularies.voc_pos.reverse(pos_id) for pos_id in silver_pos_ids]\n",
    "    def _parse_sentences_attributes(self, document_vectors, vocabularies):\n",
    "        for sentence in document_vectors:\n",
    "            silver_attribute_data = self.session.run(self.attribute_answers, feed_dict = self._feed_dict(sentence, feed_output=False))\n",
    "            result = []\n",
    "            for token_silver_attributes in silver_attribute_data:\n",
    "                result.append({vocabularies.voc_attributes.reverse(attribute_id) : value for attribute_id, value in enumerate(token_silver_attributes)})\n",
    "            yield result        \n",
    "        \n",
    "    def tag(self, test_doc, test_data, vocabularies, filename):\n",
    "        silver_tags = None\n",
    "        silver_pos = None\n",
    "        silver_attributes = None\n",
    "        if output_features.get('pos_onehot'):\n",
    "            silver_pos = self._parse_sentences_pos(test_data, vocabularies)\n",
    "        if output_features.get('tag_onehot'):\n",
    "            silver_tags = self._parse_sentences_tags(test_data, vocabularies)\n",
    "        if output_features.get('attribute_nhot'):\n",
    "            silver_attributes = self._parse_sentences_attributes(test_data, vocabularies)\n",
    "        test_doc.output_tagged(silver_tags, silver_pos, silver_attributes, filename, vocabularies = vocabularies)\n",
    "        \n",
    "    def dump(self, folder, filename = 'model.tf'):\n",
    "        self._featurefactory.dump(folder)\n",
    "        self.saver.save(self.session, folder + '/' + filename)\n",
    "        \n",
    "    def load_model(self, folder):\n",
    "        self.saver.restore(self.session, folder + '/model.tf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The actual experiment - training and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading...\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[12240,520]\n\t [[Node: train/softmax/weights/Adam/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@softmax/weights\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](train/softmax/weights/Adam, train/zeros_18)]]\nCaused by op 'train/softmax/weights/Adam/Assign', defined at:\n  File \"/usr/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/traitlets/config/application.py\", line 596, in launch_instance\n    app.start()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 162, in start\n    super(ZMQIOLoop, self).start()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 498, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2705, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2809, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2869, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-11-e1dfc88ab947>\", line 27, in <module>\n    trained_tagger = train_stuff(Tagger(featurefactory))\n  File \"<ipython-input-9-09f938c9896a>\", line 6, in __init__\n    self._prepare_graph()\n  File \"<ipython-input-9-09f938c9896a>\", line 128, in _prepare_graph\n    self.train_step = tf.train.AdamOptimizer(1e-4).minimize(loss, name='train_step')\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py\", line 198, in minimize\n    name=name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py\", line 300, in apply_gradients\n    self._create_slots(var_list)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/adam.py\", line 118, in _create_slots\n    self._zeros_slot(v, \"m\", self._name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py\", line 494, in _zeros_slot\n    named_slots[var] = slot_creator.create_zeros_slot(var, op_name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/slot_creator.py\", line 108, in create_zeros_slot\n    colocate_with_primary=colocate_with_primary)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/slot_creator.py\", line 86, in create_slot\n    return _create_slot_var(primary, val, scope)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/slot_creator.py\", line 50, in _create_slot_var\n    slot = variables.Variable(val, name=scope, trainable=False)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 211, in __init__\n    dtype=dtype)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 313, in _init_from_args\n    validate_shape=validate_shape).op\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/ops/gen_state_ops.py\", line 45, in assign\n    use_locking=use_locking, name=name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 703, in apply_op\n    op_def=op_def)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 2317, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1239, in __init__\n    self._traceback = _extract_stack()\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m    964\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 965\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m    946\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m    948\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                 \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/errors.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m    449\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 450\u001b[0;31m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m    451\u001b[0m   \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[12240,520]\n\t [[Node: train/softmax/weights/Adam/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@softmax/weights\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](train/softmax/weights/Adam, train/zeros_18)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-e1dfc88ab947>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtagger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mtrained_tagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_stuff\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTagger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeaturefactory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-9-09f938c9896a>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, featurefactory)\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_featurefactory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeaturefactory\u001b[0m \u001b[0;31m# necessary because model structure depends on vocabulary sizes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prepare_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__del__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-09f938c9896a>\u001b[0m in \u001b[0;36m_prepare_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    154\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    708\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 710\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    711\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    906\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 908\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    909\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    910\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    956\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    957\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m--> 958\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m    959\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    960\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m    976\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    979\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[12240,520]\n\t [[Node: train/softmax/weights/Adam/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@softmax/weights\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](train/softmax/weights/Adam, train/zeros_18)]]\nCaused by op 'train/softmax/weights/Adam/Assign', defined at:\n  File \"/usr/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/traitlets/config/application.py\", line 596, in launch_instance\n    app.start()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 162, in start\n    super(ZMQIOLoop, self).start()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 498, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2705, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2809, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2869, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-11-e1dfc88ab947>\", line 27, in <module>\n    trained_tagger = train_stuff(Tagger(featurefactory))\n  File \"<ipython-input-9-09f938c9896a>\", line 6, in __init__\n    self._prepare_graph()\n  File \"<ipython-input-9-09f938c9896a>\", line 128, in _prepare_graph\n    self.train_step = tf.train.AdamOptimizer(1e-4).minimize(loss, name='train_step')\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py\", line 198, in minimize\n    name=name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py\", line 300, in apply_gradients\n    self._create_slots(var_list)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/adam.py\", line 118, in _create_slots\n    self._zeros_slot(v, \"m\", self._name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py\", line 494, in _zeros_slot\n    named_slots[var] = slot_creator.create_zeros_slot(var, op_name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/slot_creator.py\", line 108, in create_zeros_slot\n    colocate_with_primary=colocate_with_primary)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/slot_creator.py\", line 86, in create_slot\n    return _create_slot_var(primary, val, scope)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/training/slot_creator.py\", line 50, in _create_slot_var\n    slot = variables.Variable(val, name=scope, trainable=False)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 211, in __init__\n    dtype=dtype)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\", line 313, in _init_from_args\n    validate_shape=validate_shape).op\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/ops/gen_state_ops.py\", line 45, in assign\n    use_locking=use_locking, name=name)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 703, in apply_op\n    op_def=op_def)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 2317, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/home/peteris/tf_source0.10/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1239, in __init__\n    self._traceback = _extract_stack()\n"
     ]
    }
   ],
   "source": [
    "print('Loading...')\n",
    "# Load both documents\n",
    "train_doc = Document(train_data_filename)\n",
    "eval_doc = Document(eval_data_filename)\n",
    "\n",
    "# Load the embeddings\n",
    "if input_features.get('wordform_embeddings'):\n",
    "    embeddings = Word2Vec.load_word2vec_format(embeddings_filename, binary=True)\n",
    "else:\n",
    "    embeddings = None\n",
    "\n",
    "# Build vocabularies and prepare vectorized data\n",
    "vocabularies = Vocabularies(train_doc)\n",
    "featurefactory = FeatureFactory(vocabularies, embeddings)    \n",
    "train_data = featurefactory.vectorize(train_doc)\n",
    "eval_data = featurefactory.vectorize(eval_doc)    \n",
    "\n",
    "def train_stuff(tagger, epochs = default_epochs):\n",
    "    # Train the model\n",
    "    print('Training...')\n",
    "    %time tagger.train(train_data, epochs, eval_data)\n",
    "    tagger.dump(output_dir)\n",
    "    print('Model saved')\n",
    "    tagger.tag(eval_doc, eval_data, vocabularies, output_dir + '/eval.tagged.txt')\n",
    "    return tagger\n",
    "    \n",
    "trained_tagger = train_stuff(Tagger(featurefactory))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Train some more epochs\n",
    "trained_tagger = train_stuff(trained_tagger, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation when reading a model from file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def tag_stuff(filename):\n",
    "    # Load embeddings\n",
    "    embeddings = Word2Vec.load_word2vec_format(embeddings_filename, binary=True)\n",
    "\n",
    "    # Load vocabularies\n",
    "    vocabularies = Vocabularies(folder = output_dir)\n",
    "    featurefactory = FeatureFactory(vocabularies, embeddings)\n",
    "\n",
    "    # Load the document and vectorize it\n",
    "    eval_doc = Document(filename, small_eval_limit)\n",
    "    eval_data = featurefactory.vectorize(eval_doc)    \n",
    "\n",
    "    # Load the tagger model\n",
    "    tagger = Tagger(featurefactory)\n",
    "    tagger.load_model(output_dir)\n",
    "    tagger.tag(eval_doc, eval_data, vocabularies, output_dir + '/eval.tagged.txt')\n",
    "    \n",
    "tag_stuff(eval_data_filename)\n",
    "tag_stuff(test_data_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}