advanced_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x117b99e30>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Author: Robert Guthrie\n",
    "\n",
    "import torch\n",
    "import torch.autograd as autograd\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper functions to make the code more readable.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argmax(vec):\n",
    "    # return the argmax as a python int\n",
    "    _, idx = torch.max(vec, 1)\n",
    "    return idx.item()\n",
    "\n",
    "\n",
    "# def prepare_sequence(seq, to_ix):\n",
    "#     idxs = [to_ix[w] for w in seq]\n",
    "#     return torch.tensor(idxs, dtype=torch.long)\n",
    "\n",
    "\n",
    "#这里增加未登录字处理\n",
    "def prepare_sequence(seq, to_ix):\n",
    "    idxs = [to_ix[w] if w in to_ix else to_ix[\"<UNK>\"] for w in seq] #to_ix.has_key(x)\n",
    "    return torch.tensor(idxs, dtype=torch.long)\n",
    "\n",
    "\n",
    "# Compute log sum exp in a numerically stable way for the forward algorithm\n",
    "def log_sum_exp(vec):\n",
    "    max_score = vec[0, argmax(vec)]\n",
    "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
    "    return max_score + torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLSTM_CRF(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, tag_to_ix, embedding_dim, hidden_dim):\n",
    "        super(BiLSTM_CRF, self).__init__()\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.vocab_size = vocab_size\n",
    "        self.tag_to_ix = tag_to_ix\n",
    "        self.tagset_size = len(tag_to_ix)\n",
    "\n",
    "        self.word_embeds = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(input_size=embedding_dim, hidden_size=hidden_dim // 2,\n",
    "                            num_layers=1, bidirectional=True)\n",
    "\n",
    "        # Maps the output of the LSTM into tag space.\n",
    "        self.hidden2tag = nn.Linear(hidden_dim, self.tagset_size)\n",
    "\n",
    "        # Matrix of transition parameters.  Entry i,j is the score of\n",
    "        # transitioning *to* i *from* j.\n",
    "        self.transitions = nn.Parameter(\n",
    "            torch.randn(self.tagset_size, self.tagset_size))\n",
    "\n",
    "        # These two statements enforce the constraint that we never transfer\n",
    "        # to the start tag and we never transfer from the stop tag\n",
    "        self.transitions.data[tag_to_ix[START_TAG], :] = -10000\n",
    "        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000\n",
    "\n",
    "        self.hidden = self.init_hidden()\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return (torch.randn(2, 1, self.hidden_dim // 2),\n",
    "                torch.randn(2, 1, self.hidden_dim // 2))\n",
    "\n",
    "    \n",
    "    #根据转移矩阵，预测序列的得分\n",
    "    def _forward_alg(self, feats):\n",
    "        # Do the forward algorithm to compute the partition function\n",
    "        init_alphas = torch.full((1, self.tagset_size), -10000.)\n",
    "        # START_TAG has all of the score.\n",
    "        init_alphas[0][self.tag_to_ix[START_TAG]] = 0.\n",
    "\n",
    "        # Wrap in a variable so that we will get automatic backprop\n",
    "        forward_var = init_alphas\n",
    "\n",
    "        # Iterate through the sentence\n",
    "        for feat in feats:\n",
    "            alphas_t = []  # The forward tensors at this timestep\n",
    "            for next_tag in range(self.tagset_size):\n",
    "                # broadcast the emission score: it is the same regardless of\n",
    "                # the previous tag\n",
    "                emit_score = feat[next_tag].view(1, -1).expand(1, self.tagset_size)\n",
    "                # the ith entry of trans_score is the score of transitioning to\n",
    "                # next_tag from i\n",
    "                trans_score = self.transitions[next_tag].view(1, -1)\n",
    "                # The ith entry of next_tag_var is the value for the\n",
    "                # edge (i -> next_tag) before we do log-sum-exp\n",
    "                next_tag_var = forward_var + trans_score + emit_score\n",
    "                # The forward variable for this tag is log-sum-exp of all the\n",
    "                # scores.\n",
    "                alphas_t.append(log_sum_exp(next_tag_var).view(1))\n",
    "            forward_var = torch.cat(alphas_t).view(1, -1)\n",
    "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
    "        alpha = log_sum_exp(terminal_var)\n",
    "        return alpha\n",
    "\n",
    "    #这个就是sentence-embedding-lstm-linear的过程，输出是linear的结果。这个输出的lstm_feats就是发射矩阵\n",
    "    def _get_lstm_features(self, sentence):\n",
    "        self.hidden = self.init_hidden()\n",
    "        embeds = self.word_embeds(sentence).view(len(sentence), 1, -1)\n",
    "        lstm_out, self.hidden = self.lstm(embeds, self.hidden)\n",
    "        lstm_out = lstm_out.view(len(sentence), self.hidden_dim)\n",
    "        lstm_feats = self.hidden2tag(lstm_out)\n",
    "        return lstm_feats\n",
    "\n",
    "    def _score_sentence(self, feats, tags):\n",
    "        # Gives the score of a provided tag sequence\n",
    "        score = torch.zeros(1)\n",
    "        tags = torch.cat([torch.tensor([self.tag_to_ix[START_TAG]], dtype=torch.long), tags])\n",
    "        for i, feat in enumerate(feats):\n",
    "            score = score + self.transitions[tags[i + 1], tags[i]] + feat[tags[i + 1]]\n",
    "        score = score + self.transitions[self.tag_to_ix[STOP_TAG], tags[-1]]\n",
    "        return score\n",
    "\n",
    "    #解码，得到预测的序列，以及预测序列的得分\n",
    "    def _viterbi_decode(self, feats):\n",
    "        backpointers = []\n",
    "\n",
    "        # Initialize the viterbi variables in log space\n",
    "        init_vvars = torch.full((1, self.tagset_size), -10000.)\n",
    "        init_vvars[0][self.tag_to_ix[START_TAG]] = 0\n",
    "\n",
    "        # forward_var at step i holds the viterbi variables for step i-1\n",
    "        forward_var = init_vvars\n",
    "        for feat in feats:\n",
    "            bptrs_t = []  # holds the backpointers for this step\n",
    "            viterbivars_t = []  # holds the viterbi variables for this step\n",
    "\n",
    "            for next_tag in range(self.tagset_size):\n",
    "                # next_tag_var[i] holds the viterbi variable for tag i at the\n",
    "                # previous step, plus the score of transitioning\n",
    "                # from tag i to next_tag.\n",
    "                # We don't include the emission scores here because the max\n",
    "                # does not depend on them (we add them in below)\n",
    "                next_tag_var = forward_var + self.transitions[next_tag]\n",
    "                best_tag_id = argmax(next_tag_var)\n",
    "                bptrs_t.append(best_tag_id)\n",
    "                viterbivars_t.append(next_tag_var[0][best_tag_id].view(1))\n",
    "            # Now add in the emission scores, and assign forward_var to the set\n",
    "            # of viterbi variables we just computed\n",
    "            forward_var = (torch.cat(viterbivars_t) + feat).view(1, -1)\n",
    "            backpointers.append(bptrs_t)\n",
    "\n",
    "        # Transition to STOP_TAG\n",
    "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
    "        best_tag_id = argmax(terminal_var)\n",
    "        path_score = terminal_var[0][best_tag_id]\n",
    "\n",
    "        # Follow the back pointers to decode the best path.\n",
    "        best_path = [best_tag_id]\n",
    "        for bptrs_t in reversed(backpointers):\n",
    "            best_tag_id = bptrs_t[best_tag_id]\n",
    "            best_path.append(best_tag_id)\n",
    "        # Pop off the start tag (we dont want to return that to the caller)\n",
    "        start = best_path.pop()\n",
    "        assert start == self.tag_to_ix[START_TAG]  # Sanity check\n",
    "        best_path.reverse()\n",
    "        return path_score, best_path\n",
    "\n",
    "    def neg_log_likelihood(self, sentence, tags):\n",
    "        feats = self._get_lstm_features(sentence)\n",
    "        forward_score = self._forward_alg(feats)\n",
    "        gold_score = self._score_sentence(feats, tags)\n",
    "        return forward_score - gold_score\n",
    "\n",
    "    def forward(self, sentence):  # dont confuse this with _forward_alg above.\n",
    "        # Get the emission scores from the BiLSTM\n",
    "        lstm_feats = self._get_lstm_features(sentence)\n",
    "\n",
    "        # Find the best path, given the features.\n",
    "        score, tag_seq = self._viterbi_decode(lstm_feats)\n",
    "        return score, tag_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自己的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "f=open(\"ner_data.txt\",\"r\",encoding='utf8')\n",
    "sentences=[]\n",
    "labels=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_data=[]\n",
    "seq_label=[]\n",
    "for i in f.readlines():\n",
    "    i=i.replace(\"\\n\", \"\")\n",
    "    lst=i.split(\" \")\n",
    "    \n",
    "    if len(lst)==2:\n",
    "        seq_data.append(lst[0])\n",
    "        seq_label.append(lst[1])\n",
    "        \n",
    "    else: #语料中是空行分隔句子\n",
    "        sent=\" \".join(seq_data)\n",
    "        seq_data.clear()\n",
    "        sentences.append(sent)\n",
    "        \n",
    "        label=\" \".join(seq_label)\n",
    "        seq_label.clear()\n",
    "        labels.append(label)   \n",
    "\n",
    "#     print(i.split(\" \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50658, 50658)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences),len(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['当 希 望 工 程 救 助 的 百 万 儿 童 成 长 起 来 ， 科 教 兴 国 蔚 然 成 风 时 ， 今 天 有 收 藏 价 值 的 书 你 没 买 ， 明 日 就 叫 你 悔 不 当 初 ！',\n",
       " '藏 书 本 来 就 是 所 有 传 统 收 藏 门 类 中 的 第 一 大 户 ， 只 是 我 们 结 束 温 饱 的 时 间 太 短 而 已 。',\n",
       " '因 有 关 日 寇 在 京 掠 夺 文 物 详 情 ， 藏 界 较 为 重 视 ， 也 是 我 们 收 藏 北 京 史 料 中 的 要 件 之 一 。',\n",
       " '我 们 藏 有 一 册 1 9 4 5 年 6 月 油 印 的 《 北 京 文 物 保 存 保 管 状 态 之 调 查 报 告 》 ， 调 查 范 围 涉 及 故 宫 、 历 博 、 古 研 所 、 北 大 清 华 图 书 馆 、 北 图 、 日 伪 资 料 库 等 二 十 几 家 ， 言 及 文 物 二 十 万 件 以 上 ， 洋 洋 三 万 余 言 ， 是 珍 贵 的 北 京 史 料 。',\n",
       " '以 家 乡 的 历 史 文 献 、 特 定 历 史 时 期 书 刊 、 某 一 名 家 或 名 著 的 多 种 出 版 物 为 专 题 ， 注 意 精 品 、 非 卖 品 、 纪 念 品 ， 集 成 系 列 ， 那 收 藏 的 过 程 就 已 经 够 您 玩 味 无 穷 了 。']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O',\n",
       " 'O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O',\n",
       " 'O O O B-LOC O O B-LOC O O O O O O O O O O O O O O O O O O O O B-LOC I-LOC O O O O O O O O O',\n",
       " 'O O O O O O O O O O O O O O O O O B-LOC I-LOC O O O O O O O O O O O O O O O O O O O O O B-LOC I-LOC O B-LOC I-LOC O B-ORG I-ORG I-ORG O B-LOC I-LOC I-LOC I-LOC I-LOC I-LOC I-LOC O B-LOC I-LOC O B-LOC O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-LOC I-LOC O O O',\n",
       " 'O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=[]\n",
    "\n",
    "for i in range(len(sentences)):\n",
    "    data.append( (sentences[i].split(), labels[i].split()) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50658"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training_data[0:5]\n",
    "# training_data[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "START_TAG = \"<START>\"\n",
    "STOP_TAG = \"<STOP>\"\n",
    "tag_to_ix = {}\n",
    "word_to_ix = {\"<UNK>\":0} #生字给id=0\n",
    "\n",
    "for sentence, tags in data:\n",
    "    for word in sentence:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)\n",
    "            \n",
    "    for tag in tags:\n",
    "        if tag not in tag_to_ix:\n",
    "            tag_to_ix[tag]=len(tag_to_ix)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4768, 7)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(word_to_ix),len(tag_to_ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'O': 0,\n",
       " 'B-LOC': 1,\n",
       " 'I-LOC': 2,\n",
       " 'B-ORG': 3,\n",
       " 'I-ORG': 4,\n",
       " 'B-PER': 5,\n",
       " 'I-PER': 6}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_to_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_to_ix[START_TAG]=7\n",
    "tag_to_ix[STOP_TAG]=8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'O': 0,\n",
       " 'B-LOC': 1,\n",
       " 'I-LOC': 2,\n",
       " 'B-ORG': 3,\n",
       " 'I-ORG': 4,\n",
       " 'B-PER': 5,\n",
       " 'I-PER': 6,\n",
       " '<START>': 7,\n",
       " '<STOP>': 8}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_to_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word_to_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "training_data,test_data=train_test_split(data,test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40526, 10132)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(training_data),len(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 50\n",
    "HIDDEN_DIM = 128\n",
    "EPOCHS=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BiLSTM_CRF(len(word_to_ix), tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, weight_decay=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0 =============\n",
      "iter 500: loss 31.331604\n",
      "iter 1000: loss 11.642281\n",
      "iter 1500: loss 13.270813\n",
      "iter 2000: loss 14.185684\n",
      "iter 2500: loss 0.524353\n",
      "iter 3000: loss 1.213226\n",
      "iter 3500: loss 7.896988\n",
      "iter 4000: loss 0.836853\n",
      "iter 4500: loss 7.738739\n",
      "iter 5000: loss 0.669624\n",
      "iter 5500: loss 9.769913\n",
      "iter 6000: loss 0.747101\n",
      "iter 6500: loss 6.507309\n",
      "iter 7000: loss 12.065659\n",
      "iter 7500: loss 8.494522\n",
      "iter 8000: loss 0.991226\n",
      "iter 8500: loss 4.538750\n",
      "iter 9000: loss 0.539566\n",
      "iter 9500: loss 7.376556\n",
      "iter 10000: loss 9.603287\n",
      "iter 10500: loss 5.832260\n",
      "iter 11000: loss 36.248260\n",
      "iter 11500: loss 4.966980\n",
      "iter 12000: loss 16.922623\n",
      "iter 12500: loss 7.618118\n",
      "iter 13000: loss 8.015076\n",
      "iter 13500: loss 0.319382\n",
      "iter 14000: loss 0.348389\n",
      "iter 14500: loss 12.123367\n",
      "iter 15000: loss 2.573944\n",
      "iter 15500: loss 1.017059\n",
      "iter 16000: loss 8.537354\n",
      "iter 16500: loss 65.105072\n",
      "iter 17000: loss 0.315872\n",
      "iter 17500: loss 0.355728\n",
      "iter 18000: loss 2.654152\n",
      "iter 18500: loss 7.330177\n",
      "iter 19000: loss 21.103943\n",
      "iter 19500: loss 1.076141\n",
      "iter 20000: loss 16.772095\n",
      "iter 20500: loss 14.800644\n",
      "iter 21000: loss 0.880295\n",
      "iter 21500: loss 0.543396\n",
      "iter 22000: loss 0.489075\n",
      "iter 22500: loss 2.176689\n",
      "iter 23000: loss 4.381592\n",
      "iter 23500: loss 2.586792\n",
      "iter 24000: loss 0.264130\n",
      "iter 24500: loss 0.194214\n",
      "iter 25000: loss 0.307465\n",
      "iter 25500: loss 5.788177\n",
      "iter 26000: loss 0.767960\n",
      "iter 26500: loss 0.746811\n",
      "iter 27000: loss 3.030045\n",
      "iter 27500: loss 0.129745\n",
      "iter 28000: loss 0.119263\n",
      "iter 28500: loss 1.423813\n",
      "iter 29000: loss 13.063049\n",
      "iter 29500: loss 0.708389\n",
      "iter 30000: loss 0.085892\n",
      "iter 30500: loss 1.779854\n",
      "iter 31000: loss 0.162308\n",
      "iter 31500: loss 0.163773\n",
      "iter 32000: loss 9.120117\n",
      "iter 32500: loss 4.252502\n",
      "iter 33000: loss 2.236168\n",
      "iter 33500: loss 0.030067\n",
      "iter 34000: loss 13.508530\n",
      "iter 34500: loss 1.117493\n",
      "iter 35000: loss 11.420410\n",
      "iter 35500: loss 9.558868\n",
      "iter 36000: loss 0.224838\n",
      "iter 36500: loss 0.156174\n",
      "iter 37000: loss 0.256409\n",
      "iter 37500: loss 0.087082\n",
      "iter 38000: loss 0.144592\n",
      "iter 38500: loss 4.547577\n",
      "iter 39000: loss 3.250000\n",
      "iter 39500: loss 11.666962\n",
      "iter 40000: loss 4.172394\n",
      "iter 40500: loss 0.345177\n",
      "time used: 3775 s\n"
     ]
    }
   ],
   "source": [
    "# Make sure prepare_sequence from earlier in the LSTM section is loaded\n",
    "import time\n",
    "\n",
    "\n",
    "for epoch in range(EPOCHS):  # again, normally you would NOT do 300 epochs, it is toy data\n",
    "    print(\"epoch %d =============\" % epoch)\n",
    "    time_start = time.time()\n",
    "    \n",
    "    count=1\n",
    "    for sentence, tags in training_data:\n",
    "        # Step 1. Remember that Pytorch accumulates gradients.\n",
    "        # We need to clear them out before each instance\n",
    "        model.zero_grad()\n",
    "\n",
    "        # Step 2. Get our inputs ready for the network, that is,\n",
    "        # turn them into Tensors of word indices.\n",
    "        sentence_in = prepare_sequence(sentence, word_to_ix) \n",
    "        targets = torch.tensor([tag_to_ix[t] for t in tags], dtype=torch.long)\n",
    "\n",
    "        # Step 3. Run our forward pass.\n",
    "        loss = model.neg_log_likelihood(sentence_in, targets)\n",
    "\n",
    "        # Step 4. Compute the loss, gradients, and update the parameters by\n",
    "        # calling optimizer.step()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if count%500==0:\n",
    "            print(\"iter %d: loss %f\" %(count,loss))\n",
    "        count+=1\n",
    "        \n",
    "    time_end=time.time()\n",
    "    print(\"time used: %d s\" % (time_end-time_start)  )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare(y, y_pred):\n",
    "    error_index = []\n",
    "    if len(y) == len(y_pred):\n",
    "        for i in range(0, len(y)):\n",
    "            if y[i] != y_pred[i]:\n",
    "                error_index.append(i)\n",
    "\n",
    "    print(\"error_index:\",error_index)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['他', '说', '，', '我', '国', '正', '在', '加', '大', '治', '理', '力', '度', '，', '把', '环', '保', '工', '作', '抓', '紧', '抓', '好', '，', '到', '2', '0', '0', '0', '年', '，', '力', '争', '使', '环', '境', '污', '染', '和', '生', '态', '破', '坏', '的', '趋', '势', '得', '到', '基', '本', '控', '制', '，', '部', '分', '城', '市', '和', '地', '区', '的', '环', '境', '质', '量', '有', '所', '改', '善', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: []\n",
      "====================\n",
      "['三', '桥', '村', '村', '干', '部', '如', '此', '胆', '大', '妄', '为', '，', '农', '民', '数', '次', '上', '访', '不', '见', '结', '果', '，', '恐', '怕', '与', '基', '层', '少', '数', '领', '导', '干', '部', '“', '睁', '一', '只', '眼', '闭', '一', '只', '眼', '”', '有', '关', '。']\n",
      "['B-LOC', 'I-LOC', 'I-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: [0, 1, 3]\n",
      "====================\n",
      "['乡', '局', '级', '后', '备', '干', '部', '队', '伍', '人', '数', '达', '五', '百', '三', '十', '人', '，', '比', '上', '年', '同', '期', '增', '长', '百', '分', '之', '四', '十', '一', '点', '七', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: []\n",
      "====================\n",
      "['去', '年', '联', '邦', '银', '行', '监', '督', '局', '和', '联', '邦', '证', '券', '交', '易', '监', '督', '局', '正', '式', '将', '这', '个', '市', '场', '纳', '入', '监', '督', '范', '围', '。']\n",
      "['O', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 3, 4, 4, 4, 4, 4, 4, 0, 3, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: [9, 10]\n",
      "====================\n",
      "['3', '、', '西', '康', '铁', '路', '指', '挥', '部', '指', '挥', '长', '施', '德', '良', '（', '右', '二', '）', '与', '科', '技', '人', '员', '现', '场', '办', '公', '。']\n",
      "['O', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'O', 'B-PER', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 3, 4, 4, 4, 4, 4, 4, 0, 0, 0, 5, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: [2, 3, 4, 5, 6, 7, 8]\n",
      "====================\n",
      "['他', '说', '，', '他', '本', '人', '曾', '多', '次', '访', '华', '，', '对', '中', '国', '怀', '有', '深', '厚', '的', '感', '情', '，', '对', '中', '国', '改', '革', '开', '放', '以', '来', '所', '取', '得', '的', '成', '果', '表', '示', '赞', '赏', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOC', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: [10]\n",
      "====================\n",
      "['本', '次', '活', '动', '将', '在', '四', '月', '十', '八', '日', '至', '九', '月', '十', '八', '日', '期', '间', '举', '行', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: []\n",
      "====================\n",
      "['今', '天', '，', '当', '《', '科', '技', '之', '光', '》', '走', '过', '3', '年', '的', '艰', '苦', '历', '程', '时', '，', '中', '国', '的', '广', '大', '观', '众', '对', '她', '已', '不', '再', '陌', '生', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'B-ORG', 'I-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 3, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: [5, 6, 7, 8]\n",
      "====================\n",
      "['但', '我', '什', '么', '也', '没', '有', '做', '，', '因', '为', '我', '知', '道', '这', '一', '切', '对', '于', '它', '都', '不', '具', '有', '意', '义', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: []\n",
      "====================\n",
      "['谈', '起', '自', '己', '的', '职', '务', '问', '题', '，', '刘', '延', '宇', '讲', '了', '一', '段', '颇', '动', '情', '的', '话', '。']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PER', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "实际tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "预测tagid: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "error_index: [12]\n",
      "====================\n"
     ]
    }
   ],
   "source": [
    "# Check predictions before training\n",
    "with torch.no_grad():  \n",
    "    \n",
    "    for pair in test_data[0:10]: #抽取部分看看效果\n",
    "        sentence=pair[0]\n",
    "        tag=pair[1]\n",
    "        \n",
    "        precheck_sent = prepare_sequence(sentence, word_to_ix)\n",
    "        precheck_tags = torch.tensor([tag_to_ix[t] for t in tag], dtype=torch.long) \n",
    "        score,y_pred=model(precheck_sent)\n",
    "            \n",
    "        print(sentence)\n",
    "        print(tag)\n",
    "        print(\"实际tagid:\",list(precheck_tags.numpy()))\n",
    "        print(\"预测tagid:\",y_pred)\n",
    "        compare(precheck_tags,y_pred)\n",
    "        \n",
    "        print(\"====================\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "预测tagid: [3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():  \n",
    "    precheck_sent = prepare_sequence(\"浙江淘宝网络有限公司法定代表人发生变更，张勇卸任，由85后的蒋凡接任。\", word_to_ix)\n",
    "    score,y_pred=model(precheck_sent)\n",
    "    print(\"预测tagid:\",y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "预测tagid: [0, 0, 1, 2, 2, 2, 0, 0, 0, 0, 0, 0, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():  \n",
    "    precheck_sent = prepare_sequence(\"他在日本帝国主义侵略者和国民党的双重压力下，威武不屈，坚持斗争。\", word_to_ix)\n",
    "    score,y_pred=model(precheck_sent)\n",
    "    print(\"预测tagid:\",y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}