命名实体识别_中文.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e70a58c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PreTrainedTokenizerFast(name_or_path='hfl/rbt6', vocab_size=21128, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[ 101, 3862, 7157, 3683, 6612, 1765, 4157, 1762, 1336, 7305,  680, 7032,\n",
       "         7305,  722, 7313, 4638, 3862, 1818,  511,  102,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0],\n",
       "        [ 101, 6821, 2429,  898, 2255,  988, 3717, 4638, 1300, 4289, 7667, 4507,\n",
       "         1744, 1079,  671, 3837, 4638, 6392, 6369, 2360,  712, 2898, 6392, 6369,\n",
       "         8024, 3146,  702, 2456, 5029, 5408, 5125, 5401, 5445, 2612, 2131,  511,\n",
       "          102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "#加载分词器\n",
    "tokenizer = AutoTokenizer.from_pretrained('hfl/rbt6')\n",
    "\n",
    "print(tokenizer)\n",
    "\n",
    "#分词测试\n",
    "tokenizer.batch_encode_plus(\n",
    "    [[\n",
    "        '海', '钓', '比', '赛', '地', '点', '在', '厦', '门', '与', '金', '门', '之', '间',\n",
    "        '的', '海', '域', '。'\n",
    "    ],\n",
    "     [\n",
    "         '这', '座', '依', '山', '傍', '水', '的', '博', '物', '馆', '由', '国', '内', '一',\n",
    "         '流', '的', '设', '计', '师', '主', '持', '设', '计', '，', '整', '个', '建', '筑',\n",
    "         '群', '精', '美', '而', '恢', '宏', '。'\n",
    "     ]],\n",
    "    truncation=True,\n",
    "    padding=True,\n",
    "    return_tensors='pt',\n",
    "    is_split_into_words=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3362a434",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at data/train/cache-534d84a68ea10ebc.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(20852,\n",
       " ['海',\n",
       "  '钓',\n",
       "  '比',\n",
       "  '赛',\n",
       "  '地',\n",
       "  '点',\n",
       "  '在',\n",
       "  '厦',\n",
       "  '门',\n",
       "  '与',\n",
       "  '金',\n",
       "  '门',\n",
       "  '之',\n",
       "  '间',\n",
       "  '的',\n",
       "  '海',\n",
       "  '域',\n",
       "  '。'],\n",
       " [0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from datasets import load_dataset, load_from_disk\n",
    "\n",
    "\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, split):\n",
    "        #names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']\n",
    "\n",
    "        #在线加载数据集\n",
    "        #dataset = load_dataset(path='peoples_daily_ner', split=split)\n",
    "\n",
    "        #离线加载数据集\n",
    "        dataset = load_from_disk(dataset_path='./data')[split]\n",
    "\n",
    "        #过滤掉太长的句子\n",
    "        def f(data):\n",
    "            return len(data['tokens']) <= 512 - 2\n",
    "\n",
    "        dataset = dataset.filter(f)\n",
    "\n",
    "        self.dataset = dataset\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        tokens = self.dataset[i]['tokens']\n",
    "        labels = self.dataset[i]['ner_tags']\n",
    "\n",
    "        return tokens, labels\n",
    "\n",
    "\n",
    "dataset = Dataset('train')\n",
    "\n",
    "tokens, labels = dataset[0]\n",
    "\n",
    "len(dataset), tokens, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e59695a4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1303\n",
      "[CLS] 马 来 西 亚 一 些 大 公 司 从 2 月 开 始 ， 展 开 了 一 场 为 期 半 年 的 [UNK] 攻 关 战 [UNK] ， 计 划 花 费 3 0 0 万 至 5 0 0 万 美 元 ， 在 全 球 范 围 做 广 告 。 [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\n",
      "tensor([7, 5, 6, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,\n",
      "        7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7])\n",
      "input_ids torch.Size([16, 88])\n",
      "token_type_ids torch.Size([16, 88])\n",
      "attention_mask torch.Size([16, 88])\n"
     ]
    }
   ],
   "source": [
    "#数据整理函数\n",
    "def collate_fn(data):\n",
    "    tokens = [i[0] for i in data]\n",
    "    labels = [i[1] for i in data]\n",
    "\n",
    "    inputs = tokenizer.batch_encode_plus(tokens,\n",
    "                                         truncation=True,\n",
    "                                         padding=True,\n",
    "                                         return_tensors='pt',\n",
    "                                         is_split_into_words=True)\n",
    "\n",
    "    lens = inputs['input_ids'].shape[1]\n",
    "\n",
    "    for i in range(len(labels)):\n",
    "        labels[i] = [7] + labels[i]\n",
    "        labels[i] += [7] * lens\n",
    "        labels[i] = labels[i][:lens]\n",
    "\n",
    "    return inputs, torch.LongTensor(labels)\n",
    "\n",
    "\n",
    "#数据加载器\n",
    "loader = torch.utils.data.DataLoader(dataset=dataset,\n",
    "                                     batch_size=16,\n",
    "                                     collate_fn=collate_fn,\n",
    "                                     shuffle=True,\n",
    "                                     drop_last=True)\n",
    "\n",
    "#查看数据样例\n",
    "for i, (inputs, labels) in enumerate(loader):\n",
    "    break\n",
    "\n",
    "print(len(loader))\n",
    "print(tokenizer.decode(inputs['input_ids'][0]))\n",
    "print(labels[0])\n",
    "\n",
    "for k, v in inputs.items():\n",
    "    print(k, v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f90b5b5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at hfl/rbt6 were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.decoder.bias', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5974.0416\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 88, 768])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoModel\n",
    "\n",
    "#加载预训练模型\n",
    "pretrained = AutoModel.from_pretrained('hfl/rbt6')\n",
    "\n",
    "#统计参数量\n",
    "print(sum(i.numel() for i in pretrained.parameters()) / 10000)\n",
    "\n",
    "#模型试算\n",
    "#[b, lens] -> [b, lens, 768]\n",
    "pretrained(**inputs).last_hidden_state.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3096c294",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 88, 8])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#定义下游模型\n",
    "class Model(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.tuneing = False\n",
    "        self.pretrained = None\n",
    "\n",
    "        self.rnn = torch.nn.GRU(768, 768,batch_first=True)\n",
    "        self.fc = torch.nn.Linear(768, 8)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        if self.tuneing:\n",
    "            out = self.pretrained(**inputs).last_hidden_state\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                out = pretrained(**inputs).last_hidden_state\n",
    "\n",
    "        out, _ = self.rnn(out)\n",
    "\n",
    "        out = self.fc(out).softmax(dim=2)\n",
    "\n",
    "        return out\n",
    "\n",
    "    def fine_tuneing(self, tuneing):\n",
    "        self.tuneing = tuneing\n",
    "        if tuneing:\n",
    "            for i in pretrained.parameters():\n",
    "                i.requires_grad = True\n",
    "\n",
    "            pretrained.train()\n",
    "            self.pretrained = pretrained\n",
    "        else:\n",
    "            for i in pretrained.parameters():\n",
    "                i.requires_grad_(False)\n",
    "\n",
    "            pretrained.eval()\n",
    "            self.pretrained = None\n",
    "\n",
    "\n",
    "model = Model()\n",
    "\n",
    "model(inputs).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "942184e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.1653,  0.9074,  1.1993, -0.0515, -0.4700,  0.3700,  0.0175, -1.0165],\n",
       "         [-0.5450,  1.7236, -0.1515, -1.9181,  0.5940, -0.5028,  1.6496,  1.7369],\n",
       "         [-0.1785, -0.5002, -0.9051,  0.2528, -0.9384, -0.4375, -1.0452,  0.6255],\n",
       "         [ 0.2369, -0.8779,  0.3852,  2.3229,  0.9584, -0.9273,  1.4566, -0.0438],\n",
       "         [ 0.0610,  0.2239,  0.1392,  0.3481,  2.3022, -0.6476, -1.1643,  0.4135],\n",
       "         [ 0.7769, -0.5040,  0.0106, -0.3306, -0.6428, -1.5164,  0.9515,  0.7806]]),\n",
       " tensor([1., 1., 1., 1., 1., 1.]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#对计算结果和label变形,并且移除pad\n",
    "def reshape_and_remove_pad(outs, labels, attention_mask):\n",
    "    #变形,便于计算loss\n",
    "    #[b, lens, 8] -> [b*lens, 8]\n",
    "    outs = outs.reshape(-1, 8)\n",
    "    #[b, lens] -> [b*lens]\n",
    "    labels = labels.reshape(-1)\n",
    "\n",
    "    #忽略对pad的计算结果\n",
    "    #[b, lens] -> [b*lens - pad]\n",
    "    select = attention_mask.reshape(-1) == 1\n",
    "    outs = outs[select]\n",
    "    labels = labels[select]\n",
    "\n",
    "    return outs, labels\n",
    "\n",
    "\n",
    "reshape_and_remove_pad(torch.randn(2, 3, 8), torch.ones(2, 3),\n",
    "                       torch.ones(2, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8dab97e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 16, 2, 16)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取正确数量和总数\n",
    "def get_correct_and_total_count(labels, outs):\n",
    "    #[b*lens, 8] -> [b*lens]\n",
    "    outs = outs.argmax(dim=1)\n",
    "    correct = (outs == labels).sum().item()\n",
    "    total = len(labels)\n",
    "\n",
    "    #计算除了0以外元素的正确率,因为0太多了,包括的话,正确率很容易虚高\n",
    "    select = labels != 0\n",
    "    outs = outs[select]\n",
    "    labels = labels[select]\n",
    "    correct_content = (outs == labels).sum().item()\n",
    "    total_content = len(labels)\n",
    "\n",
    "    return correct, total, correct_content, total_content\n",
    "\n",
    "\n",
    "get_correct_and_total_count(torch.ones(16), torch.randn(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1bd44a7c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "354.9704\n"
     ]
    }
   ],
   "source": [
    "from transformers import AdamW\n",
    "\n",
    "\n",
    "#训练\n",
    "def train(epochs):\n",
    "    lr = 2e-5 if model.tuneing else 5e-4\n",
    "\n",
    "    #训练\n",
    "    optimizer = AdamW(model.parameters(), lr=lr)\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "    model.train()\n",
    "    for epoch in range(epochs):\n",
    "        for step, (inputs, labels) in enumerate(loader):\n",
    "            #模型计算\n",
    "            #[b, lens] -> [b, lens, 8]\n",
    "            outs = model(inputs)\n",
    "\n",
    "            #对outs和label变形,并且移除pad\n",
    "            #outs -> [b, lens, 8] -> [c, 8]\n",
    "            #labels -> [b, lens] -> [c]\n",
    "            outs, labels = reshape_and_remove_pad(outs, labels,\n",
    "                                                  inputs['attention_mask'])\n",
    "\n",
    "            #梯度下降\n",
    "            loss = criterion(outs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            if step % 50 == 0:\n",
    "                counts = get_correct_and_total_count(labels, outs)\n",
    "\n",
    "                accuracy = counts[0] / counts[1]\n",
    "                accuracy_content = counts[2] / counts[3]\n",
    "\n",
    "                print(epoch, step, loss.item(), accuracy, accuracy_content)\n",
    "\n",
    "        torch.save(model, 'model/命名实体识别_中文.model')\n",
    "\n",
    "\n",
    "model.fine_tuneing(False)\n",
    "print(sum(p.numel() for p in model.parameters()) / 10000)\n",
    "#train(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bc02d6dd",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6329.012\n"
     ]
    }
   ],
   "source": [
    "model.fine_tuneing(True)\n",
    "print(sum(p.numel() for p in model.parameters()) / 10000)\n",
    "#train(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "622edfb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at data/validation/cache-80ee7b679fd38e82.arrow\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.9907604360542409 0.9553249097472925\n"
     ]
    }
   ],
   "source": [
    "#测试\n",
    "def test():\n",
    "    model_load = torch.load('model/命名实体识别_中文.model')\n",
    "    model_load.eval()\n",
    "\n",
    "    loader_test = torch.utils.data.DataLoader(dataset=Dataset('validation'),\n",
    "                                              batch_size=128,\n",
    "                                              collate_fn=collate_fn,\n",
    "                                              shuffle=True,\n",
    "                                              drop_last=True)\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "\n",
    "    correct_content = 0\n",
    "    total_content = 0\n",
    "\n",
    "    for step, (inputs, labels) in enumerate(loader_test):\n",
    "        if step == 5:\n",
    "            break\n",
    "        print(step)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            #[b, lens] -> [b, lens, 8] -> [b, lens]\n",
    "            outs = model_load(inputs)\n",
    "\n",
    "        #对outs和label变形,并且移除pad\n",
    "        #outs -> [b, lens, 8] -> [c, 8]\n",
    "        #labels -> [b, lens] -> [c]\n",
    "        outs, labels = reshape_and_remove_pad(outs, labels,\n",
    "                                              inputs['attention_mask'])\n",
    "\n",
    "        counts = get_correct_and_total_count(labels, outs)\n",
    "        correct += counts[0]\n",
    "        total += counts[1]\n",
    "        correct_content += counts[2]\n",
    "        total_content += counts[3]\n",
    "\n",
    "    print(correct / total, correct_content / total_content)\n",
    "\n",
    "\n",
    "test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "25fe8647",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at data/validation/cache-80ee7b679fd38e82.arrow\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS]胡老介绍说：菊花的品种不一样，叶子也不相同，有三岐两缺的、五岐四缺的，多到七岐五缺的，其中以五岐四缺最为常见；根据叶子的形状可以区别花朵的类型，叶裂缺刻圆钝的多为宽瓣花类，叶裂缺刻尖锐的多为细瓣花类，而叶背中肋有深色纹的以紫色花为多。[SEP]\n",
      "[CLS]7胡1····················································································································[SEP]7\n",
      "[CLS]7胡1····················································································································[SEP]7\n",
      "==========================\n",
      "[CLS]他自幼学习书法，八九岁时即闻名乡里，被誉为[UNK]神童[UNK]，少年时代被称为[UNK]东乡才子[UNK]。[SEP]\n",
      "[CLS]7··································东5乡6····[SEP]7\n",
      "[CLS]7··································东5乡6····[SEP]7\n",
      "==========================\n",
      "[CLS]周涛是以诗人的气质写散文，以『游牧』的眼光审视历史和文化，极富艺术个性。[SEP]\n",
      "[CLS]7周1涛2··································[SEP]7\n",
      "[CLS]7周1涛2··································[SEP]7\n",
      "==========================\n",
      "[CLS]皇天不负苦心人，不久，孩子放学回家老远就喊着冲进门来了：[UNK]爸[UNK][UNK][UNK]爸！[UNK][SEP]\n",
      "[CLS]7····································[SEP]7\n",
      "[CLS]7····································[SEP]7\n",
      "==========================\n",
      "[CLS]拉特纳亚克议长还向江泽民主席介绍了斯里兰卡议会和国内的情况，并转达了库马拉通加总统对他的亲切问候。[SEP]\n",
      "[CLS]7拉1特2纳2亚2克2····江1泽2民2·····斯3里4兰4卡4议4会4···········库1马2拉2通2加2··········[SEP]7\n",
      "[CLS]7拉1特2纳2亚2克2····江1泽2民2·····斯3里4兰4卡4议4会4···········库1马2拉2通2加2··········[SEP]7\n",
      "==========================\n",
      "[CLS]他说，中国共产党领导的多党合作和政治协商制度，是中国的基本政治制度，中国人民政治协商会议则是实现这个制度的组织形式。[SEP]\n",
      "[CLS]7···中3国4共4产4党4················中5国6········中3国4人4民4政4治4协4商4会4议4··············[SEP]7\n",
      "[CLS]7···中3国4共4产4党4················中5国6········中3国4人4民4·治4协4·················[SEP]7\n",
      "==========================\n",
      "[CLS]只有当时当势最大之事，只有万千人利益共存共在之事，众目所注，万念归一，其事成而社会民族喜，其事败而社会民族悲。[SEP]\n",
      "[CLS]7·······················································[SEP]7\n",
      "[CLS]7·······················································[SEP]7\n",
      "==========================\n",
      "[CLS]她的步法与腕上功夫的完美结合更是独特，看她的比赛往往使人恍如欣赏芭蕾舞表演，而忘却这是竞争激烈的赛场[UNK][UNK]本届汤尤杯赛揭幕战，她首战英国名将曼尔，仅用15分钟便以11∶0、11∶1轻松利落地为印尼队赢得一个开门红。[SEP]\n",
      "[CLS]7······················································汤1尤1·········英5国6··曼1尔2························印3尼4队4········[SEP]7\n",
      "[CLS]7······················································汤1尤2·········英5国6··曼1尔2························印3尼4队4········[SEP]7\n",
      "==========================\n",
      "[CLS]加快政府信息资源的数字化、网络化进程，建设高性能政府信息网络，提高政府工作效率和决策质量，适应快速变化的外部世界，提高政府透明度，为反腐败和廉政建设创造物质条件。[SEP]\n",
      "[CLS]7·················································································[SEP]7\n",
      "[CLS]7·················································································[SEP]7\n",
      "==========================\n",
      "[CLS]●英国副首相和墨西哥外长将访华新华社北京6月25日电外交部发言人唐国强今天在记者招待会上宣布：应国务院副总理吴邦国的邀请，大不列颠及北爱尔兰联合王国副首相约翰·普雷斯科特将于7月1日至7日对中国进行正式访问。[SEP]\n",
      "[CLS]7·英5国6····墨5西6哥6····华5新3华4社4北5京6······外3交4部4···唐1国2强2·············国3务4院4···吴1邦2国2····大5不6列6颠6及6北6爱6尔6兰6联6合6王6国6···约1翰2·2普2雷2斯2科2特2··········中5国6·······[SEP]7\n",
      "[CLS]7·英5国6····墨5西6哥6····华5新3华4社4北5京6······外3交4部4···唐1国2强2·············国3务4院4···吴1邦2国2····大5不6列6颠6·北5爱6尔6兰6联4合6王6国6···约1翰2·2普2雷2斯2科2特2··········中5国6·······[SEP]7\n",
      "==========================\n",
      "[CLS]本来会议并未邀请南平镇参加，但袁正军获悉后，火速派人赶往长沙联系。[SEP]\n",
      "[CLS]7········南5平6镇6····袁1正2军2··········长5沙6···[SEP]7\n",
      "[CLS]7········南5平6镇6····袁1正2军2··········长5沙6···[SEP]7\n",
      "==========================\n",
      "[CLS]针对平安承保的沿江、沿湖、低洼及历史受淹标的情况，办事处抽调专人对这些标的进行清点和检查，并提出整改意见及汛期预防和标的转移的具体方案，先后对8家重点单位进行了检查，以此引起各单位领导的重视和支持。[SEP]\n",
      "[CLS]7··平3安4·····················办3事4处4·······································································[SEP]7\n",
      "[CLS]7···························处4·······································································[SEP]7\n",
      "==========================\n",
      "[CLS]我国大城市普遍采用复式计次制，一些边远地区采用包月制。[SEP]\n",
      "[CLS]7···························[SEP]7\n",
      "[CLS]7···························[SEP]7\n",
      "==========================\n",
      "[CLS]据有关专家分析，印度的股市和汇市纷纷下跌的主要原因是，除了美国和日本等国家因印度进行核试验而对其采取经济制裁措施外，世界银行和亚洲开发银行等决定停止向印度提供贷款。[SEP]\n",
      "[CLS]7········印5度6···················美5国6·日5本6····印5度6··················世3界4银4行4·亚3洲4开4发4银4行4······印5度6·····[SEP]7\n",
      "[CLS]7········印5度6···················美5国6·日5本6····印5度6··················世3界4银4行4·亚3洲4开4发4银4行4······印5度6·····[SEP]7\n",
      "==========================\n",
      "[CLS]1996年初，石渠县发生特大雪灾，全县牲畜死亡率超过37％，而搞了[UNK]人草畜[UNK]配套建设的牧户牲畜死亡率平均不到10％。[SEP]\n",
      "[CLS]7·······石5渠6县6················································[SEP]7\n",
      "[CLS]7·······石5渠6县6················································[SEP]7\n",
      "==========================\n",
      "[CLS]1984年他刚到海尔的前身[UNK][UNK][UNK]青岛电冰箱总厂时，面对的是一个发不出工资、濒临倒闭的烂摊子。[SEP]\n",
      "[CLS]7········海3尔4······青3岛4电4冰4箱4总4厂4·······················[SEP]7\n",
      "[CLS]7········海3尔4······青3岛4电4冰4箱4总4厂4·······················[SEP]7\n",
      "==========================\n",
      "[CLS]本报北京6月17日讯新华社记者王黎、本报记者董洪亮报道：记者日前从教育部有关部门了解到，天津市在教育经费的投入上，多年来一直保持高于财政经常性收入增长的比例，有不少区县教育预算占地方财政一半以上；同时按政策积极组织社会力量筹措资金，在一定程度上弥补了教育经费的不足。[SEP]\n",
      "[CLS]7··北5京6······新3华4社4··王1黎2·····董1洪2亮2········教3育4部4········天5津6市6······················································································[SEP]7\n",
      "[CLS]7··北5京6······新3华4社4··王1黎2·····董1洪2亮2········教3育4部4········天5津6市6······················································································[SEP]7\n",
      "==========================\n",
      "[CLS]吴健雄教授是当代第一流的实验原子核物理学家。[SEP]\n",
      "[CLS]7吴1健2雄2···················[SEP]7\n",
      "[CLS]7吴1健2雄2···················[SEP]7\n",
      "==========================\n",
      "[CLS]1996年，为了改变海城的城市形象，市委、市政府决定对老城区进行重新规划和改造，为了解决改造过程中的一系列难题，我们一方面邀请著名的专家学者论证，一方面通过新闻媒体在全市开展了[UNK]市民建言[UNK]活动，结果收到市民的合理化建议60多条，为城市改造工程的顺利实施创造了先决条件。[SEP]\n",
      "[CLS]7··········海5城6······市3委4··················································································································[SEP]7\n",
      "[CLS]7··········海5城6······市3委4··················································································································[SEP]7\n",
      "==========================\n",
      "[CLS]然而，前方吃紧，后方紧吃，吃在成都，乐在官场。[SEP]\n",
      "[CLS]7···············成5都6······[SEP]7\n",
      "[CLS]7···············成5都6······[SEP]7\n",
      "==========================\n",
      "[CLS]1、每次飞行前，空中乘务员都要提前着装整齐在签到室里接受监督检查。[SEP]\n",
      "[CLS]7·································[SEP]7\n",
      "[CLS]7·································[SEP]7\n",
      "==========================\n",
      "[CLS]成立大会上，经济日报社社长徐心华、光明日报总编辑王晨都表示，要以组建报业集团为契机，大胆探索，勇于创新，加快改革开放步伐，为推动有中国特色的社会主义现代化报业集团的健康发展作出贡献。[SEP]\n",
      "[CLS]7······经3济4日4报4社4··徐1心2华2·光3明4日4报4···王1晨2·······································中5国6························[SEP]7\n",
      "[CLS]7······经3济4日4报4社4··徐1心2华2·光3明4日4报4···王1晨2·······································中5国6························[SEP]7\n",
      "==========================\n",
      "[CLS]我们沿着崎岖的山路来到王永祥跟前时，他正伫立在丛林中间，凝视着充满生机的一片绿色。[SEP]\n",
      "[CLS]7···········王1永2祥2···························[SEP]7\n",
      "[CLS]7···········王1永2祥2···························[SEP]7\n",
      "==========================\n",
      "[CLS]全国人大常委会法工委研究室吴高盛认为：目前我国已有了一套包括《反不正当竞争法》、《产品质量法》、《消费者权益保护法》等等在内的比较完备的市场竞争规则，但是还需要完善，还存在有法不依、执法不严的现象。[SEP]\n",
      "[CLS]7全3国4人4大4常4委4会4法4工4委4研4究4室4吴1高2盛2···················································································[SEP]7\n",
      "[CLS]7全3国4人4大4常4委4会4法4工4委4研4究4室4吴1高2盛2···················································································[SEP]7\n",
      "==========================\n",
      "[CLS]但是，对于中国汽车产业来说，丰田是强有力的竞争对手之一。[SEP]\n",
      "[CLS]7·····中5国6·······丰3田4············[SEP]7\n",
      "[CLS]7·····中5国6·······丰3田4············[SEP]7\n",
      "==========================\n",
      "[CLS]为了开辟再就业的途径，不讲清[UNK]转变观念天地宽[UNK]的道理怎么行呢？[SEP]\n",
      "[CLS]7·······························[SEP]7\n",
      "[CLS]7·······························[SEP]7\n",
      "==========================\n",
      "[CLS]为消除西方一些人在西藏问题上的偏见，卓科达先生在长期研究的基础上，撰写了有关中国宗教问题的专著。[SEP]\n",
      "[CLS]7·········西5藏6·······卓1科2达2·················中5国6········[SEP]7\n",
      "[CLS]7·········西5藏6·······卓1科2达2·················中5国6········[SEP]7\n",
      "==========================\n",
      "[CLS]经过六届人大常委会第四次、第五次会议审议，决定提请第六届全国人大第二次会议审议，会上我作了关于《中华人民共和国民族区域自治法草案》的说明。[SEP]\n",
      "[CLS]7··六3届4人4大4常4委4会4················第3六4届4全4国4人4大4················中5华6人6民6共6和6国6··············[SEP]7\n",
      "[CLS]7··六3届4人4大4常4委4会4················第3六4届4全4国4人4大4················中5华6人6民6共6和6国6··············[SEP]7\n",
      "==========================\n",
      "[CLS]不过那种乐于把诗歌的公共性乃至战斗性视为其美学要素的声音也没有停息。[SEP]\n",
      "[CLS]7··································[SEP]7\n",
      "[CLS]7··································[SEP]7\n",
      "==========================\n",
      "[CLS]不久前的一天上午，刚毕业于曼谷万颂绿叻差博学院企业管理系的少女玛裕勒和诗丽婉到吞武里一带求职未果。[SEP]\n",
      "[CLS]7·············曼3谷4万4颂4绿4叻4差4博4学4院4企4业4管4理4系4···玛1裕2勒2·诗1丽2婉2·吞5武6里6·······[SEP]7\n",
      "[CLS]7·············曼5谷6万4颂4绿4叻4差4博4学4院4企4业4管4理4系4···玛1裕2勒2·诗1丽2婉2·吞5武6里6·······[SEP]7\n",
      "==========================\n",
      "[CLS]上海工部局一两试铸银币是存世极为稀少的钱币品种。[SEP]\n",
      "[CLS]7上3海4工4部4局4···················[SEP]7\n",
      "[CLS]7上3海4工4部4局4···················[SEP]7\n",
      "==========================\n",
      "[CLS]5月23日是墨西哥全国防疫接种卫生周第一天。[SEP]\n",
      "[CLS]7······墨5西6哥6·············[SEP]7\n",
      "[CLS]7······墨5西6哥6·············[SEP]7\n",
      "==========================\n"
     ]
    }
   ],
   "source": [
    "#测试\n",
    "def predict():\n",
    "    model_load = torch.load('model/命名实体识别_中文.model')\n",
    "    model_load.eval()\n",
    "\n",
    "    loader_test = torch.utils.data.DataLoader(dataset=Dataset('validation'),\n",
    "                                              batch_size=32,\n",
    "                                              collate_fn=collate_fn,\n",
    "                                              shuffle=True,\n",
    "                                              drop_last=True)\n",
    "\n",
    "    for i, (inputs, labels) in enumerate(loader_test):\n",
    "        break\n",
    "\n",
    "    with torch.no_grad():\n",
    "        #[b, lens] -> [b, lens, 8] -> [b, lens]\n",
    "        outs = model_load(inputs).argmax(dim=2)\n",
    "\n",
    "    for i in range(32):\n",
    "        #移除pad\n",
    "        select = inputs['attention_mask'][i] == 1\n",
    "        input_id = inputs['input_ids'][i, select]\n",
    "        out = outs[i, select]\n",
    "        label = labels[i, select]\n",
    "        \n",
    "        #输出原句子\n",
    "        print(tokenizer.decode(input_id).replace(' ', ''))\n",
    "\n",
    "        #输出tag\n",
    "        for tag in [label, out]:\n",
    "            s = ''\n",
    "            for j in range(len(tag)):\n",
    "                if tag[j] == 0:\n",
    "                    s += '·'\n",
    "                    continue\n",
    "                s += tokenizer.decode(input_id[j])\n",
    "                s += str(tag[j].item())\n",
    "\n",
    "            print(s)\n",
    "        print('==========================')\n",
    "\n",
    "\n",
    "predict()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}