1-train/2-evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:16.638859Z",
     "iopub.status.busy": "2022-07-26T03:51:16.638615Z",
     "iopub.status.idle": "2022-07-26T03:51:16.648562Z",
     "shell.execute_reply": "2022-07-26T03:51:16.647959Z",
     "shell.execute_reply.started": "2022-07-26T03:51:16.638802Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# cd to your workstation, if necessary\n",
    "%cd /home/aistudio/work/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:16.650065Z",
     "iopub.status.busy": "2022-07-26T03:51:16.649754Z",
     "iopub.status.idle": "2022-07-26T03:51:22.975761Z",
     "shell.execute_reply": "2022-07-26T03:51:22.973796Z",
     "shell.execute_reply.started": "2022-07-26T03:51:16.650042Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 使用pandas读取数据集\n",
    "import pandas as pd\n",
    "\n",
    "train = pd.read_table('train.csv', sep='\\t')\n",
    "test = pd.read_table('test.csv', sep='\\t')    # 测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:22.981788Z",
     "iopub.status.busy": "2022-07-26T03:51:22.981293Z",
     "iopub.status.idle": "2022-07-26T03:51:23.027218Z",
     "shell.execute_reply": "2022-07-26T03:51:23.026426Z",
     "shell.execute_reply.started": "2022-07-26T03:51:22.981754Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:23.029947Z",
     "iopub.status.busy": "2022-07-26T03:51:23.029413Z",
     "iopub.status.idle": "2022-07-26T03:51:24.822663Z",
     "shell.execute_reply": "2022-07-26T03:51:24.821524Z",
     "shell.execute_reply.started": "2022-07-26T03:51:23.029922Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 导入所需的第三方库\n",
    "import math\n",
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import collections\n",
    "from functools import partial\n",
    "import random\n",
    "import time\n",
    "import inspect\n",
    "import importlib\n",
    "from tqdm import tqdm\n",
    "import paddle\n",
    "import paddle.nn as nn\n",
    "import paddle.nn.functional as F\n",
    "from paddle.io import IterableDataset\n",
    "from paddle.utils.download import get_path_from_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:24.829148Z",
     "iopub.status.busy": "2022-07-26T03:51:24.828907Z",
     "iopub.status.idle": "2022-07-26T03:51:40.165291Z",
     "shell.execute_reply": "2022-07-26T03:51:40.164364Z",
     "shell.execute_reply.started": "2022-07-26T03:51:24.829120Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade paddlenlp==2.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:40.169172Z",
     "iopub.status.busy": "2022-07-26T03:51:40.168830Z",
     "iopub.status.idle": "2022-07-26T03:51:40.582677Z",
     "shell.execute_reply": "2022-07-26T03:51:40.581765Z",
     "shell.execute_reply.started": "2022-07-26T03:51:40.169142Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 导入paddlenlp所需的相关包\n",
    "import paddlenlp as ppnlp\n",
    "from paddlenlp.data import JiebaTokenizer, Pad, Stack, Tuple, Vocab\n",
    "from paddlenlp.datasets import MapDataset\n",
    "from paddle.dataset.common import md5file\n",
    "from paddlenlp.datasets import DatasetBuilder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:40.586729Z",
     "iopub.status.busy": "2022-07-26T03:51:40.586273Z",
     "iopub.status.idle": "2022-07-26T03:51:48.928997Z",
     "shell.execute_reply": "2022-07-26T03:51:48.927737Z",
     "shell.execute_reply.started": "2022-07-26T03:51:40.586704Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 使用roberta-wwm-ext-large模型\n",
    "# MODEL_NAME = \"roberta-wwm-ext-large\"\n",
    "# 从保存的参数中读取\n",
    "MODEL_NAME = 'test_0.83_finished'\n",
    "# 只需指定想要使用的模型名称和文本分类的类别数即可完成Fine-tune网络定义，通过在预训练模型后拼接上一个全连接网络（Full Connected）进行分类\n",
    "model = ppnlp.transformers.RobertaForSequenceClassification.from_pretrained(MODEL_NAME, num_classes=8) # 此次分类任务为8分类任务，故num_classes设置为8\n",
    "# 定义模型对应的tokenizer，tokenizer可以把原始输入文本转化成模型model可接受的输入数据格式。需注意tokenizer类要与选择的模型相对应，具体可以查看PaddleNLP相关文档\n",
    "tokenizer = ppnlp.transformers.RobertaTokenizer.from_pretrained(MODEL_NAME)\n",
    "# tokenizer = ppnlp.transformers.RobertaTokenizer.from_pretrained('checkpoint')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:48.942941Z",
     "iopub.status.busy": "2022-07-26T03:51:48.942257Z",
     "iopub.status.idle": "2022-07-26T03:51:48.947115Z",
     "shell.execute_reply": "2022-07-26T03:51:48.946442Z",
     "shell.execute_reply.started": "2022-07-26T03:51:48.942916Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 定义要进行分类的类别\n",
    "with open(MODEL_NAME + '/label_map.json') as f:\n",
    "    label_map_str = json.load(f)\n",
    "label_map = {}\n",
    "for k, v in label_map_str.items():\n",
    "    label_map[int(k)] = v\n",
    "\n",
    "label_list = list(label_map.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:48.950352Z",
     "iopub.status.busy": "2022-07-26T03:51:48.950130Z",
     "iopub.status.idle": "2022-07-26T03:51:48.956858Z",
     "shell.execute_reply": "2022-07-26T03:51:48.956149Z",
     "shell.execute_reply.started": "2022-07-26T03:51:48.950333Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 定义数据集对应文件及其文件存储格式\n",
    "class NewsData(DatasetBuilder):\n",
    "    SPLITS = {\n",
    "        'train': 'train.csv',  # 训练集\n",
    "        'dev': 'test.csv',      # 验证集\n",
    "    }\n",
    "\n",
    "    def _get_data(self, mode, **kwargs):\n",
    "        filename = self.SPLITS[mode]\n",
    "        return filename\n",
    "\n",
    "    def _read(self, filename):\n",
    "        \"\"\"读取数据\"\"\"\n",
    "        with open(filename, 'r', encoding='utf-8') as f:\n",
    "            head = None\n",
    "            for line in f:\n",
    "                data = line.strip().split(\"\\t\")    # 以'\\t'分隔各列\n",
    "                if not head:\n",
    "                    head = data\n",
    "                else:\n",
    "                    try:\n",
    "                        text_a, label = data\n",
    "                        yield {\"text_a\": text_a, \"label\": label}  # 此次设置数据的格式为：text_a,label，可以根据具体情况进行修改\n",
    "                    except:\n",
    "                        continue\n",
    "\n",
    "    def get_labels(self):\n",
    "        return label_list   # 类别标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:48.960325Z",
     "iopub.status.busy": "2022-07-26T03:51:48.959900Z",
     "iopub.status.idle": "2022-07-26T03:51:48.964393Z",
     "shell.execute_reply": "2022-07-26T03:51:48.963717Z",
     "shell.execute_reply.started": "2022-07-26T03:51:48.960302Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 定义数据集加载函数\n",
    "def load_dataset(name=None,\n",
    "                 data_files=None,\n",
    "                 splits=None,\n",
    "                 lazy=None,\n",
    "                 **kwargs):\n",
    "   \n",
    "    reader_cls = NewsData  # 加载定义的数据集格式\n",
    "    print(reader_cls)\n",
    "    if not name:\n",
    "        reader_instance = reader_cls(lazy=lazy, **kwargs)\n",
    "    else:\n",
    "        reader_instance = reader_cls(lazy=lazy, name=name, **kwargs)\n",
    "\n",
    "    datasets = reader_instance.read_datasets(data_files=data_files, splits=splits)\n",
    "    return datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:48.967635Z",
     "iopub.status.busy": "2022-07-26T03:51:48.967429Z",
     "iopub.status.idle": "2022-07-26T03:51:51.274191Z",
     "shell.execute_reply": "2022-07-26T03:51:51.273140Z",
     "shell.execute_reply.started": "2022-07-26T03:51:48.967616Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 加载训练和验证集\n",
    "train_ds, dev_ds = load_dataset(splits=[\"train\", \"dev\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.278232Z",
     "iopub.status.busy": "2022-07-26T03:51:51.277743Z",
     "iopub.status.idle": "2022-07-26T03:51:51.284958Z",
     "shell.execute_reply": "2022-07-26T03:51:51.284251Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.278206Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 定义数据加载和处理函数\n",
    "def convert_example(example, tokenizer, max_seq_length=128, is_test=False):\n",
    "    qtconcat = example[\"text_a\"]\n",
    "    encoded_inputs = tokenizer(text=qtconcat, max_seq_len=max_seq_length)  # tokenizer处理为模型可接受的格式 \n",
    "    input_ids = encoded_inputs[\"input_ids\"]\n",
    "    token_type_ids = encoded_inputs[\"token_type_ids\"]\n",
    "\n",
    "    if not is_test:\n",
    "        label = np.array([example[\"label\"]], dtype=\"int64\")\n",
    "        return input_ids, token_type_ids, label\n",
    "    else:\n",
    "        return input_ids, token_type_ids\n",
    "\n",
    "# 定义数据加载函数dataloader\n",
    "def create_dataloader(dataset,\n",
    "                      mode='train',\n",
    "                      batch_size=1,\n",
    "                      batchify_fn=None,\n",
    "                      trans_fn=None):\n",
    "    if trans_fn:\n",
    "        dataset = dataset.map(trans_fn)\n",
    "\n",
    "    shuffle = True if mode == 'train' else False\n",
    "    # 训练数据集随机打乱，测试数据集不打乱\n",
    "    if mode == 'train':\n",
    "        batch_sampler = paddle.io.DistributedBatchSampler(\n",
    "            dataset, batch_size=batch_size, shuffle=shuffle)\n",
    "    else:\n",
    "        batch_sampler = paddle.io.BatchSampler(\n",
    "            dataset, batch_size=batch_size, shuffle=shuffle)\n",
    "\n",
    "    return paddle.io.DataLoader(\n",
    "        dataset=dataset,\n",
    "        batch_sampler=batch_sampler,\n",
    "        collate_fn=batchify_fn,\n",
    "        return_list=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.288772Z",
     "iopub.status.busy": "2022-07-26T03:51:51.288209Z",
     "iopub.status.idle": "2022-07-26T03:51:51.291744Z",
     "shell.execute_reply": "2022-07-26T03:51:51.291047Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.288748Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 参数设置：\n",
    "# 批处理大小，显存如若不足的话可以适当改小该值  \n",
    "batch_size = 32\n",
    "# 文本序列最大截断长度，需要根据文本具体长度进行确定，最长不超过512。 \n",
    "max_seq_length = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.294868Z",
     "iopub.status.busy": "2022-07-26T03:51:51.294666Z",
     "iopub.status.idle": "2022-07-26T03:51:51.300222Z",
     "shell.execute_reply": "2022-07-26T03:51:51.299558Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.294850Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 将数据处理成模型可读入的数据格式\n",
    "trans_func = partial(\n",
    "    convert_example,\n",
    "    tokenizer=tokenizer,\n",
    "    max_seq_length=max_seq_length)\n",
    "\n",
    "batchify_fn = lambda samples, fn=Tuple(\n",
    "    Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_ids\n",
    "    Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_ids\n",
    "    Stack()  # labels\n",
    "): [data for data in fn(samples)]\n",
    "\n",
    "# 训练集迭代器\n",
    "train_data_loader = create_dataloader(\n",
    "    train_ds,\n",
    "    mode='train',\n",
    "    batch_size=batch_size,\n",
    "    batchify_fn=batchify_fn,\n",
    "    trans_fn=trans_func)\n",
    "\n",
    "# 验证集迭代器\n",
    "dev_data_loader = create_dataloader(\n",
    "    dev_ds,\n",
    "    mode='dev',\n",
    "    batch_size=batch_size,\n",
    "    batchify_fn=batchify_fn,\n",
    "    trans_fn=trans_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.303464Z",
     "iopub.status.busy": "2022-07-26T03:51:51.303248Z",
     "iopub.status.idle": "2022-07-26T03:51:51.306835Z",
     "shell.execute_reply": "2022-07-26T03:51:51.306159Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.303445Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "criterion = paddle.nn.loss.CrossEntropyLoss()  # 交叉熵损失函数\n",
    "metric = paddle.metric.Accuracy()              # accuracy评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.307947Z",
     "iopub.status.busy": "2022-07-26T03:51:51.307734Z",
     "iopub.status.idle": "2022-07-26T03:51:51.314920Z",
     "shell.execute_reply": "2022-07-26T03:51:51.314223Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.307927Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 定义模型训练验证评估函数\n",
    "@paddle.no_grad()\n",
    "def evaluate(model, criterion, metric, data_loader):\n",
    "    model.eval()\n",
    "    metric.reset()\n",
    "    losses = []\n",
    "    results = []\n",
    "    for batch in tqdm(data_loader):\n",
    "        input_ids, token_type_ids, labels = batch\n",
    "        logits = model(input_ids, token_type_ids)\n",
    "\n",
    "        # 同时输出结果\n",
    "        probs = F.softmax(logits, axis=1)\n",
    "        idx = paddle.argmax(probs, axis=1).numpy()\n",
    "        idx = idx.tolist()\n",
    "        pred_labels = [label_map[i] for i in idx]\n",
    "        results.extend(pred_labels)\n",
    "\n",
    "        loss = criterion(logits, labels)\n",
    "        losses.append(loss.numpy())\n",
    "        correct = metric.compute(logits, labels)\n",
    "        metric.update(correct)\n",
    "        accu = metric.accumulate()\n",
    "    print(\"eval loss: %.5f, accu: %.5f\" % (np.mean(losses), accu))  # 输出验证集上评估效果\n",
    "    model.train()\n",
    "    metric.reset()\n",
    "    return accu, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.316518Z",
     "iopub.status.busy": "2022-07-26T03:51:51.315846Z",
     "iopub.status.idle": "2022-07-26T03:51:51.321049Z",
     "shell.execute_reply": "2022-07-26T03:51:51.320389Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.316494Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # 固定随机种子便于结果的复现\n",
    "seed = 1024\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "paddle.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T03:51:51.322147Z",
     "iopub.status.busy": "2022-07-26T03:51:51.321913Z",
     "iopub.status.idle": "2022-07-26T04:23:26.191544Z",
     "shell.execute_reply": "2022-07-26T04:23:26.190657Z",
     "shell.execute_reply.started": "2022-07-26T03:51:51.322127Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "accu, results = evaluate(model, criterion, metric, dev_data_loader)\n",
    "print(accu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-26T05:36:47.882969Z",
     "iopub.status.busy": "2022-07-26T05:36:47.881923Z",
     "iopub.status.idle": "2022-07-26T05:36:49.923456Z",
     "shell.execute_reply": "2022-07-26T05:36:49.922477Z",
     "shell.execute_reply.started": "2022-07-26T05:36:47.882936Z"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "report = classification_report(test.label, results)\n",
    "print(report)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "py35-paddle1.2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}