code/textclassification/nlp08_huggingface_transformers_albert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "version": "3.6.4",
      "file_extension": ".py",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "name": "python",
      "mimetype": "text/x-python"
    },
    "colab": {
      "name": "nlp08-huggingface-transformers-albert.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/chongzicbo/nlp-ml-dl-notes/blob/master/code/textclassification/nlp08_huggingface_transformers_albert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true,
        "id": "ER0jiM8iolTY"
      },
      "source": [
        "import torch\n",
        "from transformers import BertTokenizer,BertModel,BertConfig\n",
        "import numpy as np\n",
        "from torch.utils import data\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iACws9WVolTb"
      },
      "source": [
        "# 1.加载预训练模型"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
        "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
        "trusted": true,
        "id": "c1TC0JBWolTc"
      },
      "source": [
        "pretrained = 'voidful/albert_chinese_small'\n",
        "tokenizer = BertTokenizer.from_pretrained(pretrained)\n",
        "model=BertModel.from_pretrained(pretrained)\n",
        "config=BertConfig.from_pretrained(pretrained)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "kSFi2mlOolTf"
      },
      "source": [
        "inputtext = \"今天心情情很好啊，买了很多东西，我特别喜欢，终于有了自己喜欢的电子产品，这次总算可以好好学习了\"\n",
        "tokenized_text=tokenizer.encode(inputtext)\n",
        "input_ids=torch.tensor(tokenized_text).view(-1,len(tokenized_text))\n",
        "outputs=model(input_ids)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U2F4Gq2XolTh"
      },
      "source": [
        "输出字向量表示和句向量"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "2cxoUeJ9olTi"
      },
      "source": [
        "outputs[0].shape,outputs[1].shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jDIkcvX4olTk"
      },
      "source": [
        "# 2.搭建训练网络"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "AaNI-5WrolTl"
      },
      "source": [
        "config.hidden_size,config.embedding_size,config.max_length"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ssE4uVLzolTn"
      },
      "source": [
        "torch.nn.Dropout??"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "mqMmrmkJolTp"
      },
      "source": [
        "class AlbertClassfier(torch.nn.Module):\n",
        "    def __init__(self,bert_model,bert_config,num_class):\n",
        "        super(AlbertClassfier,self).__init__()\n",
        "        self.bert_model=bert_model\n",
        "        self.dropout=torch.nn.Dropout(0.4)\n",
        "        self.fc1=torch.nn.Linear(bert_config.hidden_size,bert_config.hidden_size)\n",
        "        self.fc2=torch.nn.Linear(bert_config.hidden_size,num_class)\n",
        "    def forward(self,token_ids):\n",
        "        bert_out=self.bert_model(token_ids)[1] #句向量 [batch_size,hidden_size]\n",
        "        bert_out=self.dropout(bert_out)\n",
        "        bert_out=self.fc1(bert_out) \n",
        "        bert_out=self.dropout(bert_out)\n",
        "        bert_out=self.fc2(bert_out) #[batch_size,num_class]\n",
        "        return bert_out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "kn1Wn5r_olTs"
      },
      "source": [
        "albertBertClassifier=AlbertClassfier(model,config,2)\n",
        "device=torch.device(\"cuda:0\") if torch.cuda.is_available() else 'cpu'\n",
        "albertBertClassifier=albertBertClassifier.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "BdgBKF9RolTu"
      },
      "source": [
        "device"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a3tnFvRAolTw"
      },
      "source": [
        "# 3.准备训练和验证数据"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "PLa0mjIKolTw"
      },
      "source": [
        "# tokenizer.encode??"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "KbQti59holTy"
      },
      "source": [
        "def get_train_test_data(pos_file_path,neg_file_path,max_length=100,test_size=0.2):\n",
        "    data=[]\n",
        "    label=[]\n",
        "    pos_df=pd.read_excel(pos_file_path,header=None)\n",
        "    pos_df.columns=['content']\n",
        "    for index, row in pos_df.iterrows():\n",
        "        row=row['content']\n",
        "        ids=tokenizer.encode(row.strip(),max_length=max_length,padding='max_length',truncation=True)\n",
        "        data.append(ids)\n",
        "        label.append(1)\n",
        "        \n",
        "    neg_df=pd.read_excel(neg_file_path,header=None)\n",
        "    neg_df.columns=['content']\n",
        "    for index, row in neg_df.iterrows():\n",
        "        row=row['content']\n",
        "        ids=tokenizer.encode(row.strip(),max_length=max_length,padding='max_length',truncation=True)\n",
        "        data.append(ids)\n",
        "        label.append(0)\n",
        "    X_train, X_test, y_train, y_test=train_test_split(data,label,test_size=test_size,shuffle=True)\n",
        "    return (X_train,y_train),(X_test,y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Pa--7FO5olT1"
      },
      "source": [
        "pos_file_path=\"../input/data01/pos.xls\"\n",
        "neg_file_path=\"../input/data01/neg.xls\"\n",
        "(X_train,y_train),(X_test,y_test)=get_train_test_data(pos_file_path,neg_file_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "rtru5JzTolT3"
      },
      "source": [
        "len(X_train),len(X_test),len(y_train),len(y_test),len(X_train[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "xS4CuRj4olT5"
      },
      "source": [
        "\" \".join([str(i) for i in X_train[0]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "wAv8xFk1olT7"
      },
      "source": [
        "tokenizer.decode(X_train[0]),y_train[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "fcINSNtvolT9"
      },
      "source": [
        "class DataGen(data.Dataset):\n",
        "    def __init__(self,data,label):\n",
        "        self.data=data\n",
        "        self.label=label\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "            \n",
        "    def __getitem__(self,index):\n",
        "        return np.array(self.data[index]),np.array(self.label[index])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "FewiPj8ColT_"
      },
      "source": [
        "train_dataset=DataGen(X_train,y_train)\n",
        "test_dataset=DataGen(X_test,y_test)\n",
        "train_dataloader=data.DataLoader(train_dataset,batch_size=256)\n",
        "test_dataloader=data.DataLoader(test_dataset,batch_size=256)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "qjrIJW47olUB"
      },
      "source": [
        "for X,y in train_dataloader:\n",
        "    print(X.shape,y.shape)\n",
        "    break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FiLloXTwolUD"
      },
      "source": [
        "# 4.定义优化器和损失函数"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "QhbOZloColUD"
      },
      "source": [
        "criterion=torch.nn.CrossEntropyLoss()\n",
        "optimizer=torch.optim.SGD(albertBertClassifier.parameters(),lr=0.01,momentum=0.9,weight_decay=1e-4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7e0-pWfJolUG"
      },
      "source": [
        "# 5.模型训练和测试"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "N567iaVrolUG"
      },
      "source": [
        "for epoch in range(50):\n",
        "    loss_sum=0.0\n",
        "    accu=0\n",
        "    albertBertClassifier.train()\n",
        "    for step,(token_ids,label) in enumerate(train_dataloader):\n",
        "        token_ids=token_ids.to(device)\n",
        "        label=label.to(device)\n",
        "        out=albertBertClassifier(token_ids)\n",
        "        loss=criterion(out,label)\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward() #反向传播\n",
        "        optimizer.step() #梯度更新\n",
        "        loss_sum+=loss.cpu().data.numpy()\n",
        "        accu+=(out.argmax(1)==label).sum().cpu().data.numpy()\n",
        "        \n",
        "    test_loss_sum=0.0\n",
        "    test_accu=0\n",
        "    albertBertClassifier.eval()\n",
        "    for step,(token_ids,label) in enumerate(test_dataloader):\n",
        "        token_ids=token_ids.to(device)\n",
        "        label=label.to(device)\n",
        "        with torch.no_grad():\n",
        "            out=albertBertClassifier(token_ids)\n",
        "            loss=criterion(out,label)\n",
        "            test_loss_sum+=loss.cpu().data.numpy()\n",
        "            test_accu+=(out.argmax(1)==label).sum().cpu().data.numpy()\n",
        "    print(\"epoch % d,train loss:%f,train acc:%f,test loss:%f,test acc:%f\"%(epoch,loss_sum/len(train_dataset),accu/len(train_dataset),test_loss_sum/len(test_dataset),test_accu/len(test_dataset)))   "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "8Mdv1fr5olUI"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}