Chapter10-2 HMM_NLP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "773c3f73",
   "metadata": {},
   "source": [
    "# 数据集：人民日报1998年中文标注语料库\n",
    "# 目的：分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "15316566",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "64be2614",
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainParameter(fileName):\n",
    "    \"\"\"\n",
    "    根据文本训练HMM模型参数PI、A、B\n",
    "    Q: 状态集合\n",
    "    V: 观测集合\n",
    "    A: 状态转移概率矩阵\n",
    "    B: 观测概率矩阵\n",
    "    O: 观测序列\n",
    "    PI: 初始概率分布\n",
    "    \"\"\"\n",
    "    \n",
    "    # 定义一个查询字典，用于映射四种标记在数组中的对应的位置\n",
    "    # B：词语的开头 M：一个词语的中间词 E：一个词语的结果 S：非词语，单个词\n",
    "    # 如：我 非常地 爱 中国\n",
    "    #       S  BME    S  BE\n",
    "    \n",
    "    statuDict = {'B': 0, 'M': 1, 'E': 2, 'S': 3}\n",
    "    \n",
    "    # 每个字有四种状态\n",
    "    # 初始化PI数组->对应四种状态\n",
    "    PI = np.zeros(4)\n",
    "    \n",
    "    # 初始化状态转移矩阵\n",
    "    A = np.zeros((4, 4))\n",
    "    # 初始化观测概率\n",
    "    # 中文分词，ord(汉字)->对应汉字编码，用65536空间保证所有汉字能存储到\n",
    "    B = np.zeros((4, 65536))\n",
    "    \n",
    "    # 读训练文本\n",
    "    fr = open(fileName, encoding = 'utf-8')\n",
    "    \n",
    "    \n",
    "    # 在文件中，每一行为一个样本\n",
    "    for line in tqdm(fr.readlines()):\n",
    "        currentLine = line.strip().split()\n",
    "        # 词的状态标记\n",
    "        wordLabel = []\n",
    "        \n",
    "        # 对每一个词遍历\n",
    "        for i in range(len(currentLine)):\n",
    "            if len(currentLine[i]) == 1:\n",
    "                label = 'S'\n",
    "            else:\n",
    "                label = 'B' + 'M' * (len(currentLine[i]) - 2) + 'E'\n",
    "                \n",
    "            if i == 0:\n",
    "                PI[statuDict[label[0]]] += 1\n",
    "            \n",
    "            # 对于该单词，统计与状态链B\n",
    "            for j in range(len(label)):\n",
    "                B[statuDict[label[j]]][ord(currentLine[i][j])] += 1\n",
    "                \n",
    "            wordLabel.extend(label)\n",
    "            \n",
    "        for i in range(1, len(wordLabel)):\n",
    "            A[statuDict[wordLabel[i - 1]]][statuDict[wordLabel[i]]] += 1\n",
    "            \n",
    "    # 对PI求和\n",
    "    sum_ = np.sum(PI)\n",
    "    for i in range(len(PI)):\n",
    "        if PI[i] == 0:\n",
    "            PI[i] = -3.14e+100\n",
    "        else:\n",
    "            PI[i] = np.log(PI[i] / sum_)\n",
    "    \n",
    "    # 对A求概率log\n",
    "    for i in range(len(A)):\n",
    "        sum_ = np.sum(A[i])\n",
    "        for j in range(len(A[i])):\n",
    "            if A[i][j] == 0:\n",
    "                A[i][j] = -3.14e+100\n",
    "            else:\n",
    "                A[i][j] = np.log(A[i][j] / sum_)\n",
    "    \n",
    "    # 对B求概率log\n",
    "    for i in range(len(B)):\n",
    "        sum_ = np.sum(B[i])\n",
    "        for j in range(len(B[i])):\n",
    "            if B[i][j] == 0:\n",
    "                B[i][j] = -3.14e+100\n",
    "            else:\n",
    "                B[i][j] = np.log(B[i][j] / sum_)\n",
    "                \n",
    "    return PI, A, B\n",
    "\n",
    "def loadArticle(fileName):\n",
    "    \"\"\"\n",
    "    加载文章\n",
    "    \"\"\"\n",
    "    article = []\n",
    "    fr = open(fileName, encoding='utf-8')\n",
    "    for line in tqdm(fr.readlines()):\n",
    "        line = line.strip()\n",
    "        article.append(line)\n",
    "    \n",
    "    return article\n",
    "\n",
    "def participle(article, PI, A, B):\n",
    "    \"\"\"\n",
    "    分词\n",
    "    维特比算法\n",
    "    \"\"\"\n",
    "    \n",
    "    # 初始化文章列表\n",
    "    retAritcle = []\n",
    "    \n",
    "    for line in article:\n",
    "        #\n",
    "        delta = [[0 for i in range(4)] for i in range(len(line))]\n",
    "        \n",
    "        for i in range(4):\n",
    "            delta[0][i] = PI[i] + B[i][ord(line[0])]\n",
    "            \n",
    "        psi = [[0 for i in range(4)] for i in range(len(line))]\n",
    "        \n",
    "        # 递推\n",
    "        for t in range(1, len(line)):\n",
    "            for i in range(4):\n",
    "                tempDelta = [0] * 4\n",
    "                for j in range(4):\n",
    "                    tempDelta[j] = delta[t - 1][j] + A[j][i]\n",
    "                    \n",
    "                maxDelta = max(tempDelta)\n",
    "                maxDeltaIndex = tempDelta.index(maxDelta)\n",
    "                \n",
    "                delta[t][i] = maxDelta + B[i][ord(line[t])]\n",
    "                \n",
    "                psi[t][i] = maxDeltaIndex\n",
    "                \n",
    "        \n",
    "        # 状态链列表\n",
    "        sequence = []\n",
    "        # 第三步：终止\n",
    "        i_opt = delta[len(line) - 1].index(max(delta[len(line) - 1]))\n",
    "        \n",
    "        sequence.append(i_opt)\n",
    "        \n",
    "        # 第四步：最优路径回溯\n",
    "        for t in range(len(line) - 1, 0, -1):\n",
    "            i_opt = psi[t][i_opt]\n",
    "            sequence.append(i_opt)\n",
    "        sequence.reverse()\n",
    "        \n",
    "        # 开始分词\n",
    "        curLine = ''\n",
    "        # 遍历该行每一个词\n",
    "        for i in range(len(line)):\n",
    "            curLine += line[i]\n",
    "            if (sequence[i] == 3 or sequence[i] == 2) and i != (len(line) - 1):\n",
    "                curLine += '|'\n",
    "                \n",
    "        retAritcle.append(curLine)\n",
    "        \n",
    "    return retAritcle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "541a0272",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 19056/19056 [00:02<00:00, 7225.67it/s]\n",
      "100%|██████████████████████████████████████████| 3/3 [00:00<00:00, 53544.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====== 打印原文 ======\n",
      "深圳有个打工者阅览室\n",
      "去年１２月，我在广东深圳市出差，听说南山区工商分局为打工者建了个免费图书阅览室，这件新鲜事引起了我的兴趣。\n",
      "１２月１８日下午，我来到了这个阅览室。阅览室位于桂庙，临南油大道，是一间轻体房，面积约有４０平方米，内部装修得整洁干净，四周的书架上摆满了书，并按政治、哲学、法律法规、文化教育、经济、科技、艺术、中国文学、外国文学等分类，屋中央有两排书架，上面也摆满了图书和杂志。一些打工青年或站或蹲，认真地阅读，不时有人到借阅台前办理借书或还书手续。南山区在深圳市西边，地处城乡结合部，外来打工者较多。去年２月，南山区工商分局局长王安全发现分局对面的公园里常有不少打工者业余时间闲逛，有时还滋扰生事。为了给这些打工者提供一个充实自己的场所，他提议由全分局工作人员捐款，兴建一个免费阅览室。领导带头，群众响应，大家捐款１．４万元，购买了近千册图书。３月６日，建在南头繁华的南新路和金鸡路交叉口的阅览室开放了。从此，这里每天都吸引了众多借书、看书的人们，其中不仅有打工者，还有机关干部、公司职员和个体户。到了夏天，由于阅览室所在地被工程征用，南山区工商分局便把阅览室迁到了桂庙。阅览室的管理人员是两名青年，男的叫张攀，女的叫赵阳。张攀自己就是湖北来的打工者，听说南山区工商分局办免费阅览室，便主动应聘来服务。阅览室每天从早９时开到晚１０时，夜里张攀就住在这里。他谈起阅览室里的图书，翻着一本本的借阅名册，如数家珍，对图书和工作的挚爱之情溢于言表。我在这里碰到南山区华英大厦一位叫聂煜的女青年，她说她也是个打工者，由于春节探家回来后就要去市内工作，很留恋这里的这个免费阅览室，想抓紧时间多看些书，她还把自己买的几本杂志捐给了阅览室。在阅览室的捐书登记簿上，记录着这样的数字：工商系统内部捐书３５５０册，社会各界捐书２５０册。我在阅览室读到了这样几封感谢信：深圳瑞兴光学厂的王志明写道：“我们这些年轻人远离了家乡，来到繁华紧张的都市打工，辛劳之余，能有机会看书读报，感到特别充实。”深圳文光灯泡厂的江虹说：“南山区工商分局的干部职工捐款、捐书，给我们打工者提供良好的学习环境，鼓励我们求知上进，真是办了一件大好事，他们是我们打工者的知音。”（本报记者罗华）\n",
      "\n",
      "====== 分词后 ======\n",
      "深圳|有个|打|工者|阅览室\n",
      "去年|１２月|，|我|在|广东|深圳|市出|差|，|听|说|南山区|工商|分局|为|打|工者|建了|个|免费|图书|阅览室|，|这件|新|鲜事|引起|了|我|的|兴趣|。\n",
      "１２月|１８日|下午|，|我来|到|了|这个|阅览室|。|阅览室|位|于|桂庙|，|临南油|大道|，|是|一间|轻|体房|，|面积|约|有４０平|方米|，|内|部装|修得|整洁|干净|，|四|周|的|书架|上|摆满|了|书|，|并|按|政治|、|哲学|、|法律|法规|、|文化|教育|、|经济|、|科技|、|艺术|、|中国|文学|、|外国|文学|等|分类|，|屋|中央|有|两排|书架|，|上面|也|摆满|了|图书|和|杂志|。|一些|打工|青年|或站|或|蹲|，|认真|地阅|读|，|不时|有|人到|借阅|台前|办理|借书|或|还书|手续|。|南山区|在|深圳|市西边|，|地处|城乡|结合部|，|外来|打|工者|较|多|。|去年|２月|，|南山区|工商|分局|局长|王|安全|发现|分局|对面|的|公园|里|常有|不少|打|工者|业余|时间|闲逛|，|有时|还|滋扰|生事|。|为|了|给|这些|打|工者|提供|一个|充实|自己|的|场|所|，|他|提议|由|全分局|工作|人员|捐款|，|兴建|一个|免费|阅览室|。|领导|带头|，|群众|响应|，|大家|捐款|１．４万|元|，|购买|了|近|千册|图书|。|３月|６日|，|建在|南头|繁华|的|南|新路|和|金鸡路|交叉口|的|阅览室|开放|了|。|从|此|，|这里|每天|都|吸引|了|众多|借书|、|看书|的|人们|，|其中|不仅|有|打|工者|，|还|有|机关|干部|、|公司|职员|和|个|体户|。|到|了|夏天|，|由于|阅览室|所|在|地|被|工程|征用|，|南山区|工商|分局|便|把|阅览室|迁到|了|桂庙|。|阅览室|的|管理|人员|是|两|名|青年|，|男|的|叫|张|攀|，|女|的|叫|赵阳|。|张|攀|自己|就|是|湖北来|的|打|工者|，|听|说|南山区|工商|分局|办|免费|阅览室|，|便|主动|应|聘来|服务|。|阅览室|每天|从|早９时|开到|晚１０时|，|夜里|张|攀|就|住|在|这里|。|他谈|起|阅览室|里|的|图书|，|翻着|一|本本|的|借阅|名册|，|如数|家珍|，|对|图书|和|工作|的|挚爱|之|情溢|于|言表|。|我|在|这里|碰到|南山区|华英|大厦|一位|叫|聂|煜|的|女|青年|，|她|说|她|也|是|个|打|工者|，|由于|春节|探家|回来|后|就|要|去市|内|工作|，|很|留恋|这里|的|这个|免费|阅览室|，|想|抓紧|时间|多|看些|书|，|她|还|把|自己|买|的|几本|杂志|捐给|了|阅览室|。|在|阅览室|的|捐书|登|记簿|上|，|记录|着|这样|的|数字|：|工商|系统|内部|捐书３５５０册|，|社会|各界|捐书２５０册|。|我|在|阅览室|读到|了|这样|几|封感|谢信|：|深圳|瑞兴|光学厂|的|王|志明|写道|：|“|我们|这些|年|轻人|远离|了|家乡|，|来|到|繁华紧|张|的|都|市|打工|，|辛劳|之|余|，|能|有|机会|看书|读报|，|感到|特别|充实|。|”|深圳|文光|灯|泡厂|的|江虹|说|：|“|南山区|工商|分局|的|干部|职工|捐款|、|捐书|，|给|我们|打|工者|提供|良好|的|学习|环境|，|鼓励|我们|求知|上进|，|真是|办|了|一件|大好|事|，|他们|是|我们|打|工者|的|知音|。|”|（|本报|记者|罗华|）\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# train hmm parmeter\n",
    "PI, A, B = trainParameter('./hmm_data/HMMTrainSet.txt')\n",
    "# test\n",
    "article = loadArticle('./hmm_data/testArtical.txt')\n",
    "\n",
    "# 原文\n",
    "print(\"====== 打印原文 ======\")\n",
    "for line in article:\n",
    "    print(line)\n",
    "    \n",
    "# 分词\n",
    "partiArticle = participle(article, PI, A, B)\n",
    "\n",
    "# 分词后结果\n",
    "print(\"\\n====== 分词后 ======\")\n",
    "for line in partiArticle:\n",
    "    print(line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91612199",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8e46174",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}