Day30-進階篇_hw.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 自製智能中文選字系統  (2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'3.7.6 (default, Jan  8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)]'"
      ]
     },
     "metadata": {},
     "execution_count": 1
    }
   ],
   "source": [
    "import sys\n",
    "sys.version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "確認版本為 python3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 資料前處理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepocess_line(line):\n",
    "    chinese_pattern = r'[\\u4E00-\\u9FFF]+'\n",
    "    segments = re.findall(chinese_pattern, line, flags=re.UNICODE)\n",
    "    return segments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "segments = []\n",
    "with open('./wiki_zh_small.txt', encoding='utf-8') as fr:\n",
    "    for line in fr.readlines():\n",
    "        segments += prepocess_line(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 斷詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Requirement already satisfied: jieba in /Users/ycchen/.pyenv/versions/3.6.5/lib/python3.6/site-packages (0.42.1)\n"
    }
   ],
   "source": [
    "!pip install jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\zero0\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.823 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['所以', '僅用', '於', '還原', '一些', '貴重', '的', '化合', '化合物']"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "list(jieba.cut_for_search(segments[6001]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cut_segments = []\n",
    "for seg in segments:\n",
    "    # 使用結巴斷詞的 cut_for_search\n",
    "    # YOUR CODE HERE\n",
    "    cut_segments.extend(list(jieba.cut_for_search(seg)))\n",
    "    # END YOUR CODE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用斷詞的結果來作Ngram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "class Counters:\n",
    "    def __init__(self, n):\n",
    "        self.n = n\n",
    "        self.counters = [Counter() for _ in range(n + 1)]\n",
    "\n",
    "    def fit(self, segments):\n",
    "        for i in range(1, 1 + self.n):\n",
    "            for segment in segments:\n",
    "                self.counters[i] += Counter(self._skip(segment, i))\n",
    "\n",
    "        base_count = sum(dict(self.counters[1]).values())\n",
    "        self.counters[0] = Counter({'': base_count})\n",
    "\n",
    "    def __getitem__(self, k):\n",
    "        return self.counters[k]\n",
    "\n",
    "    def _skip(self, segment, n):\n",
    "        assert n > 0\n",
    "        if len(segment) < n:\n",
    "            return []\n",
    "        shift = n - 1\n",
    "        for i in range(len(segment) - shift):\n",
    "            yield segment[i:i+shift+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "counters = Counters(n=5)\n",
    "counters.fit(cut_segments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Ngram:\n",
    "    def __init__(self, n, counters):\n",
    "        assert n <= counters.n\n",
    "        self.n = n\n",
    "        self.major_counter = counters[n]\n",
    "        self.minor_counter = counters[n-1]\n",
    "\n",
    "    def predict_proba(self, prefix='', top_k=5):\n",
    "        assert len(prefix) >= self.n - 1\n",
    "\n",
    "        reference = prefix[-(self.n - 1):] if self.n > 1 else ''\n",
    "        count_referecne = self.minor_counter[reference]\n",
    "        probs = []\n",
    "        for key, count in dict(self.major_counter).items():\n",
    "            if key.startswith(reference):\n",
    "                prob = count / count_referecne\n",
    "                probs.append((prob, key[-1]))\n",
    "        sorted_probs = sorted(probs, reverse=True)\n",
    "        return sorted_probs[:top_k] if top_k > 0 else sorted_probs\n",
    "\n",
    "    def get_proba_dict(self, prefix=''):\n",
    "        return {word: prob for prob, word in self.predict_proba(prefix, top_k=-1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngrams = [Ngram(i, counters) for i in range(1, 6)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用Smoothing of Language Models來建立第二版選字系統"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChineseWordRecommenderV2:\n",
    "    def __init__(self, ngrams):\n",
    "        self.ngrams = ngrams\n",
    "    \n",
    "    def predict_proba(self, prefix='', top_k=5 , mode='back-off'):\n",
    "        interpolation_lambda = 0.99\n",
    "        proba_dicts = [ngram.get_proba_dict(prefix) for ngram in ngrams[:len(prefix)+1]]\n",
    "\n",
    "        probas = []\n",
    "        # 在此你可以選擇兩種 Smoothing of Language Models 的方法：\n",
    "        # Back-off Smoothing 或 Interpolation Smoothing\n",
    "        # 如果你選擇 Interpolation Smoothing，你可以運用已經準備好的輔助函式\n",
    "        # _get_interpolation_proba 來達到此目的\n",
    "        # YOUR CODE HERE\n",
    "        if mode == 'back-off':\n",
    "            idx = 1 if prefix else 0\n",
    "            counter = self.ngrams[idx].major_counter\n",
    "            nr_map = self._get_nr(counter)\n",
    "\n",
    "        for word in proba_dicts[0].keys():\n",
    "            if mode == 'Interpolation':\n",
    "                prob=self._get_interpolation_proba(word, proba_dicts, interpolation_lambda, None)\n",
    "            elif mode == 'back-off':\n",
    "                prob = self._get_backoff_proba(prefix, word, counter, nr_map, proba_dicts, 5, idx)\n",
    "            \n",
    "            probas.append((prob, word))\n",
    "            \n",
    "        sorted_probas = sorted(probas, reverse=True)\n",
    "        # END YOUR CODE\n",
    "        return sorted_probas[:top_k] if top_k > 0 else sorted_probas\n",
    "    \n",
    "    def _get_interpolation_proba(self, word, proba_dicts, interp_lambda, idx=None):\n",
    "        if idx is None:\n",
    "            idx = len(proba_dicts) - 1\n",
    "        if idx == 0:\n",
    "            return proba_dicts[idx].get(word, 0.)\n",
    "        return interp_lambda * proba_dicts[idx].get(word, 0.) + \\\n",
    "               (1 - interp_lambda) * self._get_interpolation_proba(word, proba_dicts, interp_lambda, idx=idx-1)\n",
    "    \n",
    "    def _get_nr(self, counter):\n",
    "        nr_map={}\n",
    "        for cnt in counter.values():\n",
    "            nr_map[cnt] = nr_map.get(cnt, 0) +1\n",
    "        return nr_map\n",
    "\n",
    "    def _get_backoff_proba(self, prefix, word, counter, nr_map, proba_dicts, k=5, idx=None):\n",
    "        prefix= prefix[-1] if prefix else ''\n",
    "        r= counter[prefix+word]\n",
    "        if r>k:\n",
    "            dr = 1\n",
    "            p_katz = dr * proba_dicts[idx][word]\n",
    "        elif (r>0) and (r <= k):\n",
    "            tmp = (k+1) * nr_map[k+1] / nr_map[1]\n",
    "            dr = ((r+1) * nr_map[r+1] / (r*nr_map[r]) -tmp) / (1-tmp)\n",
    "            p_katz = dr * proba_dicts[idx][word]\n",
    "        else:\n",
    "            nonzero_p_katz = 0\n",
    "            nonzero_ml_cnt = 0\n",
    "            for w in proba_dicts[idx].keys():\n",
    "                r = counter[prefix+word]\n",
    "                p_katz = self._get_backoff_proba(prefix, w , counter, nr_map, proba_dicts, k, idx)\n",
    "                nonzero_ml_cnt += self.ngrams[idx-1].major_counter[w]\n",
    "                nonzero_p_katz += p_katz\n",
    "\n",
    "            all_unigram_cnt = sum(self.ngrams[idx-1].major_counter.values())\n",
    "            alpha = (1- nonzero_p_katz) / (1- nonzero_ml_cnt/all_unigram_cnt)\n",
    "            c_katz = alpha * self.ngrams[idx-1].major_counter.get(word,0)\n",
    "            p_katz = c_katz/all_unigram_cnt\n",
    "\n",
    "        return p_katz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ChineseWordRecommenderV2(ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[(0.05230496453900709, '國'),\n",
       " (0.044326241134751775, '律'),\n",
       " (0.030141843971631204, '院'),\n",
       " (0.028948136090029063, '的'),\n",
       " (0.020390070921985817, '語'),\n",
       " (0.01152482269503546, '蘭'),\n",
       " (0.010638297872340425, '系'),\n",
       " (0.009135433334423064, '一'),\n",
       " (0.008169332363511292, '在'),\n",
       " (0.007978723404255319, '定')]"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "probs = model.predict_proba('法', top_k=10)\n",
    "probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Script file 'D:\\Anaconda3\\Scripts\\pip-script.py' is not present.\n",
      "Script file 'D:\\Anaconda3\\Scripts\\pip-script.py' is not present.\n"
     ]
    }
   ],
   "source": [
    "!pip install -U pip\n",
    "!pip install -q ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Label(value='')",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d64a3681f24a4281a994309f6a9d3aef"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Textarea(value='')",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b6731fed0c284dc7992bf5acb9eefcc0"
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "import ipywidgets as widgets\n",
    "\n",
    "text = widgets.Textarea()\n",
    "label = widgets.Label()\n",
    "display(label, text)\n",
    "\n",
    "def func(change):\n",
    "    probs = model.predict_proba(change.new, top_k=10)\n",
    "    label.value = ' ' + '\\t'.join([word for prob, word in probs])\n",
    "\n",
    "text.observe(func, names='value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.6 64-bit (conda)",
   "metadata": {
    "interpreter": {
     "hash": "a8f61be024eba58adef938c9aa1e29e02cb3dece83a5348b1a2dafd16a070453"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}