fastai_NLP/ULMFiT_fwd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "# 多行输出\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\" "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 精调通用语言模型用于文本分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.core import *\n",
    "import bz2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 基于中文维基百科训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64\n",
    "lang = 'zh'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/lab/.fastai/data')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Config.data_path()\n",
    "data_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 提取TXT文件放在 docs 文件夹下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = f'{lang}wiki'\n",
    "path = data_path/name\n",
    "path.mkdir(exist_ok=True, parents=True)\n",
    "dest = path/'docs'\n",
    "lm_fns = [f'{lang}_wt', f'{lang}_wt_vocab']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 下载的文件为 xml 格式\n",
    "- 大概有 46480 词目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "xml_fn = f\"{lang}wiki-latest-pages-articles.xml\"\n",
    "zip_fn = f\"{xml_fn}.bz2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def bunzip(fn:PathOrStr):\n",
    "#     \"bunzip `fn`, raising exception if output already exists\"\n",
    "#     fn = Path(fn)\n",
    "#     assert fn.exists(), f\"{fn} doesn't exist\"\n",
    "#     out_fn = fn.with_suffix('')\n",
    "#     assert not out_fn.exists(), f\"{out_fn} already exists\"\n",
    "#     with bz2.BZ2File(fn, 'rb') as src, out_fn.open('wb') as dst:\n",
    "#         for d in iter(lambda: src.read(1024*1024), b''): dst.write(d)\n",
    "\n",
    "# @contextmanager\n",
    "# def working_directory(path:PathOrStr):\n",
    "#     \"Change working directory to `path` and return to previous on exit.\"\n",
    "#     prev_cwd = Path.cwd()\n",
    "#     os.chdir(path)\n",
    "#     try: yield\n",
    "#     finally: os.chdir(prev_cwd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 提取信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_wiki():\n",
    "    if not (path/xml_fn).exists():\n",
    "        print(\"downloading...\")\n",
    "        download_url(f'https://dumps.wikimedia.org/{name}/latest/{zip_fn}', path/zip_fn)\n",
    "        print(\"unzipping...\")\n",
    "        bunzip(path/zip_fn)\n",
    "        \n",
    "    with working_directory(path):\n",
    "        if not (path/'wikiextractor').exists(): os.system('git clone https://github.com/attardi/wikiextractor.git')\n",
    "        print(\"extracting...\")\n",
    "        os.system(\"python wikiextractor/WikiExtractor.py --processes 4 --no_templates \" +\n",
    "            f\"--min_text_length 1800 --filter_disambig_pages --log_file log -b 100G -q {xml_fn}\")\n",
    "    shutil.move(str(path/'text/AA/wiki_00'), str(path/name))\n",
    "    shutil.rmtree(path/'text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not (path/name).exists(): get_wiki()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/lab/.fastai/data/zhwiki/log'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/train.csv'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zhwiki'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zh_textlist_class'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/models'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/docs'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/val.csv'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zh_databunch'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zhwiki-latest-pages-articles.xml.bz2'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/wikiextractor')]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<doc id=\"13\" url=\"https://zh.wikipedia.org/wiki?curid=13\" title=\"数学\">\n",
      "数学\n",
      "\n",
      "数学是利用符号语言研究數量、结构、变化以及空间等概念的一門学科，从某种角度看屬於形式科學的一種。數學透過抽象化和邏輯推理的使用，由計數、計算、量度和對物體形狀及運動的觀察而產生。數學家們拓展這些概念，為了公式化新的猜想以及從選定的公理及定義中建立起嚴謹推導出的定理。\n",
      "\n",
      "基礎數學的知識與運用總是個人與團體生活中不可或缺的一環。對數學基本概念的完善，早在古埃及、美索不達米亞及古印度內的古代數學文本便可觀見，而在古希臘那裡有更為嚴謹的處理。從那時開始，數學的發展便持續不斷地小幅進展，至16世紀的文藝復興時期，因为新的科學發現和數學革新兩者的交互，致使數學的加速发展，直至今日。数学并成为許多國家及地區的教育範疇中的一部分。\n",
      "\n",
      "今日，數學使用在不同的領域中，包括科學、工程、醫學、經濟學和金融學等。數學對這些領域的應用通常被稱為應用數學，有時亦會激起新的數學發現，並導致全新學科的發展，例如物理学的实质性发展中建立的某些理论激发数学家对于某些问题的不同角度的思考。數學家也研究純數學，就是數學本身的实质性內容，而不以任何實際應用為目標。雖然許多研究以純數學開始，但其过程中也發現許多應用之处。\n",
      "\n",
      "西方语言中“數學”（）一詞源自於古希臘語的（），其有“學習”、“學問”、“科學”，以及另外還有個較狹義且技術性的意思－「數學研究」，即使在其語源內。其形容詞（），意思為\"和學習有關的\"或\"用功的\"，亦會被用來指\"數學的\"。其在英语中表面上的複數形式，及在法语中的表面複數形式'，可溯至拉丁文的中性複數'，由西塞罗譯自希臘文複數（），此一希臘語被亚里士多德拿來指「萬物皆數」的概念。\n",
      "\n",
      "汉字表示的「數學」一詞大約产生于中国宋元時期。多指象數之學，但有時也含有今天上的數學意義，例如，秦九韶的《數學九章》（《永樂大典》記，即《數書九章》也被宋代周密所著的《癸辛雜識》記爲《數學大略》）、《數學通軌》（明代柯尚遷著）、《数学钥》（清代杜知耕著）、《數學拾遺》（清代丁取忠撰）。直到1939年，經過中國數學名詞審查委員會研究“算學”與“數學”兩詞的使用狀況後，確認以“數學”表示今天意義上的數學含義。\n",
      "\n",
      "數學有着久遠的歷史。它被認為起源於人類早期的生產活動：中國古代的六艺之一就有「數」，數學一詞在西方有希腊语詞源（mathematikós），意思是“学问的基础”，源于（máthema，“科学，知识，学问”）。\n",
      "\n",
      "史前的人類就已嘗試用自然的法則來衡量物質的多少、時間的長短等抽象的數量關係，比如时间单位有日、季節和年等。算術（加減乘除）也自然而然地產生了。古代的石碑及泥版亦證實了當時已有幾何的知識。\n",
      "\n",
      "更進一步則需要寫作或其他可記錄數字的系統，如符木或於印加帝國內用來儲存數據的奇普。歷史上曾有過許多不同的記數系統。\n",
      "\n",
      "在最初有歷史記錄的時候，數學內的主要原理是為了做稅務和貿易等相關計算，為了解數字間的關係，為了測量土地，以及為了預測天文事件而形成的。这些需要可以简单地被概括为数学对數量、结构、空间及时间方面的研究。\n"
     ]
    }
   ],
   "source": [
    "!head -n20 {path}/zhwiki"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 分割为 TXT 文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_wiki():\n",
    "    dest.mkdir(exist_ok=True, parents=True)\n",
    "    title_re = re.compile(rf'<doc id=\"\\d+\" url=\"https://{lang}.wikipedia.org/wiki\\?curid=\\d+\" title=\"([^\"]+)\">')\n",
    "    lines = (path/name).open()\n",
    "    f=None\n",
    "\n",
    "    for i,l in enumerate(lines):\n",
    "#         if i%100000 == 0: print(i)\n",
    "        if l.startswith('<doc'):\n",
    "            title = title_re.findall(l)[0].replace('/','_')\n",
    "            if f: f.close()\n",
    "            f = (dest/f'{title}.txt').open('w')\n",
    "        else: f.write(l)\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not dest.exists(): split_wiki()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/lab/.fastai/data/zhwiki/docs/Super Junior.txt'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/docs/三九政变.txt'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/docs/1941年德克萨斯飓风.txt'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/docs/中国人民解放军浙江省舟山警备区.txt'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/docs/傑克·切斯伯.txt')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dest.ls()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Super Junior\n",
      "\n",
      "Super Junior（），簡稱為SJ 或SuJu（），是韓國SM娱樂於2005年推出的流行音樂男子組合，出道初期由12名成員組成，成員包括：利特、希澈、韓庚、藝聲、強仁、神童、晟敏、銀赫、東海、始源、厲旭及起範。2006年起，圭賢加入Super Junior成為第13位成員。2015年11月6日，在Super Junior出道十周年之際，宣布設立獨立工作室廠牌《Label SJ》，專屬負責其專輯製作、經紀管理等方面的事業。\n",
      "\n",
      "2009年至2012年，Super Junior連續四年獲得韓國唱片銷量總冠軍。另外，Super Junior獲得韓國金唱片獎三屆唱片大賞、五屆唱片本賞、七次人氣獎，成為在韓國金唱片頒獎典禮中取得最多獎項的歌手以及第一個蟬聯金唱片大賞的團體。2011年，Super Junior在Mnet亞洲音樂大獎獲得最佳男子團體、年度專輯大獎，2012年獲得全球最佳男子團體並蟬聯年度專輯大獎，同時亦成為在該音樂頒獎禮中獲得最多獎項的歌手其中之一。除此之外，他們在2011年獲得韓國首爾歌謠大賞的大賞及本賞，成為該年度韓國三大頒獎典禮的大賞得主，奠定其在樂壇不可撼動的地位。\n",
      "\n",
      "由於組合人數眾多，成員亦各有所長，為了活動不受限制，涉足不同的音樂類型，自出道以來一共建立了五個子團體。他們在歌手、演員、主持人、DJ及模特等多方面全方位發展，成員們演藝活動經驗豐富，活躍於演藝圈各個領域之中，發展優秀且獲獎無數。迄今為止，Super Junior已發行的9張韓語正規專輯及1張日語正規專輯均取得卓越成績。Super Junior作為第一隊將K-pop推向世界的組合，一直被傳媒冠以「韓流帝王」之稱號。\n",
      "\n",
      "Super Junior的官方歌迷名稱是E.L.F.（韓語：엘프），意思是Ever Lasting Friends（永遠的朋友），2006年6月2日正式公開，由隊長利特命名。應援色是寶藍色(Sapphire Blue)。\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!head {dest}/'Super Junior.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 创建数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- token和数值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = (TextList.from_folder(dest, processor=[OpenFileProcessor(), SPProcessor()])\n",
    "#             .split_by_rand_pct(0.1, seed=42)\n",
    "#             .label_for_lm()           \n",
    "#             .databunch(bs=bs, num_workers=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(data.vocab.itos),len(data.train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data.save(path/f'{lang}_databunch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/lab/.fastai/data/zhwiki/log'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/train.csv'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zhwiki'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zh_textlist_class'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/models'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/docs'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/val.csv'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zh_databunch'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/zhwiki-latest-pages-articles.xml.bz2'),\n",
       " PosixPath('/home/lab/.fastai/data/zhwiki/wikiextractor')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_data(path, f'{lang}_databunch', bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>。 广州市 广播电视 台 是 该公司 的 实际 控制 单位 。 ▁在 广州 , 该公司 的 长期 竞争 对手 是 广东 有 线 ( 省 有 线 ), 至 2011 年 因 广 电 总局 要求 “ 一 省 一 网 ” 的政策 , 并入 广东 有 线 成为 其 子公司 , 但 在香港 及其他 境外 频道 插 播 广告 方面 仍然是 两 网 各自 分开 , 所有 境外 频道</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>只有 41 人 。 1917 年 十月 革命 后 , 有 7000 名 俄国 难民 蜂 拥 抵 沪 , 其中包括 前 贵族 。 1931 年 , 日本 占领 东北 后 , 又有 大批 俄 侨 从 哈尔滨 转移到 上海 。 30 年代 上海 俄 侨 总数 为 1.5 万 到 2 万人 。 ▁ 1934 年 , 法租界 有 俄 侨 8 260 人 。 ( 公共 租界 1935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>兩年 , 並 於同年 推出 第二 張 翻唱 碟 《 e vol ve 》 , 再度 問 鼎 香港 唱片 商 會 銷量 榜 冠軍 3 週 。 ▁陳 慧 嫻 生於 香港 , 籍 貫 廣東 番 禺 , 是 家中 長 女 , 有一個 小 一 歲 的妹妹 和 一個 小 十 歲 的弟弟 ( 其 弟 是 劉 浩 龍 的 小學 同學 )。 父親 陳 澄</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>拍 。 但 这些 影片 都 称 不 上 是 恐怖 片 , 只 不过 是以 旧 时代 为 背景 的 时代 剧 而已 。 观众 看 过 这些 影片 后 还 对 鬼 会产生 怜 悯 之心 。1950 年代末 和 1960 年代初 , 韩国 从 有 “ 吸血鬼 大 本 营 ” 之 称 的 英国 海 默 电影 公司 和美国 环球 电影 公司 引进 了 一批 让 观众</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-{ 阿 迪 达 斯 | zh - hk : ad ida s ; zh - tw : 爱 迪斯 ; zh - cn : 阿 迪 达 斯 }- 有可能 奪得 「 鳥 巢 」 的 冠名 權 , 代價 是 每年 7000 萬元 人民幣 。 相當於 每 回 $ 13 44 。 ▁2009 年 4 月 2 日 , 國際 巨星 成 龍 宣佈 於 5 月 1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['第', '水', ')。', '可以', '外', '因', '天', '南', '自', '成']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.vocab.itos[120: 130]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 模型配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'emb_sz': 400,\n",
       " 'n_hid': 1152,\n",
       " 'n_layers': 3,\n",
       " 'pad_token': 1,\n",
       " 'qrnn': False,\n",
       " 'bidir': False,\n",
       " 'output_p': 0.1,\n",
       " 'hidden_p': 0.15,\n",
       " 'input_p': 0.25,\n",
       " 'embed_p': 0.02,\n",
       " 'weight_p': 0.2,\n",
       " 'tie_weights': True,\n",
       " 'out_bias': True}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "awd_lstm_lm_config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 不使用预训练权重，我们从头训练，使用中文维基百科"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data, AWD_LSTM, drop_mult=0.5, pretrained=False).to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VdW5//HPkwkIQ4CQIGEWKS2ioEQKeq9DrRb91bHan9SBWivVals73HvtHXp729vWqq3t/WkHHGmrONR6HetQJ9SqEFQggMqoIFMIJBASMp3n98fZBw7xJIScnDHf9+t1Xmfvtdfa58kR82TvtfZa5u6IiIh0VU6qAxARkcymRCIiInFRIhERkbgokYiISFyUSEREJC5KJCIiEhclEhERiYsSiYiIxEWJRERE4pKX6gCSYciQIT5mzJhUhyEiklEWL1683d1LDlavRySSMWPGUFFRkeowREQyipl90Jl6Cbu1ZWZ3mdk2M6uMKrvQzJabWcjMyjtou97MlpnZO2ZWEVU+2MyeM7NVwfugRMUvIiKdk8g+knuAmW3KKoHzgQWdaH+Ku09x9+iEcz3wvLuPB54P9kVEJIUSlkjcfQGwo03ZSnd/L47TngPMC7bnAefGcS4REekG6Tpqy4FnzWyxmc2JKh/q7psBgvfS9k5gZnPMrMLMKqqqqhIcrohIz5WuieQEdz8WOAO4xsxOPNQTuPtcdy939/KSkoMOOhARkS5Ky0Ti7puC923AI8C04NBWMxsGELxvS02EIiISkXaJxMz6mln/yDZwOuFOeoDHgNnB9mzg0eRHKCIi0RI5/Hc+8Dowwcw2mtkVZnaemW0EZgBPmtkzQd0yM3sqaDoUeNXMlgALgSfd/eng2A3AaWa2Cjgt2BcRkTa21O7l5mfeY21VXcI/K2EPJLr7rHYOPRKj7ibgzGB7LTC5nXNWA6d2V4wiItnqg+o93PriamaMK+bwkn4J/ay0u7UlIiLxq2loBqCoT37CP0uJREQkC9XWhxPJwEIlEhER6YKahiYABhYWJPyzlEhERLJQTX0zeTlG34LchH+WEomISBaqaWhmYGE+Zpbwz1IiERHJQrX1zUnpaAclEhGRrFTT0JSU/hFQIhERyUo19c0M1BWJiIh0VU19M0VJGPoLSiQiIlmptqGZgX10a0tERLqguTVEXWNLUh5GBCUSEZGsU9uQvKfaQYlERCTr1NQnb54tUCIREck6tUmcHgWUSEREsk7kikTDf0VEpEtqkjjzLyR2hcS7zGybmVVGlV1oZsvNLGRm5e20G2lmL5rZyqDut6KO/dDMPjKzd4LXmYmKX0QkU+2sD25tZcHw33uAmW3KKoHzgQUdtGsBvuvunwKmA9eY2cSo47e4+5Tg9VTsU4iI9Fy1Dc3kGPTvnbBFcA+QyKV2F5jZmDZlK4EOZ6N0983A5mB7t5mtBIYDKxIVq4hINqkJJmzMyUn8zL+Q5n0kQSI6BngzqvhaM1sa3Dob1EHbOWZWYWYVVVVVCY5URCR9hKeQT85tLUjjRGJm/YCHgevcfVdQ/FtgHDCF8FXLL9pr7+5z3b3c3ctLSkoSHq+ISLqoqW9K2jMkkKaJxMzyCSeRe939L5Fyd9/q7q3uHgJuB6alKkYRkXRVGyxqlSxpl0gs3IFyJ7DS3X/Z5tiwqN3zCHfei4hIlGROIQ+JHf47H3gdmGBmG83sCjM7z8w2AjOAJ83smaBumZlFRmCdAFwKfCbGMN8bzWyZmS0FTgG+naj4RUQyVU198ha1gsSO2prVzqFHYtTdBJwZbL8KxBxq4O6XdluAIiJZqDXk7Nrboj4SERHpml1JnvkXlEhERLJKjRKJiIjEoybJ06OAEomISFaJXJEka712UCIREckqtUmeQh6USEREssq+W1uaIkVERLoicmtrQJJm/gUlEhGRrFJT30z/3nnk5Sbv17sSiYhIFkn2PFugRCIiklVq6puSOvQXlEhERLLKznpdkYiISBxqG5qTOs8WKJGIiGSV8My/SiQiItIFoZCHO9vVRyIiIl2xu7GFkCd3wkZQIhERyRr7pkdJ4lPtkNgVEu8ys21mVhlVdqGZLTezkJmVd9B2ppm9Z2arzez6qPLBZvacma0K3gclKn4RkUxT0xCZ+Td7rkjuAWa2KasEzgcWtNfIzHKB24AzgInALDObGBy+Hnje3ccDzwf7IiJC+Kl2yKJbW+6+ANjRpmylu793kKbTgNXuvtbdm4D7gXOCY+cA84LtecC53RiyiEhGS8WiVpCefSTDgQ1R+xuDMoCh7r4ZIHgvbe8kZjbHzCrMrKKqqiphwYqIpIvaYObfIo3awmKU+aGexN3nunu5u5eXlJR0Q1giIuktcmtLDySGr0BGRu2PADYF21vNbBhA8L4tybGJiKStmoZm+hbkUpCX3F/t6ZhIFgHjzWysmRUAFwGPBcceA2YH27OBR1MQn4hIWqqpb0760F9I7PDf+cDrwAQz22hmV5jZeWa2EZgBPGlmzwR1y8zsKQB3bwGuBZ4BVgIPuvvy4LQ3AKeZ2SrgtGBfRESA2oampN/WAkjYElruPqudQ4/EqLsJODNq/yngqRj1qoFTuytGEZFsUpOCmX8hPW9tiYhIF9SkYFErUCIREckaNfXNSR/6C0okIiJZwd2pbUj+FPKgRCIikhXqGltobvWkz7MFSiQiIlmhui78VPuQfr2S/tlKJCIiWWB7XSMAQ/orkYiISBdsD65Iivuqs11ERLqgek9wRaJbWyIi0hWRPpLBuiIREZGu2F7XSFGf/KRP2AhKJCIiWaG6ronifsm/GgElEhGRrLC9rpEhfZPfPwJKJCIiWaF6j65IREQkDtvrGlMyYguUSEREMl5za4ia+mZdkYiISNfs3BM8jJhtVyRmdpeZbTOzyqiywWb2nJmtCt4HxWg3wczeiXrtMrPrgmM/NLOPoo6d2ba9iEhPE3mqfUgKniGBxF6R3APMbFN2PfC8u48Hng/2D+Du77n7FHefAkwF6jlwVcVbIseDlRRFRHq0fU+1p2CeLUhgInH3BcCONsXnAPOC7XnAuQc5zanAGnf/oJvDExHJGpEJG1MxzxYkv49kqLtvBgjeSw9S/yJgfpuya81saXDr7GO3xiLMbI6ZVZhZRVVVVXxRi4ikscj0KFnXRxIvMysAzgYeiir+LTAOmAJsBn7RXnt3n+vu5e5eXlJSktBYRURSaXtdEwW5OQzonZeSz092ItlqZsMAgvdtHdQ9A3jL3bdGCtx9q7u3unsIuB2YltBoRUQyQHVdI8X9CjCzlHx+shPJY8DsYHs28GgHdWfR5rZWJAkFzgMqERHp4bYHiSRVEjn8dz7wOjDBzDaa2RXADcBpZrYKOC3Yx8zKzOypqLaFwfG/tDntjWa2zMyWAqcA305U/CIimaJ6TxPFKZpnCyBhN9TcfVY7h06NUXcTcGbUfj1QHKPepd0WoIhIlqiua+KI0n4p+/y07WwXEZGDc3e21zVSkqIRW6BEIiKS0eoaW2hsCWVnH4mIiCTevmdIUthHokQiIpLBItOj6IpERES6ZN+EjeojERGRrojMs6VEIiIiXRLpIxmcogkbQYlERCSjVdc1MqB3HgV5qft1rkQiIpLBtu9pStk6JBFKJCIiGay6rpEhKRz6C0okIiIZbXtdU0qH/oISiYhIRqtO8cy/oEQiIpKxWlpD7KxvTunQX1AiERHJWDvqU7vEbkSnEomZjTOzXsH2yWb2TTMbmNjQRESkI9t3B0+1p/AZEuj8FcnDQKuZHQHcCYwF7uuogZndZWbbzKwyqmywmT1nZquC90HttF0fLGD1jplVHGp7EZGeYP88WxlwRQKE3L2F8PK2v3L3bwPDDtLmHmBmm7LrgefdfTzwfLDfnlPcfYq7l3exvYhIVqveN89WZlyRNJvZLMLrrD8RlOV31MDdFwA72hSfA8wLtucB53by87urvYhI1ojMs5UpVySXAzOAn7j7OjMbC/ypC5831N03AwTvpe3Uc+BZM1tsZnO60F5EJOttr2siP9cY0Dthq6Z3Sqc+3d1XAN8ECPol+rv7DQmM6wR332RmpcBzZvZucIXTaUECmgMwatSoRMQoIpJS1XWNFPfthZmlNI7Ojtp6ycwGmNlgYAlwt5n9sguft9XMhgXnHAZsi1XJ3TcF79uAR4Bph9I+aDvX3cvdvbykpKQLoYqIpLfqPU0M6Z/a/hHo/K2tInffBZwP3O3uU4HPduHzHiPcz0Lw/mjbCmbW18z6R7aB04HKzrYXEekpIlckqdbZRJIXXAF8kf2d7R0ys/nA68AEM9toZlcANwCnmdkq4LRgHzMrM7OngqZDgVfNbAmwEHjS3Z8OjsVsLyLSE23d1UhJimf+hU72kQA/Ap4BXnP3RWZ2OLCqowbuPqudQ6fGqLsJODPYXgtMbuec1bHai4j0NC2tIarqGjlsQO9Uh9LpzvaHgIei9tcCX0hUUCIi0rHtdU20hpzDilKfSDrb2T7CzB4JnlTfamYPm9mIRAcnIiKxbdm1FyAtrkg620dyN+GO7jJgOPB4UCYiIimwpTZIJJlyRQKUuPvd7t4SvO4BNKZWRCRFttQ2AJmVSLab2SVmlhu8LgGqExmYiIi0b8uuRvJzjcGFmfMcyVcID/3dAmwGLiA8bYqIiKTA1l17GTqgNzk5qX2qHTqZSNz9Q3c/291L3L3U3c8l/HCiiIikwObahrToaIf4Vkj8TrdFISIih2Trrsa06B+B+BJJ6q+nRER6IHfPmisS77YoRESk03Y1tLC3OZQ2VyQdPtluZruJnTAM6JOQiEREpEP7HkbMhETi7v2TFYiIiHTO5sgzJFlwa0tERFJgayZdkfR0v/rb+zz2zqYutz9YJ9JBRyvYgfXaroLWtn30YWtztL0F1CLnjHUu2/f5Ft4PDljkeNB+/364MMf2t8kx23eeHLPwsTbv4fL99feV5Ri5ZuTkhPfzcozcnBxycyA3JyfYD5fn5eaQn2vk5+aQF7z3ysuhV14uvfLD24UFefTJz6WwIJd+vfIY0Cef3DQYgy9yqDYH06OU9lciSXtlRX04cnhRXOdo79fUwZKMux9Yr00Db1PgHnu7o7reTn3wA+q4e/C+f599++G60dshB/dQsB3VNtgOebheayh8npA7IYdQaP/x1pATCoXLWz283RLa/94aclqDel1lBgN65zOoMJ+hA3ozcnAho4LXCUcMSYt1HkRi2bprL0P6FVCQlx43lZRIOvDF40byxeNGpjoM6YB7OLG0tDotoRDNrU5za4imlhBNrSEam0M0trSytznE3uZW6ptaqW9qoa6xhZ31zdTUN7GzvpkttQ28sqqKrbsaAcjPNU6feBizpo3i+HHFafH0sEjEltq9aXNbCxKYSMzsLuDzwDZ3nxSUDQYeAMYA64EvuvvONu1GAn8ADgNCwFx3/3Vw7IfAlUBVUP1f3f0ppMcys+CWFkBu3Ofb29zK6m11PPL2Rzz81kaeXLaZ0cWFfO3EcVwwdUTa/AUoPdvm2r2MGJQ+A2cT+X/FPcDMNmXXA8+7+3jg+WC/rRbgu+7+KWA6cI2ZTYw6fou7TwleSiLSrXrn5zJpeBH/8fmJvPH9U/nV/53CwMIC/vWRZZxy80vc9+aHNLWEUh2m9HCRebbSRcISibsvAHa0KT4HmBdszwPOjdFus7u/FWzvBlYSXgNFJKl65+dy7jHD+d+vH889lx9HSf9e+xLKPa+to6GpNdUhSg+0t7mVnfXNDEujW1vJvk4f6u6bIZwwgNKOKpvZGOAY4M2o4mvNbKmZ3WVmgxIVqEiEmXHyhFIeCRLKYUW9+eHjK/iHn7/ArS+sorahOdUhSg8SGfrbI65I4mVm/YCHgevcfVdQ/FtgHDCF8HT2v+ig/RwzqzCziqqqqvaqiXRaJKE8fPXxPPi1GRw1ooibn32fE254gZ88uWLfQ2IiiRRZGXFYUfr0kSR71NZWMxvm7pvNbBiwLVYlM8snnETudfe/RMrdfWtUnduBJ9r7IHefC8wFKC8v17xg0q2mjR3MtLHTWL6plrkL1nLXa+u5+7X1nD2ljK+fPI4jSjUphCTG/ulR0md4erKvSB4DZgfbs4FH21aw8BNydwIr3f2XbY4Ni9o9D6hMUJwinXJkWRG/vugYXvreyVwyfTR/XbaF029ZwL8+soyq3Y2pDk+yUOSKpEfc2jKz+cDrwAQz22hmVwA3AKeZ2SrgtGAfMyszs8gIrBOAS4HPmNk7wevM4NiNZrbMzJYCpwDfTlT8Iodi5OBCfnj2kbz6L6dw6fTRPLhoAyff9CK3vrCKvc3qlJfus7l2L/165dG/d36qQ9nH/OOPNWed8vJyr6ioSHUY0oOsqarjhr++y3MrtnJEaT9+fdEUjiyLb5YEEYCr/7SY97fu5vnvnpzwzzKzxe5efrB6advZLpLJxpX04/bLypn3lWnsamjm3Nte4/cvryEUx5QuIhDuI0mnjnZQIhFJqJM+UcIz153IqZ8cys/++i5fuuMNtu3em+qwJINtqU2vhxFBiUQk4Qb1LeC3lxzLTRcczZINtZx32995b8vuVIclGag15Gzb3ZhWI7ZAiUQkKcyMC8tH8tBVM2huDXHBb//Ogvf1fJMcmuq6RlpDzmG6tSXSc00aXsT/XnMCwwf14fJ7FnHfmx+mOiTJIJF1SNJlZcQIJRKRJCsb2Ic/X308J44fwr8+soxbX1hFTxg9KfHb9zCiEomI9OuVx+2XlXP+McO5+dn3+fnT7ymZyEFFHkZMp7VIQAtbiaRMXm4ON184mT4Fufzu5TXUN7Xww7OO1CJa0q4tu/aSn2sU9y1IdSgHUCIRSaGcHOO/z51Ev155/H7BWvY0tnLjBUdrLXmJaXNNA6X9e6fdHxtKJCIpZmZcf8YnKSzI45a/vY+7c9OFk5VM5GPWV9czurgw1WF8jBKJSBowM7712fHkGPziufcBlEzkYz6o3sMZRw07eMUkUyIRSSPfOHU8oGQiH1db38zO+mbGFvdNdSgfo0Qikmaik4mZcdMFR6fdPXFJvnXVewB0a0tEOucbp44n5HDL396nqE8+//H5TxFeqkd6qvXbw4lk7BBdkYhIJ33z1COoaWjirtfWMbhvPtd+ZnyqQ5IUWl+9B7Pw2jfpRolEJE2ZGf/xfyZSU9/Mzc++T1FhAZdOH53qsCRF1m/fQ1lRH3rn56Y6lI9J5AqJd5nZNjOrjCobbGbPmdmq4H1QO21nmtl7ZrbazK4/1PYi2SInx7jxgqM59ZOl/ODRSp5YuinVIUmKrKuuZ8yQ9LsagcROkXIPMLNN2fXA8+4+Hng+2D+AmeUCtwFnABOBWWY2sbPtRbJNfm4Ot118LFNHDeI7Dy5h8Qc7Uh2SpMD67XsYk4YjtiCBicTdFwBt/8WfA8wLtucB58ZoOg1Y7e5r3b0JuD9o19n2Ilmnd34ucy8rp6yoN1f+YTEfBCN4pGeoqW+itqE5LTvaIfmTNg51980AwXtpjDrDgQ1R+xuDss62F8lKg/sWcPfl0wi5c/ndi9i5pynVIUmSrNseGfqrRNJZscY4HvK0qGY2x8wqzKyiqkoLCEl2GDukL7dfVs7GnQ187Y+LaWxpTXVIkgTrqyNDf3teH0ksW81sGEDwvi1GnY3AyKj9EUCkh7Ez7QFw97nuXu7u5SUlJd0SvEg6OG7MYG668GgWrt/B9/+yTNPP9wDrtten7dBfSH4ieQyYHWzPBh6NUWcRMN7MxppZAXBR0K6z7UWy3jlThnPdZ8fzl7c+4rcvr4nrXC2tIX738hq2BosmSfr5oDo89LdXXvoN/YXEDv+dD7wOTDCzjWZ2BXADcJqZrQJOC/YxszIzewrA3VuAa4FngJXAg+6+PDhtzPYiPdG3Th3PWZPLuPHp93i6cnOXz7Nw3Q5u+Ou7XHrnm9TUq98lHa3fvidtO9ohgQ8kuvusdg6dGqPuJuDMqP2ngKdi1KuO1V6kJ4rMw7VhRz3ffmAJIwYVMml40SGfZ+H6HZjB+u31fOWeRdz71en0KUjPv3x7Indn3fY9nD2lLNWhtCsdO9tFpJPCw4KnMqgwn6/Oq9i3FOuhWLR+BxOG9ud/Zk3hnQ01XH3vYppbQwmIVrqipr6ZXXtb0vYZElAiEcl4pf17c8fs49i9t5kr5i1iT2NLp9s2t4Z4+8Mapo0dzMxJw/jJeUfx0ntV/POflxIKqRM/HURm/VUiEZGEmlg2gFsvPpaVm3fxzflv09rJJLBi0y7qm1o5bsxgAGZNG8U/fW4Cj7z9ETc8/W4iQ5ZOisz6OyaN+0iUSESyxCkTSvmvcybx/Lvb+PETKzrVZtH68OQTkUQC8PWTx3HZjNHMXbCWO19dl5BYpfPWb99DjsHIwX1SHUq7NPuvSBa5dPpoPti+hzteXcfo4kIuP2Fsh/UXrd/ByMF9OKyo974yM+M/zzqSbbsa+fETKyjt34uzJqdvR2+2W19dT9nA9B36C7oiEck63z/zU5w+cSg/fmIFL77X7jO7uDsV63cecDUSkZtj/OqiKRw3ZhDffXAJf1+zPZEhSwfWV6f30F9QIhHJOpEkMOGwAXzzvrdZvW13zHprt++hek9TzEQC4RFhd1x2HKOLC7lyXgVvrK1OZNgSQ2Tobzp3tIMSiUhWKizI447Z5fTKz+WKeRUxJ3hctO7j/SNtFRXm88crPs1hRb2ZfddCXny3/Ssc6X479jSxe29LWne0gxKJSNYaPrAPv790Kptr9nL1vYtpajnw2ZBF63cyuG8B40o6/iV1WFFvHvzaDI4o7ceVf6jgyaVdf4peDs366noAxhSn5xxbEUokIlls6uhB/PyCo3hj7Q5+8GjlARM8Llq/g/LRgzCLNeH2gYr79eK+K6czZeRAvjH/LR6q2HDQNhK/TBj6C0okIlnvvGNGcM0p47h/0QZue3E1AFt37eXDHfVMG9v+ba22ivrk84crpnH8uCH888NLeXyJlv1NtPe27qYgN4dRaTrrb4SG/4r0AN87fQKbavZy87PvM6yoD73yw39DdtQ/EkthQR63X1bO7LsW8u0H3qGwIJdTPzU0ESELsHRjDZ8qG0B+bnr/zZ/e0YlItzAzfv6Fozl+XDH/8vBS7nx1HX3yc5lYNuCQz9WnIJc7v1zOxLIBXH3vWxoanCChkFP50S6O7sJEnMmmRCLSQxTk5fC7S6dyRGk/3v6whmNHD+zyX7r9e+cz7/JpjCku5KvzKnjrw53dHK2sq95DXWMLRymRiEg6GdA7n7svP47DS/oyc9KwuM41qG8Bf7ri05T278XsOxeyZENNN0UpAMs21gJw1AglEhFJM8OK+vDCd0/m0umj4z5X6YDe3HfldAb1LeDSO9/c98tP4rd0Yy298nIYX9ov1aEcVEoSiZl9y8wqzWy5mV0X4/g/mdk7wavSzFrNbHBwbL2ZLQuOVSQ/ehGJVjawD/PnTGdAn3wuufNNKj9SMukOlR/VcmTZAPLSvKMdUpBIzGwScCUwDZgMfN7MxkfXcfeb3H2Ku08Bvg+87O47oqqcEhwvT1rgItKu4QP7MP/K6fTrlccld77J8k1KJvFoDTmVm2o5esTAVIfSKalIdZ8C3nD3+mB99peB8zqoPwuYn5TIRKTLRg4uZP6V0+mTn8vFd+jKJB5rq+qob2rNiI52SE0iqQRONLNiMyskvFb7yFgVg+MzgYejih141swWm9mchEcrIp02qriQB+bMoG9BHl+6/Q2WblQHfFcszaCOdkhBInH3lcDPgeeAp4ElQHtrg54FvNbmttYJ7n4scAZwjZmdGKuhmc0xswozq6iqquq+H0BEOjSquJD7gz6Ti+94k7c1NPiQLfuolj75uYwrSf+OdkhRZ7u73+nux7r7icAOYFU7VS+izW0td98UvG8DHiHc1xLrM+a6e7m7l5eUlHRf8CJyUCMHF/LA12YwqLCAy+5cyOIPdhy8keyzdGMNk4YPIDfn4POgpYNUjdoqDd5HAecTow/EzIqAk4BHo8r6mln/yDZwOuFbZSKSZoYP7MMDX5vOkP69uPTOhVrPpJNaWkOs2LyLo4ZnRkc7pO45kofNbAXwOHCNu+80s6vM7KqoOucBz7r7nqiyocCrZrYEWAg86e5PJy9sETkUw4r68MCc6ZQN7MOX717Iq6s0ncrBrK6qY29ziKMzpH8EUjRpo7v/Y4yy37XZvwe4p03ZWsJDhkUkQ5QO6M39c6ZzyR1v8pV5i/j9JVM55ZOlqQ4rbWVaRzvoyXYRSYIh/Xox/8rpfGJoP+b8sYInlmoK+vYs21hLv155jE3z5XWjKZGISFIM6lvAvV+NLI71Nve++UGqQ0pLS4Mn2nMypKMdlEhEJImK+uTzh698mlMmlPJvj1Ry24urD1i1sadragmxcvOujOofASUSEUmyPgW5/P7SqZwzpYybnnmPnzy5klBIyQTgvS27aWoJMSlDnmiP0AqJIpJ0+bk53PLFKQwqLOCOV9exY08TP7/g6LRfCTDRnn93K2YwY1xxqkM5JEokIpISOTnGf541keK+BfziuffZWd/EbRcfS2FBz/219HTlFspHD6K0f+9Uh3JIenb6F5GUMjO+cep4fnreUbz8fhUX3/EmO/c0pTqslFi3fQ/vbtkd94JjqaBEIiIp96VPj+I3Fx/L8k27+OwvX+bBRRs67DfZsaeJf/nzUmb+agF3vLKW3XubkxhtYjxduQWAmZMOS3Ekh856woiJ8vJyr6jQGlgi6W75plp+8OhyFn+wk8kjB/Kjs49k8sj9U4WEQs79izZw4zPvUre3hU8O60/lR7vo3yuPi6aN5MyjhtGvVx6983MpLMhlcN8CzDJjGO05t76KA49d+w+pDmUfM1vcmXWfeu7NSBFJO0eWFfHnq2bwyNsf8dOn3uWc215jcN8CSvv3onRAb6rrGlm+aRefHjuYH587iU8M7c/SjTXc/so67nptPbe/su6A85UV9ebMo4bxf44expSRA9M2qXxU08CSjbX888wJqQ6lS3RFIiJpaffeZuYv/JD11fVs29XItt17aWoJ8bWTDufcKcM/lhQ21TSwcvMu6ptaaWhqZXdjC6+v2c7kHa+cAAAMu0lEQVTL71fR3OqUFfVmXGk/BhYWMLgwn9IBvblg6giGDkh9x/Zdr67jR0+s4IXvnsThaTR1fGevSJRIRCSr1TY087cVW/nbyq1srt1LTX0TO+ubqW1opiAvhy9NG8XXTx5HaQoTyhd/9zq1Dc088+2YyyuljG5tiYgQfpr+C1NH8IWpIw4o/7C6nltfXMUf3/iA+Qs/5JLpo7n65HEM6dcrqfFV7W5k0Qc7+OZnxif1c7uTRm2JSI80qriQGy+YzAvfPYmzJpdx92vrOPHGF7npmXeprU/eKLBnV2zBHc44KvNGa0UokYhIjza6uC83XziZ575zEp/5ZCm3vbiGf7zxBW57cTV7GttbBbz7PF25hTHFhUwY2j/hn5UoSiQiIsC4kn7c+qVjefKb/8BxYwZz0zPvceKNL3LHK2vZ29yakM9cva2O19dUM3PSsLQdUdYZqVpq91tmVmlmy83suhjHTzazWjN7J3j9IOrYTDN7z8xWm9n1yY1cRLLdkWVF3Pnl4/jL149nYtkA/vvJlZx444v86Y0PaG4NddvntIac7z20hH698/jKP4zptvOmQtITiZlNAq4EphFe7fDzZharl+kVd58SvH4UtM0FbgPOACYCs8xsYpJCF5Ee5NhRg/jjFZ/mgTnTGVPcl3//30o+d8sCnq7c0i1T39/+ylre2VDDf519ZMbNrdVWKq5IPgW84e717t4CvEx4ffbOmAasdve17t4E3A+ck6A4RUT49OHFPPC16dxxWTk5OcZVf1rMBb97nbc+3Nnlc67etptfPvc+nztyKGdPLuvGaFMjFYmkEjjRzIrNrBA4ExgZo94MM1tiZn81syODsuHAhqg6G4MyEZGEMTM+O3EoT3/rH7nh/KPYsKOe83/zd667/2021zYc0rlaWkN896Gl9C3I5b/PPSqj+0Yikv4cibuvNLOfA88BdcASoO3QiLeA0e5eZ2ZnAv8LjAdifeMxrzHNbA4wB2DUqFHdFL2I9GR5uTlcNG0UZ00u47cvrWHuK2t5ZvlW5px4OGdNLmNcSd8OE0NTS4hf/e19lmyo4f/NOoaS/sl9ZiVRUv5ku5n9FNjo7r/poM56oJxwMvmhu38uKP8+gLv/rKPP0JPtIpIIG3bU87O/ruSpZeGZe0v692L64cVMGzOII0r7M660LyX9elG1u5F73/yQ+xZ+SNXuRs6eXMavL5qS9lcjaf1ku5mVuvs2MxsFnA/MaHP8MGCru7uZTSN8C64aqAHGm9lY4CPgIuBLyY1eRCRs5OBCfnPxVD6o3sPf11Tz+ppqXl9bzeNLNu2r0793Hg1NrbSEnFMmlHDZ8WM4aXxJ2ieRQ5GqKVIeNrNioBm4xt13mtlVAO7+O+AC4GozawEagIs8fOnUYmbXAs8AucBd7r48NT+CiEjY6OK+jC7uy6xpo3B3NtXuZW1VHWu21bGmag+FvXKZddwoxgzpm+pQEyLlt7aSQbe2REQOXWdvbenJdhERiYsSiYiIxEWJRERE4qJEIiIicVEiERGRuCiRiIhIXJRIREQkLkokIiISlx7xQKKZ1QKrYhwqAmo7uX+w7SHA9i6E1/YzO3s8Vvmhxh9v7B3Fd7Dj+u7T57vvKN7o/WR+9x3Vifffjr77zn/3o9295KBndPesfwFzO1Pe0f7BtoGK7oztUGPvSvzxxt6d8eu7T5/42/tZkvndJ/Lfjr77rn/37b16yq2txztZ3tF+Z7a74mDtOxt7rLKDxRxv7J05h7779P/u25a197Mk87vvqE68/3b03XdPnX16xK2tZDCzCu/EnDTpKJNjh8yOP5Njh8yOP5Njh/SKv6dckSTD3FQHEIdMjh0yO/5Mjh0yO/5Mjh3SKH5dkYiISFx0RSIiInFRImnDzO4ys21mVtmFtlPNbJmZrTaz/7GoJdDM7ItmtsLMlpvZfd0b9QExdHv8ZvZlM6sys3eC11e7P/LEfffB8QvMzM0sYfeUE/TdXxWUv2Nmr5rZxO6PfF8MiYj/O8G/+6Vm9ryZje7+yBMW+4lm9paZtZjZBekUczvnm21mq4LX7KjysWb2ZlD+gJkVdMfnHaCrw9+y9QWcCBwLVHah7ULCywYb8FfgjKB8PPA2MCjYL82w+L8M3JqJ331wrD+wAHgDKM+k+IEBUXXOBp7OsPhPAQqD7auBBzIo9jHA0cAfgAvSJWbgJWBMm7LBwNrgfVCwHfl98yDhVWYBfgdc3d0/i65I2nD3BcCO6DIzG2dmT5vZYjN7xcw+2badmQ0j/D/96x7+L/YH4Nzg8JXAbe6+M/iMbRkWf1IkMPYfAzcCexMYfkLid/ddUVX7Agnr1ExQ/C+6e31Q9Q1gRAbFvt7dlwKhdIq5HZ8DnnP3HcHvmeeAmcHV1WeAPwf15pGA/6+VSDpnLvANd58KfA/4TYw6w4GNUfsbgzKATwCfMLPXzOwNM5uZ0Gg/Lt74Ab4Q3J74s5mNTFyoHxNX7GZ2DDDS3Z9IdKDtiPu7N7NrzGwN4WT4zQTGGkt3/NuJuILwX/zJ0p2xJ0tnYo5lOLAhaj/ycxQDNe7e0qa8W+V19wmzjZn1A44HHoq67d4rVtUYZZG/HvMI3946mfBfZK+Y2SR3r+neaGME1T3xPw7Md/dGM7uK8F81n+nuWD8WUJyxm1kOcAvhW3NJ103fPe5+G3CbmX0J+Hdgdoz63a674g/OdQlQDpzUnTG2pztjT5aOYjazy4FvBWVHAE+ZWROwzt3Po/2fIyk/nxLJweUQzuhTogvNLBdYHOw+BvyWAy/bRwCbgu2NwBvu3gysM7P3CCeWRYkMPBB3/O5eHVV+O/DzhEV7oHhj7w9MAl4K/sc8DHjMzM5294oExw7d828n2v1B3WTplvjN7LPAvwEnuXtjQiPer7u/+2SIGTOAu98N3A1gZi8BX3b39VFVNhL+QzViBOG+lO3AQDPLC65KEvPzdXenSza8CHeyVUbt/x24MNg2YHI77RYB09nfaXdmUD4TmBdsDyF8CVqcQfEPi6pzHuGkmBGxt6nzEgnsbE/Qdz8+qs5ZxDE/VIriPwZYE/1zZErsUcfvIQGd7V2NmfY729cR7mgfFGwPDo49xIGd7V/v9p8j0f9xM+0FzAc2A82Es/wVwFjgaWAJsAL4QTtty4HK4H+cW9n/wKcBvwzaLov8R82g+H8GLA/avwh8MlNib1PnJRI7aisR3/2vg+/+neC7PzLD4v8bsDWI/x3gsQyK/bjgXHuAamB5OsRMjEQSlH8FWB28Lo8qP5zwyLTVhJNKr+7+/vVku4iIxEWjtkREJC5KJCIiEhclEhERiYsSiYiIxEWJRERE4qJEIj2SmdUl+fPu6K6Ze82s1cKzAVea2eNmNvAg9Qea2de747NFYtHwX+mRzKzO3ft14/kiTw4nXHTsZjYPeN/df9JB/THAE+4+KRnxSc+jKxKRgJmVmNnDZrYoeJ0QlE8zs7+b2dvB+4Sg/Mtm9pCZPQ48a2Ynm9lLwcSW75rZvcHsqwTl5cF2nZn9xMyWBJN4Dg3KxwX7i8zsR528anqd/RNU9rPwmh9vWXh9jXOCOjcA44KrmJuCuv8UfM5SM/uvbvwapQdSIhHZ79fALe5+HPAF4I6g/F3gRHc/BvgB8NOoNjOA2e4emcTyGOA6YCLhJ4pPiPE5fQlPMzOZ8DopV0Z9/q+Dzz/ofEjBvFGnEp4zCsLT5J/n7scSXgfkF0Eiux5Y4+5T3P2fzOx0wnO9TQOmAFPN7MSDfZ5IezRpo8h+nwUmRs28OsDM+gNFwDwzG0945tT8qDbPuXv0mhIL3X0jgJm9Q3gupVfbfE4TEJnWfjFwWrA9g/1rRdwH3NxOnH2izr2Y8NoTEJ6K56dBUggRvlIZGqP96cHr7WC/H+HEsqCdzxPpkBKJyH45wAx3b4guNLP/B7zo7ucF/Q0vRR3e0+Yc0bPbthL7/7Fm39852V6djjS4+xQzKyKckK4B/ge4GCgBprp7s5mtB3rHaG/Az9z994f4uSIx6daWyH7PAtdGdswsMp13EfBRsP3lBH7+G4RvqQFcdLDK7l5LeKGr75lZPuE4twVJ5BQgsj76bsJT6kc8A3wlWP8CMxtuZqXd9DNID6REIj1VoZltjHp9h/Av5fKgA3oFcFVQ90bgZ2b2GpCbwJiuA75jZguBYUDtwRq4+9uEZ4q9CLiXcPwVhK9O3g3qVAOvBcOFb3L3ZwnfOnvdzJYRXoa1f8wPEOkEDf8VSRNmVkj4tpWb2UXALHc/52DtRFJNfSQi6WMqcGsw0qqG8PoSImlPVyQiIhIX9ZGIiEhclEhERCQuSiQiIhIXJRIREYmLEomIiMRFiUREROLy/wFNzZh410NAHAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-2\n",
    "lr *= bs/48  # Scale learning rate by batch size\n",
    "# 所有层都是可以训练的\n",
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='10', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/10 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='21081' class='' max='25376', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      83.07% [21081/25376 1:11:46<14:37 5.1408]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(10, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 保存模型权重与字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdl_path = path/'models'\n",
    "mdl_path.mkdir(exist_ok=True)\n",
    "learn.to_fp32().save(mdl_path/lm_fns[0], with_opt=False)\n",
    "learn.data.vocab.save(mdl_path/(lm_fns[1].with_suffix('.pkl')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = \"中国\"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n\\n\".join(learn.predict(TEXT, N_WORDS, temperature=0.9) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 进行语义分析任务\n",
    "- 参看 [中文数据集](BERT.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = Config.data_path()/r'xiecheng'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp {root}/'train.csv' {root}/'val.csv' {path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(path/'train.csv')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(path/'val.csv')\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df,test_df], sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = (TextList.from_df(df, path, cols='review', processor=[\n",
    "    OpenFileProcessor(), SPProcessor.load(dest)], vocab=data.vocab)\n",
    "    .split_by_rand_pct(0.1, seed=42)\n",
    "    .label_for_lm()\n",
    "    .databunch(bs=bs, num_workers=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm.save(f'{lang}_clas_databunch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = load_data(f'{lang}_clas_databunch', bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用我们之前训练的模型学习一个语言模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- `pretrained_fnames=lm_fns`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm_fns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm = language_model_learner(data_lm, AWD_LSTM, pretrained_fnames=lm_fns, drop_mult=1.0, wd=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "lr *= bs/48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.fit_one_cycle(2, lr*10, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.unfreeze()\n",
    "learn_lm.fit_one_cycle(8, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save(f'{lang}fine_tuned')\n",
    "learn_lm.save_encoder(f'{lang}fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = '我喜欢'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n\\n\".join(learn_lm.predict(TEXT, N_WORDS, temperature=0.9) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_lm.train_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = (TextList.from_df(train_df, path, , processor=[\n",
    "    OpenFileProcessor(), SPProcessor.load(dest)], vocab=data_lm.vocab, cols='review')\n",
    "    .split_by_rand_pct(0.1, seed=42)\n",
    "    .label_from_df(cols='label')\n",
    "    .databunch(bs=bs, num_workers=4))\n",
    "\n",
    "data_clas.save(f'{lang}_textlist_class')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = load_data(path, f'{lang}_textlist_class', bs=bs, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@np_func\n",
    "def f1(inp,targ): return f1_score(targ, np.argmax(inp, axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c = text_classifier_learner(data_clas, AWD_LSTM, pretrained=False, drop_mult=0.5, wd=0.1, metrics=[f1, accuracy]).to_fp16()\n",
    "learn_c.load_encoder(f'{lang}fine_tuned_enc')\n",
    "learn_c.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=2e-2\n",
    "lr *= bs/48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c.fit_one_cycle(8, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c.freeze_to(-2)\n",
    "learn_c.fit_one_cycle(3, slice(lr/(2.6**4),lr), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c.freeze_to(-3)\n",
    "learn_c.fit_one_cycle(3, slice(lr/(2.6**4),lr), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c.unfreeze()\n",
    "learn_c.fit_one_cycle(5, slice(lr/10/(2.6**4),lr/10), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c.save(f'{lang}clas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}