pos/blackboard_treebank_prompt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'blackboard-treebank'...\n",
      "remote: Enumerating objects: 235, done.\u001b[K\n",
      "remote: Counting objects: 100% (235/235), done.\u001b[K\n",
      "remote: Compressing objects: 100% (234/234), done.\u001b[K\n",
      "remote: Total 235 (delta 191), reused 0 (delta 0), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (235/235), 68.27 MiB | 2.60 MiB/s, done.\n",
      "Resolving deltas: 100% (191/191), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://bitbucket.org/kaamanita/blackboard-treebank.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = Path(os.path.join('.','blackboard-treebank','thai10'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_files = [f for f in train.iterdir() if not f.name.startswith('.')]\n",
    "len(train_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readfile(path):\n",
    "    _data = []\n",
    "    with open(path,'r',encoding=\"utf-8-sig\") as f:\n",
    "        _temp = [i for i in f.read().split(\"\\n\\n\") if i != '']\n",
    "    for j in _temp:\n",
    "        _temp_data = []\n",
    "        bad_data = False\n",
    "        for i in j.split(\"\\n\"):\n",
    "            if i.startswith('#') == False and not bad_data:\n",
    "                _t = i.strip().split(\"\\t\")\n",
    "                if _t[1]!=\"$$\":\n",
    "                    if _t[3]==\"N\":\n",
    "                        _temp_data=[]\n",
    "                        bad_data=True\n",
    "                    else:\n",
    "                        _temp_data.append((_t[1],_t[3]))\n",
    "        if _temp_data!=[]:\n",
    "            _data.append(_temp_data)\n",
    "    return _data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in train_files:\n",
    "    train_data.extend(readfile(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "130560"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ขณะนี้', 'NN'),\n",
       " ('พอ', 'AV'),\n",
       " ('จะ', 'AX'),\n",
       " ('ทราบ', 'VV'),\n",
       " ('ตัว', 'NN'),\n",
       " ('คนร้าย', 'NN'),\n",
       " ('แล้ว', 'AX')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.tag.blackboard import post_process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_ud = [post_process(i,to_ud=True) for i in train_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ที่', 'CC'),\n",
       " ('มี', 'VV'),\n",
       " ('ผล', 'NN'),\n",
       " ('ใน', 'PS'),\n",
       " ('การ', 'FX'),\n",
       " ('ยับยั้ง', 'VV')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ที่', 'CCONJ'),\n",
       " ('มี', 'VERB'),\n",
       " ('ผล', 'NOUN'),\n",
       " ('ใน', 'ADP'),\n",
       " ('การ', 'NOUN'),\n",
       " ('ยับยั้ง', 'VERB')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_ud[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_dict = defaultdict(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in train_data_ud:\n",
    "    _word= [k for k,_ in i]\n",
    "    _pos = '/'.join([k for _,k in i])\n",
    "    pos_dict[_pos].append(_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp=\"จงสร้างประโยคตามโครงสร้าง {pos}:\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs=[]\n",
    "targets=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in sorted(list(pos_dict.keys())):\n",
    "    if i.count(\"/\")<1:\n",
    "        continue\n",
    "    for k in pos_dict[i]:\n",
    "        inputs.append(temp.format(pos=i))\n",
    "        targets.append(''.join(k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('จงสร้างประโยคตามโครงสร้าง VERB/VERB/VERB/VERB/VERB/VERB/VERB/ADV/VERB/NOUN:',\n",
       " 'โกรธคือโง่โมโหคือบ้าอย่าไปถือสาคน')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[-1],targets[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = {\n",
    "    \"inputs\":inputs,\n",
    "    \"targets\":targets,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "จงสร้างประโยคตามโครงสร้าง ADJ/ADJ/AUX/VERB/VERB/NOUN/VERB/ADP/NOUN/ADV:\n",
      "ก่อนอื่นต้องขอแสดงความยินดีกับยานินาด้วย\n"
     ]
    }
   ],
   "source": [
    "id=0\n",
    "print(df_train[\"inputs\"][id])\n",
    "print(df_train[\"targets\"][id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset, DatasetDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = DatasetDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds['train'] = Dataset.from_dict(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['inputs', 'targets'],\n",
       "        num_rows: 130454\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Pushing split train to the Hub.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da39ee7d3e3a43a4ad9d431d16dd65eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Pushing dataset shards to the dataset hub:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds.push_to_hub(\"pythainlp/blackboard_treebank_prompt\",private=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a1d6ff38954a1cdba4cf61ffa51e42f4658fc35985cd256cd89123cae8466a39"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}