T5_style_mask_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'Toxic shock syndrome(TSS)は黄色ぶどう球菌やA群 β 溶血性連鎖球菌が産生するスーパー抗原に起因した感染性疾患である.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('ja_ginza')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask_ratio = 0.15\n",
    "span_length = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_chunk_num(doc):\n",
    "    mask_token = math.ceil(len(doc)*mask_ratio)\n",
    "    chunk_num = math.ceil(mask_token/span_length)\n",
    "    return chunk_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_num = get_chunk_num(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chunk_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_sequence = [i for i in range(len(doc)-span_length+1)]\n",
    "for i in range(len(index_sequence)):\n",
    "    index_sequence[i] = [i for i in range(i,i+span_length)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask_index = []\n",
    "while chunk_num > 0:\n",
    "    masked_spans = index_sequence[randint(0,len(index_sequence))]\n",
    "    mask_index.extend(masked_spans)\n",
    "    index_id = index_sequence.index(masked_spans)\n",
    "    del index_sequence[index_id-span_length+1:index_id+span_length]\n",
    "    chunk_num = chunk_num - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask_index.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[9, 10, 11, 19, 20, 21]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_sequence = [i for i in range(len(doc))]\n",
    "unmask_index = list(set(text_sequence) - set(mask_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "unmask_index.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 12,\n",
       " 13,\n",
       " 14,\n",
       " 15,\n",
       " 16,\n",
       " 17,\n",
       " 18,\n",
       " 22,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 28,\n",
       " 29,\n",
       " 30]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unmask_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = [None] * len(doc)\n",
    "for i in unmask_index:\n",
    "    input_text[i] = doc[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Toxic,\n",
       " shock,\n",
       " syndrome,\n",
       " (,\n",
       " TSS,\n",
       " ),\n",
       " は,\n",
       " 黄色,\n",
       " ぶどう,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " 群,\n",
       " β,\n",
       " 溶血性,\n",
       " 連鎖,\n",
       " 球菌,\n",
       " が,\n",
       " 産生,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " に,\n",
       " 起因,\n",
       " し,\n",
       " た,\n",
       " 感染性,\n",
       " 疾患,\n",
       " で,\n",
       " ある,\n",
       " .]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_input_text = []\n",
    "for i in range(len(input_text)-1):\n",
    "        if str(input_text[i]) == 'None' and str(input_text[i+1]) == 'None':\n",
    "            pass\n",
    "        else:\n",
    "            re_input_text.append(input_text[i])\n",
    "        i+=2               \n",
    "\n",
    "re_input_text.append(input_text[len(input_text)-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Toxic,\n",
       " shock,\n",
       " syndrome,\n",
       " (,\n",
       " TSS,\n",
       " ),\n",
       " は,\n",
       " 黄色,\n",
       " ぶどう,\n",
       " None,\n",
       " 群,\n",
       " β,\n",
       " 溶血性,\n",
       " 連鎖,\n",
       " 球菌,\n",
       " が,\n",
       " 産生,\n",
       " None,\n",
       " に,\n",
       " 起因,\n",
       " し,\n",
       " た,\n",
       " 感染性,\n",
       " 疾患,\n",
       " で,\n",
       " ある,\n",
       " .]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_input_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_token_id = 0\n",
    "for i in range(len(re_input_text)):\n",
    "    if str(re_input_text[i]) == 'None':\n",
    "        re_input_text[i] = f'<extra_id_{sent_token_id}>'\n",
    "        sent_token_id += 1    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Toxic,\n",
       " shock,\n",
       " syndrome,\n",
       " (,\n",
       " TSS,\n",
       " ),\n",
       " は,\n",
       " 黄色,\n",
       " ぶどう,\n",
       " '<extra_id_0>',\n",
       " 群,\n",
       " β,\n",
       " 溶血性,\n",
       " 連鎖,\n",
       " 球菌,\n",
       " が,\n",
       " 産生,\n",
       " '<extra_id_1>',\n",
       " に,\n",
       " 起因,\n",
       " し,\n",
       " た,\n",
       " 感染性,\n",
       " 疾患,\n",
       " で,\n",
       " ある,\n",
       " .]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_input_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = ''.join(str(e) for e in re_input_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Toxicshocksyndrome(TSS)は黄色ぶどう<extra_id_0>群β溶血性連鎖球菌が産生<extra_id_1>に起因した感染性疾患である.'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_text = [None] * len(doc)\n",
    "for i in mask_index:\n",
    "    target_text[i] = doc[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " 球菌,\n",
       " や,\n",
       " A,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " する,\n",
       " スーパー,\n",
       " 抗原,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_target_text = []\n",
    "for i in range(len(target_text)-1):\n",
    "        if str(target_text[i]) == 'None' and str(target_text[i+1]) == 'None':\n",
    "            pass\n",
    "        else:\n",
    "            re_target_text.append(target_text[i])\n",
    "        i+=2               \n",
    "\n",
    "re_target_text.append(target_text[len(target_text)-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, 球菌, や, A, None, する, スーパー, 抗原, None]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_target_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_token_id = 0\n",
    "for i in range(len(re_target_text)):\n",
    "    if str(re_target_text[i]) == 'None':\n",
    "        re_target_text[i] = f'<extra_id_{sent_token_id}>'\n",
    "        sent_token_id += 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<extra_id_0>', 球菌, や, A, '<extra_id_1>', する, スーパー, 抗原, '<extra_id_2>']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_target_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = ''.join(str(e) for e in re_target_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<extra_id_0>球菌やA<extra_id_1>するスーパー抗原<extra_id_2>'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Toxicshocksyndrome(TSS)は黄色ぶどう<extra_id_0>群β溶血性連鎖球菌が産生<extra_id_1>に起因した感染性疾患である.'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}