8.GPT3/8.3.gpt2_fewshot_NSMC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 주의!!\n",
    "\n",
    "이 실습은 가급적 NVIDIA GPU가 설치된 컴퓨터 환경이거나 Google Colab에서 진행해주세요."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2NmYZYYhXrcZ"
   },
   "source": [
    "## 환경 준비 \n",
    "(Google Colab 환경에서 사용하세요)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6-bFpckCXrcb"
   },
   "outputs": [],
   "source": [
    "!wget https://raw.githubusercontent.com/NLP-kr/tensorflow-ml-nlp-tf2/master/requirements.txt -O requirements.txt\n",
    "!pip install -r requirements.txt\n",
    "!pip install tensorflow==2.2.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cvFHjoTCXrcc"
   },
   "source": [
    "## 데이터 다운로드\n",
    "(Google Colab 환경에서 사용하세요)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HbKNloVoXrcd"
   },
   "outputs": [],
   "source": [
    "!mkdir -p data_in/KOR/naver_movie\n",
    "!wget https://raw.githubusercontent.com/NLP-kr/tensorflow-ml-nlp-tf2/master/7.PRETRAIN_METHOD/data_in/KOR/naver_movie/ratings_train.txt \\\n",
    "              -O data_in/KOR/naver_movie/ratings_train.txt\n",
    "!wget https://raw.githubusercontent.com/NLP-kr/tensorflow-ml-nlp-tf2/master/7.PRETRAIN_METHOD/data_in/KOR/naver_movie/ratings_test.txt \\\n",
    "              -O data_in/KOR/naver_movie/ratings_test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xs88fDX8Xrcd",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from transformers import TFGPT2LMHeadModel\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import gluonnlp as nlp\n",
    "from gluonnlp.data import SentencepieceTokenizer\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import re\n",
    "\n",
    "import random\n",
    "from random import sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XgV0aK1KXrce"
   },
   "source": [
    "아레 실행 커멘드는 gpt_ckpt 폴더가 있지 않은 경우에만 실행해주세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XmofLC_rXrce"
   },
   "outputs": [],
   "source": [
    "import wget\n",
    "import zipfile\n",
    "\n",
    "wget.download('https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/releases/download/v1.0/gpt_ckpt.zip')\n",
    "\n",
    "with zipfile.ZipFile('gpt_ckpt.zip') as z:\n",
    "    z.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TVExOYgEXrcf"
   },
   "outputs": [],
   "source": [
    "# 시각화\n",
    "\n",
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string], '')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "s6dM4ebxXrcg"
   },
   "outputs": [],
   "source": [
    "SEED_NUM = 1234\n",
    "tf.random.set_seed(SEED_NUM)\n",
    "np.random.seed(SEED_NUM)\n",
    "random.seed(SEED_NUM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WSYro-2hvbOI"
   },
   "source": [
    "## 퓨샷 러닝을 위한 네이버 영화 리뷰 모델 구성\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lAaKKUqbXrch"
   },
   "outputs": [],
   "source": [
    "TOKENIZER_PATH = './gpt_ckpt/gpt2_kor_tokenizer.spiece'\n",
    "\n",
    "tokenizer = SentencepieceTokenizer(TOKENIZER_PATH)\n",
    "vocab = nlp.vocab.BERTVocab.from_sentencepiece(TOKENIZER_PATH,\n",
    "                                               mask_token=None,\n",
    "                                               sep_token='<unused0>',\n",
    "                                               cls_token=None,\n",
    "                                               unknown_token='<unk>',\n",
    "                                               padding_token='<pad>',\n",
    "                                               bos_token='<s>',\n",
    "                                               eos_token='</s>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AypWVja1Xrcj"
   },
   "outputs": [],
   "source": [
    "class TFGPT2FewshotClassifier(tf.keras.Model):\n",
    "    def __init__(self, dir_path):\n",
    "        super(TFGPT2FewshotClassifier, self).__init__()\n",
    "        self.gpt2 = TFGPT2LMHeadModel.from_pretrained(dir_path)\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        outputs = self.gpt2({'input_ids': inputs})[0][:, -1, :]\n",
    "\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9J5VOzCwXrcj",
    "outputId": "537cde6c-958a-4bc7-f98d-996b3bb13bb3"
   },
   "outputs": [],
   "source": [
    "BASE_MODEL_PATH = './gpt_ckpt'\n",
    "cls_model = TFGPT2FewshotClassifier(dir_path=BASE_MODEL_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pCN8Lh7gXrch"
   },
   "source": [
    "## 퓨샷 러닝을 위한 네이버 영화 리뷰 데이터 구성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ct1IbwATXrci"
   },
   "outputs": [],
   "source": [
    "# 데이터 전처리 준비\n",
    "DATA_IN_PATH = './data_in/KOR'\n",
    "DATA_OUT_PATH = './data_out/KOR'\n",
    "\n",
    "DATA_TRAIN_PATH = os.path.join(DATA_IN_PATH, 'naver_movie', 'ratings_train.txt')\n",
    "DATA_TEST_PATH = os.path.join(DATA_IN_PATH, 'naver_movie', 'ratings_test.txt')\n",
    "\n",
    "train_data = pd.read_csv(DATA_TRAIN_PATH, header = 0, delimiter = '\\t', quoting = 3)\n",
    "train_data = train_data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WED9P9SUSyR9",
    "outputId": "5c4ba8bd-9a78-49fa-ad19-9fe14603723f"
   },
   "outputs": [],
   "source": [
    "print('데이터 positive 라벨: ', tokenizer('긍정'))\n",
    "print('데이터 negative 라벨: ', tokenizer('부정'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WaQ1miXfwQfn",
    "outputId": "6d506ff6-ac64-4478-b3cc-dbbf7aa9526d"
   },
   "outputs": [],
   "source": [
    "print('학습 예시 케이스 구조: ', tokenizer('문장: 오늘 기분이 좋아\\n감정: 긍정\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7h0USc0RxQqG",
    "outputId": "7ad08962-3798-4653-ce76-b6a69a1eb4e9"
   },
   "outputs": [],
   "source": [
    "print('gpt2 최대 토큰 길이: ', cls_model.gpt2.config.n_ctx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MRwI-RcOyFRj",
    "outputId": "ccbf5e78-27a2-4c4e-8b74-327de16c6673"
   },
   "outputs": [],
   "source": [
    "sent_lens = [len(tokenizer(s)) for s in train_data['document']]\n",
    "\n",
    "print('Few shot 케이스 토큰 평균 길이: ', np.mean(sent_lens))\n",
    "print('Few shot 케이스 토큰 최대 길이: ', np.max(sent_lens))\n",
    "print('Few shot 케이스 토큰 길이 표준편차: ',np.std(sent_lens))\n",
    "print('Few shot 케이스 토큰 길이 80 퍼센타일: ',np.percentile(sent_lens, 80))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PdIWfc6Pzyfz"
   },
   "outputs": [],
   "source": [
    "train_fewshot_data = []\n",
    "\n",
    "for train_sent, train_label in train_data[['document', 'label']].values:\n",
    "    tokens = vocab[tokenizer(train_sent)]\n",
    "\n",
    "    if len(tokens) <= 25:\n",
    "        train_fewshot_data.append((train_sent, train_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4jFe7XMeXrcl"
   },
   "source": [
    "## 네이버 영화 리뷰 데이터를 활용한 퓨샷 러닝 및 평가"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "1_OhF3hVhK0y",
    "outputId": "6a661ba0-e27e-4aaf-ba2e-5d49809ef866"
   },
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(DATA_TEST_PATH, header=0, delimiter='\\t', quoting=3)\n",
    "test_data = test_data.dropna()\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "liE91_rhsQdY"
   },
   "outputs": [],
   "source": [
    "sample_size = 5000\n",
    "\n",
    "train_fewshot_samples = []\n",
    "\n",
    "for _ in range(sample_size):\n",
    "    fewshot_examples = sample(train_fewshot_data, 30)\n",
    "    train_fewshot_samples.append(fewshot_examples)\n",
    "\n",
    "if sample_size < len(test_data['id']):\n",
    "    test_data = test_data.sample(sample_size, random_state=SEED_NUM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "s-ZiFs-aRIXy"
   },
   "outputs": [],
   "source": [
    "def build_prompt_text(sent):\n",
    "    return \"문장: \" + sent + '\\n감정: '\n",
    "\n",
    "def clean_text(sent):\n",
    "    sent_clean = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \"\", sent)\n",
    "    return sent_clean\n",
    "\n",
    "real_labels = []\n",
    "pred_tokens = []\n",
    "\n",
    "for i, (test_sent, test_label) in enumerate(test_data[['document','label']].values):\n",
    "    tokens = [vocab[vocab.bos_token]]\n",
    "\n",
    "    for ex in train_fewshot_samples[i]:\n",
    "        example_text, example_label = ex\n",
    "        cleaned_example_text = clean_text(example_text)\n",
    "        appended_prompt_example_text = build_prompt_text(cleaned_example_text)\n",
    "        appended_prompt_example_text += '긍정' if example_label == 1 else '부정' + '\\n'\n",
    "\n",
    "        tokens += vocab[tokenizer(appended_prompt_example_text)]\n",
    "\n",
    "    cleaned_sent = clean_text(test_sent)\n",
    "    appended_prompt_sent = build_prompt_text(cleaned_sent)\n",
    "    test_tokens = vocab[tokenizer(appended_prompt_sent)]\n",
    "\n",
    "    tokens += test_tokens\n",
    "\n",
    "    pred = tf.argmax(cls_model(np.array([tokens], dtype=np.int64)), axis=-1).numpy()\n",
    "    label = vocab[tokenizer('긍정')] if test_label == 1 else vocab[tokenizer('부정')]\n",
    "\n",
    "    pred_tokens.append(pred[0])\n",
    "    real_labels.append(label[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0oZ1GfUPeuec",
    "outputId": "becdea25-d59b-4dd2-d6d6-9caece719dad"
   },
   "outputs": [],
   "source": [
    "accuracy_match = [p == t for p, t in zip(pred_tokens, real_labels)]\n",
    "accuracy = len([m for m in accuracy_match if m]) / len(real_labels)\n",
    "\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vMYgRRGI-Gu4"
   },
   "outputs": [],
   "source": [
    "def build_prompt_text(sent):\n",
    "    return '감정 분석 문장: ' + sent + '\\n결과: '\n",
    "\n",
    "real_labels = []\n",
    "pred_tokens = []\n",
    "\n",
    "\n",
    "for i, (test_sent, test_label) in enumerate(test_data[['document','label']].values):\n",
    "    tokens = [vocab[vocab.bos_token]]\n",
    "\n",
    "    for ex in train_fewshot_samples[i]:\n",
    "        example_text, example_label = ex\n",
    "        cleaned_example_text = clean_text(example_text)\n",
    "        appended_prompt_example_text = build_prompt_text(cleaned_example_text)\n",
    "        appended_prompt_example_text += '긍정' if example_label == 1 else '부정' + '\\n'\n",
    "\n",
    "        tokens += vocab[tokenizer(appended_prompt_example_text)]\n",
    "\n",
    "    cleaned_sent = clean_text(test_sent)\n",
    "    appended_prompt_sent = build_prompt_text(cleaned_sent)\n",
    "    test_tokens = vocab[tokenizer(appended_prompt_sent)]\n",
    "\n",
    "    tokens += test_tokens\n",
    "\n",
    "    pred = tf.argmax(cls_model(np.array([tokens], dtype=np.int64)), axis=-1).numpy()\n",
    "    label = vocab[tokenizer('긍정')] if test_label == 1 else vocab[tokenizer('부정')]\n",
    "\n",
    "    pred_tokens.append(pred[0])\n",
    "    real_labels.append(label[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8ufjRihAzNBK",
    "outputId": "580a3580-8120-41bd-b0e6-5b94616fe0d1"
   },
   "outputs": [],
   "source": [
    "accuracy_match = [p == t for p, t in zip(pred_tokens, real_labels)]\n",
    "accuracy = len([m for m in accuracy_match if m]) / len(real_labels)\n",
    "\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uDbMaaxr2kfL"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "7.4.2.gpt2_fewshot_NSMC.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}