evaluations.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qb72kIdtNlmc"
      },
      "source": [
        "## pip install"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "09EGc6gANiPw",
        "outputId": "79cae889-c207-4cb3-9bb8-93f2e4cf8b75"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting konlpy\n",
            "  Downloading konlpy-0.6.0-py2.py3-none-any.whl (19.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.4/19.4 MB\u001b[0m \u001b[31m25.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting JPype1>=0.7.0 (from konlpy)\n",
            "  Downloading JPype1-1.4.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (465 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m465.3/465.3 kB\u001b[0m \u001b[31m30.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.10/dist-packages (from konlpy) (4.9.2)\n",
            "Requirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.10/dist-packages (from konlpy) (1.22.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from JPype1>=0.7.0->konlpy) (23.1)\n",
            "Installing collected packages: JPype1, konlpy\n",
            "Successfully installed JPype1-1.4.1 konlpy-0.6.0\n"
          ]
        }
      ],
      "source": [
        "!pip install konlpy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j76NWHXcNrpN"
      },
      "source": [
        "## 상수 설정"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A4Jh0nU3N1P0"
      },
      "outputs": [],
      "source": [
        "EVALUATION_DATASET_PATH = './drive/MyDrive/Models/eval_queries.jsonl'\n",
        "EVALUATED_INFERENCE_PATH = './drive/MyDrive/Models/eval_response_kogpt2F.txt'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-mks5rQmNpg_"
      },
      "source": [
        "## import"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DJ0ruoGMNgMd",
        "outputId": "4bac290f-e82b-40cf-e3cc-74fbaa868556"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from collections import Counter\n",
        "from nltk import ngrams\n",
        "import json\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "phTZbO2_Ng2H",
        "outputId": "ba403aa3-50f3-4b67-e319-4ac81ffec153"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['응, 맞아. 그런 느낌이야. 너도 그런 특수 시술을 받고 싶어해?', '그래서 조금은 지칠 때가 있잖아. 어떻게 하면 좀 쉬어질까?', '사랑한다는 의지와 의지가 있으시군요! 어떤 상황에서 그런 인연을', '아마 도움이 필요하면 같이 할 것 같아.', '그랬군요. 그러면 나중에 다른 영화 추천해주세요!']\n",
            "[['어쨌든 괜찮을 거 같은데? 길게 안 하면', '무슨 특수 펌인지 잘 모르겠어. 설명해줄 수 있을까?', '그러게. 처음 보는 스타일이라서 뭔지 잘 모르겠어.'], ['아고 그렇겠다 진짜 ㅠㅠ 대기업인 만큼 진짜 일 많이 한다더라', '정말 힘들구나. 스트레스 때문에 그런 거라면 같이 얘기하거나 해소 방법을 찾는 게 좋을 것 같아.', '그랬구나 ㅜㅜ 잘 위로해주고 응원해줘.'], ['너랑 얘기하다 보니까 그냥 결혼 안 하고 동물들이랑 사는 게 좋은가 싶기도 하다 야...', '무슨 뜻인지 잘 이해가 안 돼. 왜 사랑해서 결혼하는 게 기를 꺾는 거야? 좀 더 설명해줄래?', '그러게 말이다. 서로 잘 지내야 하는데 말이야.'], ['가만히 있는 건 아니니까! 움직이잖아', '건강은 우리 삶에서 정말 중요한 요소야. 건강하면 더 행복하고 활기찬 삶을 살 수 있어.', '운동이 되니까 몸에 좋아.'], ['그래도 그나마 그런 플라스틱은 눈에 보이니까 구조를 할 수 있었다지만 요새는 또 미세플라스틱이 논란이잖아요', '미안하다는 말을 하고 싶은 거야? 무슨 일이 있었는지 말해줄래?', '조심할 필요가 있어. 다음부터는 미안한 일 만들지 말자.']]\n"
          ]
        }
      ],
      "source": [
        "with open(EVALUATION_DATASET_PATH, 'r') as r:\n",
        "  eval_queries = json.load(r)\n",
        "  references = [query['references'] for query in eval_queries]\n",
        "\n",
        "with open(EVALUATED_INFERENCE_PATH, 'r') as rf:\n",
        "\tcandidates = rf.read().split('\\n')\n",
        "\n",
        "\n",
        "print(candidates[:5])\n",
        "print(references[:5])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q6LTBgJXOJ_I"
      },
      "source": [
        "# BLEU Score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7EftxpMcOHQX"
      },
      "source": [
        "## KKma with smoothing function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z-ACbm4lOSla"
      },
      "outputs": [],
      "source": [
        "from nltk.translate.bleu_score import corpus_bleu, sentence_bleu, SmoothingFunction\n",
        "from konlpy.tag import *\n",
        "\n",
        "kkma = Kkma()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-wAUNB1xOG1K",
        "outputId": "7aa44994-c20c-43d5-daa7-f06b4ec3e7dd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Smoothed BLEU score for sentence 1:  0.023956565612760213\n",
            "Smoothed BLEU score for sentence 2:  0.012673718536830811\n",
            "Smoothed BLEU score for sentence 3:  0.025281168697394947\n",
            "Smoothed BLEU score for sentence 4:  0.05344445934790546\n",
            "Smoothed BLEU score for sentence 5:  0.02603075873708404\n",
            "Smoothed BLEU score for sentence 6:  0.041081071978771734\n",
            "Smoothed BLEU score for sentence 7:  0.02101654884156187\n",
            "Smoothed BLEU score for sentence 8:  0.009678976922165645\n",
            "Smoothed BLEU score for sentence 9:  0.07786777655685517\n",
            "Smoothed BLEU score for sentence 10:  0.02625142896527897\n",
            "Smoothed BLEU score for sentence 11:  0.16731078418090023\n",
            "Smoothed BLEU score for sentence 12:  0.029176300840900796\n",
            "Smoothed BLEU score for sentence 13:  0.016451990399717707\n",
            "Smoothed BLEU score for sentence 14:  0.016562494897332036\n",
            "Smoothed BLEU score for sentence 15:  0.03440134693333791\n",
            "Smoothed BLEU score for sentence 16:  0.03058760346458022\n",
            "Smoothed BLEU score for sentence 17:  0.010088815311934719\n",
            "Smoothed BLEU score for sentence 18:  0.025400289715190986\n",
            "Smoothed BLEU score for sentence 19:  0.036374995670960676\n",
            "Smoothed BLEU score for sentence 20:  0.01618861356572822\n",
            "Smoothed BLEU score for sentence 21:  0.010893272947401535\n",
            "Smoothed BLEU score for sentence 22:  0.08442462941572415\n",
            "Smoothed BLEU score for sentence 23:  0.01067759367648771\n",
            "Smoothed BLEU score for sentence 24:  0.062284969540134474\n",
            "Smoothed BLEU score for sentence 25:  0.2545984531673679\n",
            "Smoothed BLEU score for sentence 26:  0.06439931429457921\n",
            "Smoothed BLEU score for sentence 27:  0.030064545690526145\n",
            "Smoothed BLEU score for sentence 28:  0.03621439574176347\n",
            "Smoothed BLEU score for sentence 29:  0.015815216252224023\n",
            "Smoothed BLEU score for sentence 30:  0.06770149544242768\n",
            "Smoothed BLEU score for sentence 31:  0.01618861356572822\n",
            "Smoothed BLEU score for sentence 32:  0.06142473685154091\n",
            "Smoothed BLEU score for sentence 33:  0.06146106554886093\n",
            "Smoothed BLEU score for sentence 34:  0.08291705932865436\n",
            "Smoothed BLEU score for sentence 35:  0.02642138995497447\n",
            "Smoothed BLEU score for sentence 36:  0.012990806323574158\n",
            "Smoothed BLEU score for sentence 37:  0.03825655967767164\n",
            "Smoothed BLEU score for sentence 38:  0.023233421968350106\n",
            "Smoothed BLEU score for sentence 39:  0.012673718536830811\n",
            "Smoothed BLEU score for sentence 40:  0.09528070628428423\n",
            "Smoothed BLEU score for sentence 41:  0.05550478662233608\n",
            "Smoothed BLEU score for sentence 42:  0.03994041449448101\n",
            "Smoothed BLEU score for sentence 43:  0.009037176334497899\n",
            "Smoothed BLEU score for sentence 44:  0.023233421968350106\n",
            "Smoothed BLEU score for sentence 45:  0.0264961961473786\n",
            "Smoothed BLEU score for sentence 46:  0.0236661293256479\n",
            "Smoothed BLEU score for sentence 47:  0.20380384515111735\n",
            "Smoothed BLEU score for sentence 48:  0.019765441873349214\n",
            "Smoothed BLEU score for sentence 49:  0.02277284496167085\n",
            "Smoothed BLEU score for sentence 50:  0.015047149445189388\n",
            "Smoothed BLEU score for sentence 51:  0.04841524713034601\n",
            "Smoothed BLEU score for sentence 52:  0.044147681940077975\n",
            "Smoothed BLEU score for sentence 53:  0.02016600660236953\n",
            "Smoothed BLEU score for sentence 54:  0.036362270465000714\n",
            "Smoothed BLEU score for sentence 55:  0.03250512812904205\n",
            "Smoothed BLEU score for sentence 56:  0.044353395455270214\n",
            "Smoothed BLEU score for sentence 57:  0.012990806323574158\n",
            "Smoothed BLEU score for sentence 58:  0.012285946074705657\n",
            "Smoothed BLEU score for sentence 59:  0.028517539529041507\n",
            "Smoothed BLEU score for sentence 60:  0.018790600374018796\n",
            "Smoothed BLEU score for sentence 61:  0.026658376817028855\n",
            "Smoothed BLEU score for sentence 62:  0.017339765977421966\n",
            "Smoothed BLEU score for sentence 63:  0.020200106912694155\n",
            "Smoothed BLEU score for sentence 64:  0.06024757292375468\n",
            "Smoothed BLEU score for sentence 65:  0.023482657122638518\n",
            "Smoothed BLEU score for sentence 66:  0.025281168697394947\n",
            "Smoothed BLEU score for sentence 67:  0.03592143420342892\n",
            "Smoothed BLEU score for sentence 68:  0.062284969540134474\n",
            "Smoothed BLEU score for sentence 69:  0.031529572123304546\n",
            "Smoothed BLEU score for sentence 70:  0.09033762441909504\n",
            "Smoothed BLEU score for sentence 71:  0.01935526899434156\n",
            "Smoothed BLEU score for sentence 72:  0.01757079221363004\n",
            "Smoothed BLEU score for sentence 73:  0.03219965714728962\n",
            "Smoothed BLEU score for sentence 74:  0.009680961449665638\n",
            "Smoothed BLEU score for sentence 75:  0.022416933501922302\n",
            "Smoothed BLEU score for sentence 76:  0.01626739260030573\n",
            "Smoothed BLEU score for sentence 77:  0.01794073254089892\n",
            "Smoothed BLEU score for sentence 78:  0.009186672262412125\n",
            "Smoothed BLEU score for sentence 79:  0.010331208012220438\n",
            "Smoothed BLEU score for sentence 80:  0.09242725334244041\n",
            "Smoothed BLEU score for sentence 81:  0.027748702735605823\n",
            "Smoothed BLEU score for sentence 82:  0.02814392937863503\n",
            "Smoothed BLEU score for sentence 83:  0.07075494147561419\n",
            "Smoothed BLEU score for sentence 84:  0.04127548532835144\n",
            "Smoothed BLEU score for sentence 85:  0.029222610172138452\n",
            "Smoothed BLEU score for sentence 86:  0.04740604259878138\n",
            "Smoothed BLEU score for sentence 87:  0.011371681934875243\n",
            "Smoothed BLEU score for sentence 88:  0.0223550930962921\n",
            "Smoothed BLEU score for sentence 89:  0.011383800122282609\n",
            "Smoothed BLEU score for sentence 90:  0.021972813874997166\n",
            "Smoothed BLEU score for sentence 91:  0.015413757980002104\n",
            "Smoothed BLEU score for sentence 92:  0.011856660123276007\n",
            "Smoothed BLEU score for sentence 93:  0.021972813874997166\n",
            "Smoothed BLEU score for sentence 94:  0.016562494897332036\n",
            "Smoothed BLEU score for sentence 95:  0.04048346741523882\n",
            "Smoothed BLEU score for sentence 96:  0.01763647856350297\n",
            "Smoothed BLEU score for sentence 97:  0.17181529671327242\n",
            "Smoothed BLEU score for sentence 98:  0.010351123016819801\n",
            "Smoothed BLEU score for sentence 99:  0.03182084684007777\n",
            "Smoothed BLEU score for sentence 100:  0.15955799528969325\n",
            "Avg BLEU score : 0.039553158579675715\n"
          ]
        }
      ],
      "source": [
        "smoothing = SmoothingFunction().method1\n",
        "\n",
        "score = 0\n",
        "\n",
        "for i in range(len(candidates)):\n",
        "    candidate = kkma.morphs(candidates[i])\n",
        "    reference = [kkma.morphs(ref) for ref in references[i]]\n",
        "\n",
        "    bleu_score_smooth = sentence_bleu(reference, candidate, smoothing_function=smoothing)\n",
        "    score += bleu_score_smooth\n",
        "\n",
        "    print(f'Smoothed BLEU score for sentence {i+1}: ', bleu_score_smooth)\n",
        "\n",
        "print(f'Avg BLEU score : {score / len(candidates)}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kvohKgrNZt43"
      },
      "source": [
        "## BERT Score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6oftQVdIZyd3",
        "outputId": "ea2acf7d-99e8-466e-9aa9-0e464557fabd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: bert-score in /usr/local/lib/python3.10/dist-packages (0.3.13)\n",
            "Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from bert-score) (2.0.1+cu118)\n",
            "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from bert-score) (1.5.3)\n",
            "Requirement already satisfied: transformers>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from bert-score) (4.29.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bert-score) (1.22.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bert-score) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.31.1 in /usr/local/lib/python3.10/dist-packages (from bert-score) (4.65.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from bert-score) (3.7.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from bert-score) (23.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score) (2022.7.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score) (3.12.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score) (16.0.5)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=3.0.0->bert-score) (0.14.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=3.0.0->bert-score) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=3.0.0->bert-score) (2022.10.31)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=3.0.0->bert-score) (0.13.3)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score) (1.0.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score) (4.39.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score) (1.4.4)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score) (8.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score) (3.0.9)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score) (3.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers>=3.0.0->bert-score) (2023.4.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas>=1.0.1->bert-score) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.0.0->bert-score) (2.1.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.0.0->bert-score) (1.3.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: rouge-score in /usr/local/lib/python3.10/dist-packages (0.1.2)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.4.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score) (3.8.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.22.4)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (8.1.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (2022.10.31)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (4.65.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install bert-score\n",
        "!pip install rouge-score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qaP9bVcUhnXG"
      },
      "source": [
        "## METEOR Score\n",
        "\n",
        "BLEU처럼 automated evaluation 방식.\n",
        "유의어를 반영할 수 없음.\n",
        "unigram에서 precision, recall의 조화평균 값이므로 candidate에서 같은 단어 잘 찍기만 하면 성능 올라감.\n",
        "\n",
        "candidate에 대응하는 reference들(3개)에 대해 각각 meteor score를 구한 후, 그 중 최댓값이 해당 candidate의 METEOR Score.\n",
        "\n",
        "전체 candidate의 METEOR Score의 평균을 계산할 예정.\n",
        "\n",
        "https://misconstructed.tistory.com/64"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rMfWCFILaUjY",
        "outputId": "d5289119-928a-4037-94c6-b727e0cf80a2"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UZQ0NaVlZz4l",
        "outputId": "92f07078-7092-436d-d190-33f5c14d9162"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "./drive/MyDrive/Models/eval_response_kogpt2F.txt 평균 METEOR 점수 : 0.1779385710150873\n"
          ]
        }
      ],
      "source": [
        "from nltk.translate import meteor_score\n",
        "from konlpy.tag import Kkma\n",
        "kkma = Kkma()\n",
        "\n",
        "sum_meteor_score = 0.0\n",
        "\n",
        "for i in range(len(candidates)):\n",
        "    candidate = kkma.morphs(candidates[i])\n",
        "    met_max = 0.0 \n",
        "\n",
        "    for j in range(3):\n",
        "      reference = kkma.morphs(references[i][j])\n",
        "      met_score  = meteor_score.single_meteor_score(reference, candidate)\n",
        "\n",
        "      if met_score > met_max:\n",
        "          met_max = met_score\n",
        "    \n",
        "    sum_meteor_score += met_max\n",
        "\n",
        "print(f\"{EVALUATED_INFERENCE_PATH} 평균 METEOR 점수 : {sum_meteor_score / len(candidates)}\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kXnDihIrYHKm"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4kMxTSyPiXPE"
      },
      "source": [
        "## Rouge Score\n",
        "Machine translation에서 주로 사용하는 BLEU가 n-gram Precision에 기반한 지표라면,\n",
        "\n",
        "ROUGE는 이름 그대로 n-gram Recall에 기반하여 계산됩니다.\n",
        "\n",
        "rougeL, rougeLSum은 순서에 영향을 받는 지표이기 때문에, 어순이 달라져도 의미가 통하는 한국어 특성 상 부정확한 지표입니다.\n",
        "\n",
        "hugging face library 사용했습니다.\n",
        "https://huggingface.co/spaces/evaluate-metric/rouge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oq2bBNqGoKoW",
        "outputId": "25616f90-1eb3-47d9-fcc9-9bbe4216f95d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (0.4.0)\n",
            "Requirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.12.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.22.4)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.3.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from evaluate) (4.65.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate) (3.2.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.70.14)\n",
            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2023.4.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.14.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from evaluate) (23.1)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.18.0)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (3.8.4)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.12.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.4)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2022.7.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install evaluate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hvUqjyApmN1B",
        "outputId": "40484fa8-5735-4a75-afe3-903b6fd1e8c5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "./drive/MyDrive/Models/eval_response_kogpt2F.txt ROUGE Score : {'rouge1': 0.016666666666666666, 'rouge2': 0.0, 'rougeL': 0.016666666666666666, 'rougeLsum': 0.016666666666666666}\n"
          ]
        }
      ],
      "source": [
        "import evaluate\n",
        "\n",
        "rouge = evaluate.load('rouge')\n",
        "results = rouge.compute(predictions=candidates, references=references)\n",
        "print(f\"{EVALUATED_INFERENCE_PATH} ROUGE Score : {results}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m62PLvrohy2c"
      },
      "source": [
        "## BERTScore\n",
        "BERT 모델을 이용해서 문장을 토크나이징함. 따라서 유의어 반영 가능!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "nSPJFBUVa9KK",
        "outputId": "44bcaae6-3609-4edb-ff6c-9dd8b028a7fb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "./drive/MyDrive/Models/eval_response_kogpt2F.txt BERT F1 score : 0.7283183652162551\n"
          ]
        }
      ],
      "source": [
        "from bert_score import score\n",
        "from transformers import logging\n",
        "logging.set_verbosity_error()\n",
        "\n",
        "sum_bert_score = 0.0\n",
        "\n",
        "for i in range(len(candidates)):\n",
        "  max_score = 0.0\n",
        "\n",
        "  for j in range(3):\n",
        "    P, R, F1 = score([candidates[i]], [references[i][j]], lang='ko', model_type='bert-base-multilingual-cased')\n",
        "    if max_score < F1.item():\n",
        "      max_score = F1.item()\n",
        "  \n",
        "  sum_bert_score += max_score\n",
        "  \n",
        "\n",
        "print(f\"{EVALUATED_INFERENCE_PATH} BERT F1 score : {sum_bert_score / len(candidates)}\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "toc_visible": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}