rating.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from glicko2 import glicko2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "556"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"review_gpt-4.jsonl\", \"r\", encoding=\"utf-8\") as f:\n",
    "    matches = [json.loads(line) for line in f.readlines()]\n",
    "len(matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11,\n",
       " dict_keys(['AIBunCho/japanese-novel-gpt-j-6b', 'GPT-3.5/ChatGPT-August-3', 'llm-jp/llm-jp-13b-instruct-full-dolly-oasst-v1.0', 'rinna/bilingual-gpt-neox-4b-instruction-ppo', 'supertrin-beta', 'GPT-4/ChatGPT-August-3', 'cyberagent/calm2-7b-chat', 'stabilityai/japanese-stablelm-instruct-alpha-7b-v2', 'stabilityai/japanese-stablelm-instruct-gamma-7b', 'line-corporation/japanese-large-lm-3.6b-instruction-sft', 'elyza/ELYZA-japanese-Llama-2-7b-fast-instruct']))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_id_list = list(\n",
    "    set(\n",
    "        [match[\"model_id_A\"] for match in matches]\n",
    "        + [match[\"model_id_B\"] for match in matches]\n",
    "    )\n",
    ")\n",
    "player_dict = {model_id: glicko2.Player() for model_id in model_id_list}\n",
    "len(player_dict), player_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for match in matches:\n",
    "    p1 = player_dict[match[\"model_id_A\"]]\n",
    "    p2 = player_dict[match[\"model_id_B\"]]\n",
    "\n",
    "    if match[\"model_id_A\"] == match[\"winner\"]:\n",
    "        p1.update_player([p2.rating], [p2.rd], [1])\n",
    "        p2.update_player([p1.rating], [p1.rd], [0])\n",
    "    else:\n",
    "        p1.update_player([p2.rating], [p2.rd], [0])\n",
    "        p2.update_player([p1.rating], [p1.rd], [1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>rd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>GPT-4/ChatGPT-August-3</td>\n",
       "      <td>1521.987940</td>\n",
       "      <td>73.695453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>supertrin-beta</td>\n",
       "      <td>1497.858852</td>\n",
       "      <td>66.674500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cyberagent/calm2-7b-chat</td>\n",
       "      <td>1420.753659</td>\n",
       "      <td>64.143678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GPT-3.5/ChatGPT-August-3</td>\n",
       "      <td>1376.120160</td>\n",
       "      <td>65.370350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>stabilityai/japanese-stablelm-instruct-gamma-7b</td>\n",
       "      <td>1251.056905</td>\n",
       "      <td>64.578941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>stabilityai/japanese-stablelm-instruct-alpha-7...</td>\n",
       "      <td>1248.081241</td>\n",
       "      <td>65.160776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>elyza/ELYZA-japanese-Llama-2-7b-fast-instruct</td>\n",
       "      <td>1178.005332</td>\n",
       "      <td>64.372931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>line-corporation/japanese-large-lm-3.6b-instru...</td>\n",
       "      <td>1145.851870</td>\n",
       "      <td>66.507452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AIBunCho/japanese-novel-gpt-j-6b</td>\n",
       "      <td>1142.043393</td>\n",
       "      <td>72.268411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>llm-jp/llm-jp-13b-instruct-full-dolly-oasst-v1.0</td>\n",
       "      <td>979.126761</td>\n",
       "      <td>76.788827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rinna/bilingual-gpt-neox-4b-instruction-ppo</td>\n",
       "      <td>975.860305</td>\n",
       "      <td>74.490569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             model_id       rating         rd\n",
       "5                              GPT-4/ChatGPT-August-3  1521.987940  73.695453\n",
       "4                                      supertrin-beta  1497.858852  66.674500\n",
       "6                            cyberagent/calm2-7b-chat  1420.753659  64.143678\n",
       "1                            GPT-3.5/ChatGPT-August-3  1376.120160  65.370350\n",
       "8     stabilityai/japanese-stablelm-instruct-gamma-7b  1251.056905  64.578941\n",
       "7   stabilityai/japanese-stablelm-instruct-alpha-7...  1248.081241  65.160776\n",
       "10      elyza/ELYZA-japanese-Llama-2-7b-fast-instruct  1178.005332  64.372931\n",
       "9   line-corporation/japanese-large-lm-3.6b-instru...  1145.851870  66.507452\n",
       "0                    AIBunCho/japanese-novel-gpt-j-6b  1142.043393  72.268411\n",
       "2    llm-jp/llm-jp-13b-instruct-full-dolly-oasst-v1.0   979.126761  76.788827\n",
       "3         rinna/bilingual-gpt-neox-4b-instruction-ppo   975.860305  74.490569"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = []\n",
    "for model_id, player in player_dict.items():\n",
    "    results.append([model_id, player.rating, player.rd])\n",
    "df = pd.DataFrame(results, columns=[\"model_id\", \"rating\", \"rd\"]).sort_values(\n",
    "    by=\"rating\", ascending=False\n",
    ")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Rating ± RD\"] = df.apply(\n",
    "    lambda row: f'{row[\"rating\"]:.0f} ± {row[\"rd\"]:.0f}', axis=1\n",
    ")\n",
    "df[\"ERP Score\"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| model_id                                                | Rating ± RD   | ERP Score   |\n",
      "|:--------------------------------------------------------|:--------------|:------------|\n",
      "| GPT-4/ChatGPT-August-3                                  | 1522 ± 74     |             |\n",
      "| supertrin-beta                                          | 1498 ± 67     |             |\n",
      "| cyberagent/calm2-7b-chat                                | 1421 ± 64     |             |\n",
      "| GPT-3.5/ChatGPT-August-3                                | 1376 ± 65     |             |\n",
      "| stabilityai/japanese-stablelm-instruct-gamma-7b         | 1251 ± 65     |             |\n",
      "| stabilityai/japanese-stablelm-instruct-alpha-7b-v2      | 1248 ± 65     |             |\n",
      "| elyza/ELYZA-japanese-Llama-2-7b-fast-instruct           | 1178 ± 64     |             |\n",
      "| line-corporation/japanese-large-lm-3.6b-instruction-sft | 1146 ± 67     |             |\n",
      "| AIBunCho/japanese-novel-gpt-j-6b                        | 1142 ± 72     |             |\n",
      "| llm-jp/llm-jp-13b-instruct-full-dolly-oasst-v1.0        | 979 ± 77      |             |\n",
      "| rinna/bilingual-gpt-neox-4b-instruction-ppo             | 976 ± 74      |             |\n"
     ]
    }
   ],
   "source": [
    "print(df[[\"model_id\", \"Rating ± RD\", \"ERP Score\"]].to_markdown(index=False))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}