applications/DeepSpeed-Chat/training/step1_supervised_finetuning/FastChat/fastchat/llm_judge/noteboooks/mt_bench_radar.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4eYlKr9RrPu2",
        "outputId": "59f3cbfd-988a-42ea-c94d-4a08b3058e60"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting plotly\n",
            "  Downloading plotly-5.15.0-py2.py3-none-any.whl (15.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.5/15.5 MB\u001b[0m \u001b[31m66.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hCollecting kaleido\n",
            "  Downloading kaleido-0.2.1-py2.py3-none-manylinux1_x86_64.whl (79.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.9/79.9 MB\u001b[0m \u001b[31m34.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tenacity>=6.2.0 in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from plotly) (8.2.2)\n",
            "Requirement already satisfied: packaging in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from plotly) (23.1)\n",
            "Installing collected packages: kaleido, plotly\n",
            "Successfully installed kaleido-0.2.1 plotly-5.15.0\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install -U plotly kaleido"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "m2tG_vDyqWZw"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "import plotly.express as px\n",
        "import plotly.graph_objects as go\n",
        "\n",
        "\n",
        "CATEGORIES = [\n",
        "    \"Writing\",\n",
        "    \"Roleplay\",\n",
        "    \"Reasoning\",\n",
        "    \"Math\",\n",
        "    \"Coding\",\n",
        "    \"Extraction\",\n",
        "    \"STEM\",\n",
        "    \"Humanities\",\n",
        "]\n",
        "\n",
        "\n",
        "def get_model_df():\n",
        "    cnt = 0\n",
        "    q2result = []\n",
        "    fin = open(\"../data/mt_bench/model_judgment/gpt-4_single.jsonl\", \"r\")\n",
        "    for line in fin:\n",
        "        obj = json.loads(line)\n",
        "        obj[\"category\"] = CATEGORIES[(obj[\"question_id\"] - 81) // 10]\n",
        "        q2result.append(obj)\n",
        "    df = pd.DataFrame(q2result)\n",
        "    return df\n",
        "\n",
        "\n",
        "def toggle(res_str):\n",
        "    if res_str == \"win\":\n",
        "        return \"loss\"\n",
        "    elif res_str == \"loss\":\n",
        "        return \"win\"\n",
        "    return \"tie\"\n",
        "\n",
        "\n",
        "def get_model_df_pair():\n",
        "    fin = open(\"gpt-4_pair.jsonl\", \"r\")\n",
        "    cnt = 0\n",
        "    q2result = []\n",
        "    for line in fin:\n",
        "        obj = json.loads(line)\n",
        "\n",
        "        result = {}\n",
        "        result[\"qid\"] = str(obj[\"question_id\"])\n",
        "        result[\"turn\"] = str(obj[\"turn\"])\n",
        "        if obj[\"g1_winner\"] == \"model_1\" and obj[\"g2_winner\"] == \"model_1\":\n",
        "            result[\"result\"] = \"win\"\n",
        "        elif obj[\"g1_winner\"] == \"model_2\" and obj[\"g2_winner\"] == \"model_2\":\n",
        "            result[\"result\"] = \"loss\"\n",
        "        else:\n",
        "            result[\"result\"] = \"tie\"\n",
        "        result[\"category\"] = CATEGORIES[(obj[\"question_id\"] - 81) // 10]\n",
        "        result[\"model\"] = obj[\"model_1\"]\n",
        "        q2result.append(result)\n",
        "\n",
        "    df = pd.DataFrame(q2result)\n",
        "\n",
        "    return df\n",
        "\n",
        "\n",
        "df = get_model_df()\n",
        "# df_pair = get_model_df_pair()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "wUw1sxfmaGuK",
        "outputId": "974a5755-ebe2-4bdb-c43e-769aa38b5b29"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question_id</th>\n",
              "      <th>model</th>\n",
              "      <th>judge</th>\n",
              "      <th>user_prompt</th>\n",
              "      <th>judgment</th>\n",
              "      <th>score</th>\n",
              "      <th>turn</th>\n",
              "      <th>tstamp</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>81</td>\n",
              "      <td>saiga_7b_v1</td>\n",
              "      <td>[gpt-4, single-v1]</td>\n",
              "      <td>[Instruction]\\nPlease act as an impartial judg...</td>\n",
              "      <td>The assistant's response is highly detailed, e...</td>\n",
              "      <td>10.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1.689691e+09</td>\n",
              "      <td>Writing</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>81</td>\n",
              "      <td>saiga_7b_v1</td>\n",
              "      <td>[gpt-4, single-v1-multi-turn]</td>\n",
              "      <td>&lt;|The Start of Assistant A's Conversation with...</td>\n",
              "      <td>The assistant's response is not relevant or he...</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2</td>\n",
              "      <td>1.689691e+09</td>\n",
              "      <td>Writing</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>81</td>\n",
              "      <td>saiga_7b_v1</td>\n",
              "      <td>[gpt-4, single-v1]</td>\n",
              "      <td>[Instruction]\\nPlease act as an impartial judg...</td>\n",
              "      <td>The assistant's response is highly detailed, a...</td>\n",
              "      <td>10.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1.689692e+09</td>\n",
              "      <td>Writing</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   question_id        model                          judge  \\\n",
              "0           81  saiga_7b_v1             [gpt-4, single-v1]   \n",
              "1           81  saiga_7b_v1  [gpt-4, single-v1-multi-turn]   \n",
              "2           81  saiga_7b_v1             [gpt-4, single-v1]   \n",
              "\n",
              "                                         user_prompt  \\\n",
              "0  [Instruction]\\nPlease act as an impartial judg...   \n",
              "1  <|The Start of Assistant A's Conversation with...   \n",
              "2  [Instruction]\\nPlease act as an impartial judg...   \n",
              "\n",
              "                                            judgment  score  turn  \\\n",
              "0  The assistant's response is highly detailed, e...   10.0     1   \n",
              "1  The assistant's response is not relevant or he...    1.0     2   \n",
              "2  The assistant's response is highly detailed, a...   10.0     1   \n",
              "\n",
              "         tstamp category  \n",
              "0  1.689691e+09  Writing  \n",
              "1  1.689691e+09  Writing  \n",
              "2  1.689692e+09  Writing  "
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MpBKLuVmqZ7O",
        "outputId": "80477e24-2132-4d2f-b23a-ee52cf74c86b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['saiga_7b_v1' 'saiga_7b_v2' 'xglm_4.5b_v10_epoch_6_step_41141'\n",
            " 'saiga_7b_v2_ru' 'saiga_7b_v1_ru' 'xglm_4.5b_v10_epoch_6_step_41141_ru'\n",
            " 'mt_bench_ru_xglm_4.5B_lora_our_dataset'\n",
            " 'mt_bench_en_xglm_4.5B_lora_our_dataset' 'mt_bench_ru_gigachat'\n",
            " 'mt_bench_en_gigachat' 'mt_bench_ru_saiga2_7b_our_dataset'\n",
            " 'mt_bench_ru_saiga2_13b_our_dataset' 'mt_bench_ru_saiga2_7b'\n",
            " 'mt_bench_ru_saiga2_13b' 'mt_bench_ru_gigasaiga_13b'\n",
            " 'mt_bench_ru_rugpt_13B_our_dataset'\n",
            " 'mt_bench_ru_xglm_4.5B_lora_saiga_dataset' 'mt_bench_ru_yandexgpt']\n"
          ]
        }
      ],
      "source": [
        "all_models = df[\"model\"].unique()\n",
        "print(all_models)\n",
        "scores_all = []\n",
        "for model in all_models:\n",
        "    for cat in CATEGORIES:\n",
        "        # filter category/model, and score format error (<1% case)\n",
        "        res = df[(df[\"category\"] == cat) & (df[\"model\"] == model) & (df[\"score\"] >= 0)]\n",
        "        score = res[\"score\"].mean()\n",
        "\n",
        "        # # pairwise result\n",
        "        # res_pair = df_pair[(df_pair[\"category\"]==cat) & (df_pair[\"model\"]==model)][\"result\"].value_counts()\n",
        "        # wincnt = res_pair[\"win\"] if \"win\" in res_pair.index else 0\n",
        "        # tiecnt = res_pair[\"tie\"] if \"tie\" in res_pair.index else 0\n",
        "        # winrate = wincnt/res_pair.sum()\n",
        "        # winrate_adjusted = (wincnt + tiecnt)/res_pair.sum()\n",
        "        # # print(winrate_adjusted)\n",
        "\n",
        "        # scores_all.append({\"model\": model, \"category\": cat, \"score\": score, \"winrate\": winrate, \"wtrate\": winrate_adjusted})\n",
        "        scores_all.append({\"model\": model, \"category\": cat, \"score\": score})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "5i8R0l-XqkgO",
        "outputId": "0bc4b3fe-835a-4a4d-aa1d-4f49772ce95f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[{'model': 'saiga_7b_v1_ru', 'category': 'Writing', 'score': 4.95}, {'model': 'saiga_7b_v1_ru', 'category': 'Roleplay', 'score': 4.2}, {'model': 'saiga_7b_v1_ru', 'category': 'Reasoning', 'score': 2.25}, {'model': 'saiga_7b_v1_ru', 'category': 'Math', 'score': 1.7}, {'model': 'saiga_7b_v1_ru', 'category': 'Coding', 'score': 1.95}, {'model': 'saiga_7b_v1_ru', 'category': 'Extraction', 'score': 2.65}, {'model': 'saiga_7b_v1_ru', 'category': 'STEM', 'score': 4.95}, {'model': 'saiga_7b_v1_ru', 'category': 'Humanities', 'score': 6.55}, {'model': 'mt_bench_ru_gigachat', 'category': 'Writing', 'score': 6.1}, {'model': 'mt_bench_ru_gigachat', 'category': 'Roleplay', 'score': 5.4}, {'model': 'mt_bench_ru_gigachat', 'category': 'Reasoning', 'score': 4.1}, {'model': 'mt_bench_ru_gigachat', 'category': 'Math', 'score': 1.65}, {'model': 'mt_bench_ru_gigachat', 'category': 'Coding', 'score': 1.8}, {'model': 'mt_bench_ru_gigachat', 'category': 'Extraction', 'score': 1.65}, {'model': 'mt_bench_ru_gigachat', 'category': 'STEM', 'score': 5.15}, {'model': 'mt_bench_ru_gigachat', 'category': 'Humanities', 'score': 6.45}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Writing', 'score': 4.9}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Roleplay', 'score': 5.05}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Reasoning', 'score': 2.6}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Math', 'score': 1.5}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Coding', 'score': 2.7}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Extraction', 'score': 3.8}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'STEM', 'score': 5.6}, {'model': 'mt_bench_ru_saiga2_7b', 'category': 'Humanities', 'score': 7.4}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Writing', 'score': 6.1}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Roleplay', 'score': 5.8}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Reasoning', 'score': 3.35}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Math', 'score': 1.9}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Coding', 'score': 3.6}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Extraction', 'score': 5.25}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'STEM', 'score': 6.45}, {'model': 'mt_bench_ru_saiga2_13b', 'category': 'Humanities', 'score': 7.9}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Writing', 'score': 4.7}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Roleplay', 'score': 5.05}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Reasoning', 'score': 2.6}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Math', 'score': 1.05}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Coding', 'score': 1.55}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Extraction', 'score': 1.45}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'STEM', 'score': 3.95}, {'model': 'mt_bench_ru_gigasaiga_13b', 'category': 'Humanities', 'score': 5.5}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Writing', 'score': 5.15}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Roleplay', 'score': 5.05}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Reasoning', 'score': 2.45}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Math', 'score': 1.45}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Coding', 'score': 2.45}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Extraction', 'score': 4.65}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'STEM', 'score': 7.2}, {'model': 'mt_bench_ru_yandexgpt', 'category': 'Humanities', 'score': 7.3}]\n"
          ]
        },
        {
          "data": {
            "application/vnd.plotly.v1+json": {
              "config": {
                "plotlyServerURL": "https://plot.ly"
              },
              "data": [
                {
                  "hovertemplate": "model=saiga_7b<br>score=%{r}<br>category=%{theta}<extra></extra>",
                  "legendgroup": "saiga_7b",
                  "line": {
                    "color": "rgb(102, 197, 204)",
                    "dash": "solid"
                  },
                  "marker": {
                    "symbol": "circle"
                  },
                  "mode": "lines+markers",
                  "name": "saiga_7b",
                  "r": [
                    4.95,
                    4.2,
                    2.25,
                    1.7,
                    1.95,
                    2.65,
                    4.95,
                    6.55,
                    4.95
                  ],
                  "showlegend": true,
                  "subplot": "polar",
                  "theta": [
                    "Writing",
                    "Roleplay",
                    "Reasoning",
                    "Math",
                    "Coding",
                    "Extraction",
                    "STEM",
                    "Humanities",
                    "Writing"
                  ],
                  "type": "scatterpolar"
                },
                {
                  "hovertemplate": "model=mt_bench_ru_yandexgpt<br>score=%{r}<br>category=%{theta}<extra></extra>",
                  "legendgroup": "mt_bench_ru_yandexgpt",
                  "line": {
                    "color": "rgb(246, 207, 113)",
                    "dash": "solid"
                  },
                  "marker": {
                    "symbol": "circle"
                  },
                  "mode": "lines+markers",
                  "name": "mt_bench_ru_yandexgpt",
                  "r": [
                    5.15,
                    5.05,
                    2.45,
                    1.45,
                    2.45,
                    4.65,
                    7.2,
                    7.3,
                    5.15
                  ],
                  "showlegend": true,
                  "subplot": "polar",
                  "theta": [
                    "Writing",
                    "Roleplay",
                    "Reasoning",
                    "Math",
                    "Coding",
                    "Extraction",
                    "STEM",
                    "Humanities",
                    "Writing"
                  ],
                  "type": "scatterpolar"
                },
                {
                  "hovertemplate": "model=mt_bench_ru_saiga2_7b<br>score=%{r}<br>category=%{theta}<extra></extra>",
                  "legendgroup": "mt_bench_ru_saiga2_7b",
                  "line": {
                    "color": "rgb(248, 156, 116)",
                    "dash": "solid"
                  },
                  "marker": {
                    "symbol": "circle"
                  },
                  "mode": "lines+markers",
                  "name": "mt_bench_ru_saiga2_7b",
                  "r": [
                    4.9,
                    5.05,
                    2.6,
                    1.5,
                    2.7,
                    3.8,
                    5.6,
                    7.4,
                    4.9
                  ],
                  "showlegend": true,
                  "subplot": "polar",
                  "theta": [
                    "Writing",
                    "Roleplay",
                    "Reasoning",
                    "Math",
                    "Coding",
                    "Extraction",
                    "STEM",
                    "Humanities",
                    "Writing"
                  ],
                  "type": "scatterpolar"
                },
                {
                  "hovertemplate": "model=mt_bench_ru_saiga2_13b<br>score=%{r}<br>category=%{theta}<extra></extra>",
                  "legendgroup": "mt_bench_ru_saiga2_13b",
                  "line": {
                    "color": "rgb(220, 176, 242)",
                    "dash": "solid"
                  },
                  "marker": {
                    "symbol": "circle"
                  },
                  "mode": "lines+markers",
                  "name": "mt_bench_ru_saiga2_13b",
                  "r": [
                    6.1,
                    5.8,
                    3.35,
                    1.9,
                    3.6,
                    5.25,
                    6.45,
                    7.9,
                    6.1
                  ],
                  "showlegend": true,
                  "subplot": "polar",
                  "theta": [
                    "Writing",
                    "Roleplay",
                    "Reasoning",
                    "Math",
                    "Coding",
                    "Extraction",
                    "STEM",
                    "Humanities",
                    "Writing"
                  ],
                  "type": "scatterpolar"
                },
                {
                  "hovertemplate": "model=mt_bench_ru_gigasaiga_13b<br>score=%{r}<br>category=%{theta}<extra></extra>",
                  "legendgroup": "mt_bench_ru_gigasaiga_13b",
                  "line": {
                    "color": "rgb(135, 197, 95)",
                    "dash": "solid"
                  },
                  "marker": {
                    "symbol": "circle"
                  },
                  "mode": "lines+markers",
                  "name": "mt_bench_ru_gigasaiga_13b",
                  "r": [
                    4.7,
                    5.05,
                    2.6,
                    1.05,
                    1.55,
                    1.45,
                    3.95,
                    5.5,
                    4.7
                  ],
                  "showlegend": true,
                  "subplot": "polar",
                  "theta": [
                    "Writing",
                    "Roleplay",
                    "Reasoning",
                    "Math",
                    "Coding",
                    "Extraction",
                    "STEM",
                    "Humanities",
                    "Writing"
                  ],
                  "type": "scatterpolar"
                },
                {
                  "hovertemplate": "model=mt_bench_ru_gigachat<br>score=%{r}<br>category=%{theta}<extra></extra>",
                  "legendgroup": "mt_bench_ru_gigachat",
                  "line": {
                    "color": "rgb(158, 185, 243)",
                    "dash": "solid"
                  },
                  "marker": {
                    "symbol": "circle"
                  },
                  "mode": "lines+markers",
                  "name": "mt_bench_ru_gigachat",
                  "r": [
                    6.1,
                    5.4,
                    4.1,
                    1.65,
                    1.8,
                    1.65,
                    5.15,
                    6.45,
                    6.1
                  ],
                  "showlegend": true,
                  "subplot": "polar",
                  "theta": [
                    "Writing",
                    "Roleplay",
                    "Reasoning",
                    "Math",
                    "Coding",
                    "Extraction",
                    "STEM",
                    "Humanities",
                    "Writing"
                  ],
                  "type": "scatterpolar"
                }
              ],
              "layout": {
                "legend": {
                  "title": {
                    "text": "model"
                  },
                  "tracegroupgap": 0
                },
                "margin": {
                  "t": 60
                },
                "polar": {
                  "angularaxis": {
                    "categoryarray": [
                      "Writing",
                      "Roleplay",
                      "Reasoning",
                      "Math",
                      "Coding",
                      "Extraction",
                      "STEM",
                      "Humanities"
                    ],
                    "categoryorder": "array",
                    "direction": "clockwise",
                    "rotation": 90
                  },
                  "domain": {
                    "x": [
                      0,
                      1
                    ],
                    "y": [
                      0,
                      1
                    ]
                  }
                },
                "template": {
                  "data": {
                    "bar": [
                      {
                        "error_x": {
                          "color": "#2a3f5f"
                        },
                        "error_y": {
                          "color": "#2a3f5f"
                        },
                        "marker": {
                          "line": {
                            "color": "#E5ECF6",
                            "width": 0.5
                          },
                          "pattern": {
                            "fillmode": "overlay",
                            "size": 10,
                            "solidity": 0.2
                          }
                        },
                        "type": "bar"
                      }
                    ],
                    "barpolar": [
                      {
                        "marker": {
                          "line": {
                            "color": "#E5ECF6",
                            "width": 0.5
                          },
                          "pattern": {
                            "fillmode": "overlay",
                            "size": 10,
                            "solidity": 0.2
                          }
                        },
                        "type": "barpolar"
                      }
                    ],
                    "carpet": [
                      {
                        "aaxis": {
                          "endlinecolor": "#2a3f5f",
                          "gridcolor": "white",
                          "linecolor": "white",
                          "minorgridcolor": "white",
                          "startlinecolor": "#2a3f5f"
                        },
                        "baxis": {
                          "endlinecolor": "#2a3f5f",
                          "gridcolor": "white",
                          "linecolor": "white",
                          "minorgridcolor": "white",
                          "startlinecolor": "#2a3f5f"
                        },
                        "type": "carpet"
                      }
                    ],
                    "choropleth": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "type": "choropleth"
                      }
                    ],
                    "contour": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "colorscale": [
                          [
                            0,
                            "#0d0887"
                          ],
                          [
                            0.1111111111111111,
                            "#46039f"
                          ],
                          [
                            0.2222222222222222,
                            "#7201a8"
                          ],
                          [
                            0.3333333333333333,
                            "#9c179e"
                          ],
                          [
                            0.4444444444444444,
                            "#bd3786"
                          ],
                          [
                            0.5555555555555556,
                            "#d8576b"
                          ],
                          [
                            0.6666666666666666,
                            "#ed7953"
                          ],
                          [
                            0.7777777777777778,
                            "#fb9f3a"
                          ],
                          [
                            0.8888888888888888,
                            "#fdca26"
                          ],
                          [
                            1,
                            "#f0f921"
                          ]
                        ],
                        "type": "contour"
                      }
                    ],
                    "contourcarpet": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "type": "contourcarpet"
                      }
                    ],
                    "heatmap": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "colorscale": [
                          [
                            0,
                            "#0d0887"
                          ],
                          [
                            0.1111111111111111,
                            "#46039f"
                          ],
                          [
                            0.2222222222222222,
                            "#7201a8"
                          ],
                          [
                            0.3333333333333333,
                            "#9c179e"
                          ],
                          [
                            0.4444444444444444,
                            "#bd3786"
                          ],
                          [
                            0.5555555555555556,
                            "#d8576b"
                          ],
                          [
                            0.6666666666666666,
                            "#ed7953"
                          ],
                          [
                            0.7777777777777778,
                            "#fb9f3a"
                          ],
                          [
                            0.8888888888888888,
                            "#fdca26"
                          ],
                          [
                            1,
                            "#f0f921"
                          ]
                        ],
                        "type": "heatmap"
                      }
                    ],
                    "heatmapgl": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "colorscale": [
                          [
                            0,
                            "#0d0887"
                          ],
                          [
                            0.1111111111111111,
                            "#46039f"
                          ],
                          [
                            0.2222222222222222,
                            "#7201a8"
                          ],
                          [
                            0.3333333333333333,
                            "#9c179e"
                          ],
                          [
                            0.4444444444444444,
                            "#bd3786"
                          ],
                          [
                            0.5555555555555556,
                            "#d8576b"
                          ],
                          [
                            0.6666666666666666,
                            "#ed7953"
                          ],
                          [
                            0.7777777777777778,
                            "#fb9f3a"
                          ],
                          [
                            0.8888888888888888,
                            "#fdca26"
                          ],
                          [
                            1,
                            "#f0f921"
                          ]
                        ],
                        "type": "heatmapgl"
                      }
                    ],
                    "histogram": [
                      {
                        "marker": {
                          "pattern": {
                            "fillmode": "overlay",
                            "size": 10,
                            "solidity": 0.2
                          }
                        },
                        "type": "histogram"
                      }
                    ],
                    "histogram2d": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "colorscale": [
                          [
                            0,
                            "#0d0887"
                          ],
                          [
                            0.1111111111111111,
                            "#46039f"
                          ],
                          [
                            0.2222222222222222,
                            "#7201a8"
                          ],
                          [
                            0.3333333333333333,
                            "#9c179e"
                          ],
                          [
                            0.4444444444444444,
                            "#bd3786"
                          ],
                          [
                            0.5555555555555556,
                            "#d8576b"
                          ],
                          [
                            0.6666666666666666,
                            "#ed7953"
                          ],
                          [
                            0.7777777777777778,
                            "#fb9f3a"
                          ],
                          [
                            0.8888888888888888,
                            "#fdca26"
                          ],
                          [
                            1,
                            "#f0f921"
                          ]
                        ],
                        "type": "histogram2d"
                      }
                    ],
                    "histogram2dcontour": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "colorscale": [
                          [
                            0,
                            "#0d0887"
                          ],
                          [
                            0.1111111111111111,
                            "#46039f"
                          ],
                          [
                            0.2222222222222222,
                            "#7201a8"
                          ],
                          [
                            0.3333333333333333,
                            "#9c179e"
                          ],
                          [
                            0.4444444444444444,
                            "#bd3786"
                          ],
                          [
                            0.5555555555555556,
                            "#d8576b"
                          ],
                          [
                            0.6666666666666666,
                            "#ed7953"
                          ],
                          [
                            0.7777777777777778,
                            "#fb9f3a"
                          ],
                          [
                            0.8888888888888888,
                            "#fdca26"
                          ],
                          [
                            1,
                            "#f0f921"
                          ]
                        ],
                        "type": "histogram2dcontour"
                      }
                    ],
                    "mesh3d": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "type": "mesh3d"
                      }
                    ],
                    "parcoords": [
                      {
                        "line": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "parcoords"
                      }
                    ],
                    "pie": [
                      {
                        "automargin": true,
                        "type": "pie"
                      }
                    ],
                    "scatter": [
                      {
                        "fillpattern": {
                          "fillmode": "overlay",
                          "size": 10,
                          "solidity": 0.2
                        },
                        "type": "scatter"
                      }
                    ],
                    "scatter3d": [
                      {
                        "line": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scatter3d"
                      }
                    ],
                    "scattercarpet": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scattercarpet"
                      }
                    ],
                    "scattergeo": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scattergeo"
                      }
                    ],
                    "scattergl": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scattergl"
                      }
                    ],
                    "scattermapbox": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scattermapbox"
                      }
                    ],
                    "scatterpolar": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scatterpolar"
                      }
                    ],
                    "scatterpolargl": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scatterpolargl"
                      }
                    ],
                    "scatterternary": [
                      {
                        "marker": {
                          "colorbar": {
                            "outlinewidth": 0,
                            "ticks": ""
                          }
                        },
                        "type": "scatterternary"
                      }
                    ],
                    "surface": [
                      {
                        "colorbar": {
                          "outlinewidth": 0,
                          "ticks": ""
                        },
                        "colorscale": [
                          [
                            0,
                            "#0d0887"
                          ],
                          [
                            0.1111111111111111,
                            "#46039f"
                          ],
                          [
                            0.2222222222222222,
                            "#7201a8"
                          ],
                          [
                            0.3333333333333333,
                            "#9c179e"
                          ],
                          [
                            0.4444444444444444,
                            "#bd3786"
                          ],
                          [
                            0.5555555555555556,
                            "#d8576b"
                          ],
                          [
                            0.6666666666666666,
                            "#ed7953"
                          ],
                          [
                            0.7777777777777778,
                            "#fb9f3a"
                          ],
                          [
                            0.8888888888888888,
                            "#fdca26"
                          ],
                          [
                            1,
                            "#f0f921"
                          ]
                        ],
                        "type": "surface"
                      }
                    ],
                    "table": [
                      {
                        "cells": {
                          "fill": {
                            "color": "#EBF0F8"
                          },
                          "line": {
                            "color": "white"
                          }
                        },
                        "header": {
                          "fill": {
                            "color": "#C8D4E3"
                          },
                          "line": {
                            "color": "white"
                          }
                        },
                        "type": "table"
                      }
                    ]
                  },
                  "layout": {
                    "annotationdefaults": {
                      "arrowcolor": "#2a3f5f",
                      "arrowhead": 0,
                      "arrowwidth": 1
                    },
                    "autotypenumbers": "strict",
                    "coloraxis": {
                      "colorbar": {
                        "outlinewidth": 0,
                        "ticks": ""
                      }
                    },
                    "colorscale": {
                      "diverging": [
                        [
                          0,
                          "#8e0152"
                        ],
                        [
                          0.1,
                          "#c51b7d"
                        ],
                        [
                          0.2,
                          "#de77ae"
                        ],
                        [
                          0.3,
                          "#f1b6da"
                        ],
                        [
                          0.4,
                          "#fde0ef"
                        ],
                        [
                          0.5,
                          "#f7f7f7"
                        ],
                        [
                          0.6,
                          "#e6f5d0"
                        ],
                        [
                          0.7,
                          "#b8e186"
                        ],
                        [
                          0.8,
                          "#7fbc41"
                        ],
                        [
                          0.9,
                          "#4d9221"
                        ],
                        [
                          1,
                          "#276419"
                        ]
                      ],
                      "sequential": [
                        [
                          0,
                          "#0d0887"
                        ],
                        [
                          0.1111111111111111,
                          "#46039f"
                        ],
                        [
                          0.2222222222222222,
                          "#7201a8"
                        ],
                        [
                          0.3333333333333333,
                          "#9c179e"
                        ],
                        [
                          0.4444444444444444,
                          "#bd3786"
                        ],
                        [
                          0.5555555555555556,
                          "#d8576b"
                        ],
                        [
                          0.6666666666666666,
                          "#ed7953"
                        ],
                        [
                          0.7777777777777778,
                          "#fb9f3a"
                        ],
                        [
                          0.8888888888888888,
                          "#fdca26"
                        ],
                        [
                          1,
                          "#f0f921"
                        ]
                      ],
                      "sequentialminus": [
                        [
                          0,
                          "#0d0887"
                        ],
                        [
                          0.1111111111111111,
                          "#46039f"
                        ],
                        [
                          0.2222222222222222,
                          "#7201a8"
                        ],
                        [
                          0.3333333333333333,
                          "#9c179e"
                        ],
                        [
                          0.4444444444444444,
                          "#bd3786"
                        ],
                        [
                          0.5555555555555556,
                          "#d8576b"
                        ],
                        [
                          0.6666666666666666,
                          "#ed7953"
                        ],
                        [
                          0.7777777777777778,
                          "#fb9f3a"
                        ],
                        [
                          0.8888888888888888,
                          "#fdca26"
                        ],
                        [
                          1,
                          "#f0f921"
                        ]
                      ]
                    },
                    "colorway": [
                      "#636efa",
                      "#EF553B",
                      "#00cc96",
                      "#ab63fa",
                      "#FFA15A",
                      "#19d3f3",
                      "#FF6692",
                      "#B6E880",
                      "#FF97FF",
                      "#FECB52"
                    ],
                    "font": {
                      "color": "#2a3f5f"
                    },
                    "geo": {
                      "bgcolor": "white",
                      "lakecolor": "white",
                      "landcolor": "#E5ECF6",
                      "showlakes": true,
                      "showland": true,
                      "subunitcolor": "white"
                    },
                    "hoverlabel": {
                      "align": "left"
                    },
                    "hovermode": "closest",
                    "mapbox": {
                      "style": "light"
                    },
                    "paper_bgcolor": "white",
                    "plot_bgcolor": "#E5ECF6",
                    "polar": {
                      "angularaxis": {
                        "gridcolor": "white",
                        "linecolor": "white",
                        "ticks": ""
                      },
                      "bgcolor": "#E5ECF6",
                      "radialaxis": {
                        "gridcolor": "white",
                        "linecolor": "white",
                        "ticks": ""
                      }
                    },
                    "scene": {
                      "xaxis": {
                        "backgroundcolor": "#E5ECF6",
                        "gridcolor": "white",
                        "gridwidth": 2,
                        "linecolor": "white",
                        "showbackground": true,
                        "ticks": "",
                        "zerolinecolor": "white"
                      },
                      "yaxis": {
                        "backgroundcolor": "#E5ECF6",
                        "gridcolor": "white",
                        "gridwidth": 2,
                        "linecolor": "white",
                        "showbackground": true,
                        "ticks": "",
                        "zerolinecolor": "white"
                      },
                      "zaxis": {
                        "backgroundcolor": "#E5ECF6",
                        "gridcolor": "white",
                        "gridwidth": 2,
                        "linecolor": "white",
                        "showbackground": true,
                        "ticks": "",
                        "zerolinecolor": "white"
                      }
                    },
                    "shapedefaults": {
                      "line": {
                        "color": "#2a3f5f"
                      }
                    },
                    "ternary": {
                      "aaxis": {
                        "gridcolor": "white",
                        "linecolor": "white",
                        "ticks": ""
                      },
                      "baxis": {
                        "gridcolor": "white",
                        "linecolor": "white",
                        "ticks": ""
                      },
                      "bgcolor": "#E5ECF6",
                      "caxis": {
                        "gridcolor": "white",
                        "linecolor": "white",
                        "ticks": ""
                      }
                    },
                    "title": {
                      "x": 0.05
                    },
                    "xaxis": {
                      "automargin": true,
                      "gridcolor": "white",
                      "linecolor": "white",
                      "ticks": "",
                      "title": {
                        "standoff": 15
                      },
                      "zerolinecolor": "white",
                      "zerolinewidth": 2
                    },
                    "yaxis": {
                      "automargin": true,
                      "gridcolor": "white",
                      "linecolor": "white",
                      "ticks": "",
                      "title": {
                        "standoff": 15
                      },
                      "zerolinecolor": "white",
                      "zerolinewidth": 2
                    }
                  }
                }
              }
            },
            "text/html": [
              "<div>                            <div id=\"11a95f6d-2ec0-45f0-b85e-e80717543f82\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"11a95f6d-2ec0-45f0-b85e-e80717543f82\")) {                    Plotly.newPlot(                        \"11a95f6d-2ec0-45f0-b85e-e80717543f82\",                        [{\"hovertemplate\":\"model=saiga_7b\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"saiga_7b\",\"line\":{\"color\":\"rgb(102, 197, 204)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"saiga_7b\",\"r\":[4.95,4.2,2.25,1.7,1.95,2.65,4.95,6.55,4.95],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=mt_bench_ru_yandexgpt\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"mt_bench_ru_yandexgpt\",\"line\":{\"color\":\"rgb(246, 207, 113)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"mt_bench_ru_yandexgpt\",\"r\":[5.15,5.05,2.45,1.45,2.45,4.65,7.2,7.3,5.15],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=mt_bench_ru_saiga2_7b\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"mt_bench_ru_saiga2_7b\",\"line\":{\"color\":\"rgb(248, 156, 116)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"mt_bench_ru_saiga2_7b\",\"r\":[4.9,5.05,2.6,1.5,2.7,3.8,5.6,7.4,4.9],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=mt_bench_ru_saiga2_13b\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"mt_bench_ru_saiga2_13b\",\"line\":{\"color\":\"rgb(220, 176, 242)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"mt_bench_ru_saiga2_13b\",\"r\":[6.1,5.8,3.35,1.9,3.6,5.25,6.45,7.9,6.1],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=mt_bench_ru_gigasaiga_13b\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"mt_bench_ru_gigasaiga_13b\",\"line\":{\"color\":\"rgb(135, 197, 95)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"mt_bench_ru_gigasaiga_13b\",\"r\":[4.7,5.05,2.6,1.05,1.55,1.45,3.95,5.5,4.7],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"},{\"hovertemplate\":\"model=mt_bench_ru_gigachat\\u003cbr\\u003escore=%{r}\\u003cbr\\u003ecategory=%{theta}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"mt_bench_ru_gigachat\",\"line\":{\"color\":\"rgb(158, 185, 243)\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"mt_bench_ru_gigachat\",\"r\":[6.1,5.4,4.1,1.65,1.8,1.65,5.15,6.45,6.1],\"showlegend\":true,\"subplot\":\"polar\",\"theta\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\",\"Writing\"],\"type\":\"scatterpolar\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"polar\":{\"domain\":{\"x\":[0.0,1.0],\"y\":[0.0,1.0]},\"angularaxis\":{\"direction\":\"clockwise\",\"rotation\":90,\"categoryorder\":\"array\",\"categoryarray\":[\"Writing\",\"Roleplay\",\"Reasoning\",\"Math\",\"Coding\",\"Extraction\",\"STEM\",\"Humanities\"]}},\"legend\":{\"title\":{\"text\":\"model\"},\"tracegroupgap\":0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('11a95f6d-2ec0-45f0-b85e-e80717543f82');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                });            </script>        </div>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "target_models = [\n",
        "    # \"saiga_7b_v1\",\n",
        "    # \"saiga_7b_v2\",\n",
        "    # \"xglm_4.5b_v10_epoch_6_step_41141\",\n",
        "    # \"mt_bench_en_xglm_4.5B_lora_our_dataset\",\n",
        "    # \"mt_bench_en_gigachat\",\n",
        "    \"saiga_7b_v1_ru\",\n",
        "    \"mt_bench_ru_gigachat\",\n",
        "    \"mt_bench_ru_saiga2_7b\",\n",
        "    \"mt_bench_ru_saiga2_13b\",\n",
        "    \"mt_bench_ru_gigasaiga_13b\",\n",
        "    \"mt_bench_ru_yandexgpt\",\n",
        "    # ------------- our dataset and models\n",
        "    # \"saiga_7b_v2_ru\",\n",
        "    # \"xglm_4.5b_v10_epoch_6_step_41141_ru\",\n",
        "    # \"mt_bench_ru_xglm_4.5B_lora_our_dataset\",\n",
        "    # \"mt_bench_ru_saiga2_7b_our_dataset\",\n",
        "    # \"mt_bench_ru_saiga2_13b_our_dataset\",\n",
        "    # \"mt_bench_ru_rugpt_13B_our_dataset\",\n",
        "    # \"mt_bench_ru_xglm_4.5B_lora_saiga_dataset\",\n",
        "]\n",
        "# target_models = all_models\n",
        "# target_models = [item.replace(\"mt_bench_ru_\", \"\") for item in target_models]\n",
        "scores_target = [\n",
        "    scores_all[i]\n",
        "    for i in range(len(scores_all))\n",
        "    if scores_all[i][\"model\"] in target_models\n",
        "]\n",
        "# scores_target = [\n",
        "#     scores_all[i]\n",
        "#     for i in range(len(scores_all))\n",
        "#     if 'ru' in scores_all[i][\"model\"] and 'our' in scores_all[i][\"model\"]\n",
        "# ]\n",
        "print(scores_target)\n",
        "\n",
        "# sort by target_models\n",
        "scores_target = sorted(scores_target, key=lambda x: x[\"model\"], reverse=True)\n",
        "\n",
        "df_score = pd.DataFrame(scores_target)\n",
        "df_score = df_score[df_score[\"model\"].isin(target_models)]\n",
        "\n",
        "rename_map = {\n",
        "    # \"llama-13b\": \"LLaMA-13B\",\n",
        "    # \"alpaca-13b\": \"Alpaca-13B\",\n",
        "    # \"vicuna-33b-v1.3\": \"Vicuna-33B\",\n",
        "    # \"vicuna-13b-v1.3\": \"Vicuna-13B\",\n",
        "    # \"gpt-3.5-turbo\": \"GPT-3.5-turbo\",\n",
        "    # \"claude-v1\": \"Claude-v1\",\n",
        "    # \"gpt-4\": \"GPT-4\",\n",
        "    \"saiga_7b_v2_ru\":\"llama-1_our_dataset\",\n",
        "    \"saiga_7b_v1_ru\":\"saiga_7b\"\n",
        "}\n",
        "\n",
        "for k, v in rename_map.items():\n",
        "    df_score.replace(k, v, inplace=True)\n",
        "\n",
        "fig = px.line_polar(\n",
        "    df_score,\n",
        "    r=\"score\",\n",
        "    theta=\"category\",\n",
        "    line_close=True,\n",
        "    category_orders={\"category\": CATEGORIES},\n",
        "    color=\"model\",\n",
        "    markers=True,\n",
        "    color_discrete_sequence=px.colors.qualitative.Pastel,\n",
        ")\n",
        "\n",
        "fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting nbformat\n",
            "  Downloading nbformat-5.9.2-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.6/77.6 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hCollecting fastjsonschema (from nbformat)\n",
            "  Downloading fastjsonschema-2.18.0-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: jsonschema>=2.6 in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from nbformat) (4.17.3)\n",
            "Requirement already satisfied: jupyter-core in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from nbformat) (4.12.0)\n",
            "Requirement already satisfied: traitlets>=5.1 in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from nbformat) (5.9.0)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat) (22.2.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /cephfs/home/kosenko/miniconda3/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat) (0.19.3)\n",
            "Installing collected packages: fastjsonschema, nbformat\n",
            "Successfully installed fastjsonschema-2.18.0 nbformat-5.9.2\n",
            "\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
            "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!pip install nbformat -U"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MaBaUN4IqvJI"
      },
      "outputs": [],
      "source": [
        "# fig = px.line_polar(df_score, r = 'wtrate', theta = 'category', line_close = True, category_orders = {\"category\": CATEGORIES},\n",
        "#                     color = 'model', markers=True, color_discrete_sequence=px.colors.qualitative.Pastel)\n",
        "# fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "4l1bzYM2bgDW"
      },
      "outputs": [],
      "source": [
        "fig.update_layout(\n",
        "    font=dict(\n",
        "        size=18,\n",
        "    ),\n",
        ")\n",
        "fig.write_image(\"fig.png\", width=800, height=600, scale=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nfpERnxFANhV"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}