10_RankSVM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "10_RankSVM.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNm8ttCt09F08HcLbaHNDz5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "032aa6615c714e00bdc277f7174b99e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a52cb29a33b040528bda2becdb61f96b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ecc47f7f4b404f69ad92909c64a1566e",
              "IPY_MODEL_1811ac8a0d1e47f7909b4b84c7f0d5b0"
            ]
          }
        },
        "a52cb29a33b040528bda2becdb61f96b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ecc47f7f4b404f69ad92909c64a1566e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c67b2176eb634a3a92a692d48fa20127",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 604,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 604,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_54ac97e8c11844088253e5a2b3b0ee72"
          }
        },
        "1811ac8a0d1e47f7909b4b84c7f0d5b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98a1c89b230041c290453735261b6598",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 604/604 [00:01&lt;00:00, 401.13it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f24a34a506db4561a796da820dee8677"
          }
        },
        "c67b2176eb634a3a92a692d48fa20127": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "54ac97e8c11844088253e5a2b3b0ee72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98a1c89b230041c290453735261b6598": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f24a34a506db4561a796da820dee8677": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "322def7939e34174867a3c87ba9cbdbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_08756163141e4022978ded9b8524bc60",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a5ffba7d42c449ff8bd245f44ce35717",
              "IPY_MODEL_0424fb23e42146fa9ad54830941e64f5"
            ]
          }
        },
        "08756163141e4022978ded9b8524bc60": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a5ffba7d42c449ff8bd245f44ce35717": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eaa8b66be9af46748b69ec87759d96a7",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 604,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 604,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a95c3a6bed645689d93b0141d516214"
          }
        },
        "0424fb23e42146fa9ad54830941e64f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_177be703258549a98c7c25e5283b10cc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 604/604 [00:02&lt;00:00, 293.81it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0a5e57e8f60541aabbcc9aaa3b731c19"
          }
        },
        "eaa8b66be9af46748b69ec87759d96a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a95c3a6bed645689d93b0141d516214": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "177be703258549a98c7c25e5283b10cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0a5e57e8f60541aabbcc9aaa3b731c19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ivoryRabbit/RecSys/blob/master/10_RankSVM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bboT3ONRcxSz"
      },
      "source": [
        "# Learning to Rank: SVM ver.\r\n",
        "\r\n",
        "- [Learning to Rank: From Pairwise Approach to Listwise Approach](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/tr-2007-40.pdf)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8FDcw4rVboe"
      },
      "source": [
        "import glob\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm.notebook import tqdm\n",
        "from typing import Callable, Tuple, List\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from scipy.sparse import csr_matrix\n",
        "\n",
        "from multiprocessing import Pool\n",
        "from sklearn.svm import LinearSVC\n",
        "from xgboost import XGBRanker\n",
        "from lightgbm import LGBMRanker\n",
        "\n",
        "from tensorflow.keras.utils import get_file\n",
        "import zipfile\n",
        "\n",
        "np.random.seed(777)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdjqodyYVf9I"
      },
      "source": [
        "def load_data(data_size : str) -> pd.DataFrame:\n",
        "    ''' load Movie Lens data '''\n",
        "    if data_size == '1m':\n",
        "        fname = 'ml-1m.zip'\n",
        "        data = 'ml-1m/ratings.dat'\n",
        "    elif data_size == '10m':\n",
        "        fname = 'ml-10m.zip'\n",
        "        data = 'ml-10M100K/ratings.dat'\n",
        "    elif data_size == '20m':\n",
        "        fname = 'ml-20m.zip'\n",
        "        data = 'ml-20m/ratings.csv'\n",
        "    elif data_size == '25m':\n",
        "        fname = 'ml-25m.zip'\n",
        "        data = 'ml-25m/ratings.csv'\n",
        "    if not glob.glob(data):\n",
        "        origin = f'http://files.grouplens.org/datasets/movielens/{fname}'\n",
        "        file = get_file(fname, origin)\n",
        "        zip_ref = zipfile.ZipFile(file, 'r')\n",
        "        zip_ref.extractall()\n",
        "\n",
        "    col_names = ['user_id', 'movie_id', 'rating', 'timestamp']\n",
        "    if data_size in ['20m', '25m']:\n",
        "        ratings = pd.read_csv(data, names = col_names, engine = 'python')\n",
        "    else:\n",
        "        ratings = pd.read_csv(data, sep = '|', delimiter = '::', names = col_names, engine = 'python')\n",
        "    print(ratings.shape)\n",
        "    return ratings"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RZf8Gzf0VhYV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 213
        },
        "outputId": "04ee7306-1931-4fad-a5e2-87a7f4e1abe0"
      },
      "source": [
        "ratings = load_data('1m')\n",
        "ratings.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000209, 4)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1193</td>\n",
              "      <td>5</td>\n",
              "      <td>978300760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>661</td>\n",
              "      <td>3</td>\n",
              "      <td>978302109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>914</td>\n",
              "      <td>3</td>\n",
              "      <td>978301968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>3408</td>\n",
              "      <td>4</td>\n",
              "      <td>978300275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>2355</td>\n",
              "      <td>5</td>\n",
              "      <td>978824291</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id  movie_id  rating  timestamp\n",
              "0        1      1193       5  978300760\n",
              "1        1       661       3  978302109\n",
              "2        1       914       3  978301968\n",
              "3        1      3408       4  978300275\n",
              "4        1      2355       5  978824291"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q61otQWA3tFe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5d44069e-7fa1-40bc-b36f-87ca936ff708"
      },
      "source": [
        "print(f'# of users = {ratings.user_id.nunique()}')\n",
        "print(f'# of items = {ratings.movie_id.nunique()}')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "# of users = 6040\n",
            "# of items = 3706\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O9kecujeV2qP"
      },
      "source": [
        "def binarizer(df: pd.DataFrame, threshold = 4) -> pd.DataFrame:\n",
        "    df = df.assign(rating = np.where(df.rating >= threshold, 1, 0))\n",
        "    return df[df.rating > 0.0].reset_index(drop = True)\n",
        "\n",
        "def make_warm(df: pd.DataFrame, threshold = 5) -> pd.DataFrame: # remove cold starters\n",
        "    positive = df.groupby('user_id')['movie_id'].count()\n",
        "    positive = positive.index[positive >= threshold]\n",
        "    return df[df.user_id.isin(positive)].reset_index(drop = True)\n",
        "\n",
        "def train_valid_test_split(df: pd.DataFrame, size: float) -> pd.DataFrame:\n",
        "    train_user, test_user = train_test_split(df.user_id.unique(), test_size = 2 * size, random_state = 777)\n",
        "    valid_user, test_user = train_test_split(test_user, test_size = 0.5, random_state = 777)\n",
        "    train, valid, test = map(lambda x: df[df.user_id.isin(x)], (train_user, valid_user, test_user))\n",
        "    train, valid, test = map(lambda df: df.reset_index(drop = True), (train, valid, test))\n",
        "    return train, valid, test\n",
        "\n",
        "def query_relev_split(df: pd.DataFrame, size: float) -> pd.DataFrame:\n",
        "    timeorder = df.groupby('user_id')['timestamp'].rank(method = 'first', ascending = True)\n",
        "    seen_cnts = df.groupby('user_id')['movie_id'].transform('count')\n",
        "    df = df.assign(seen_cnts = seen_cnts, timeorder = timeorder)\n",
        "    query = df[df.timeorder < df.seen_cnts * (1-size)]\n",
        "    relev = df[df.timeorder >= df.seen_cnts * (1-size)]\n",
        "    relev = relev[relev.user_id.isin(query.user_id.unique())]\n",
        "    query, relev = map(lambda df: df.drop(columns = ['timeorder', 'seen_cnts']), (query, relev))\n",
        "    query, relev = map(lambda df: df.reset_index(drop = True), (query, relev))\n",
        "    return query, relev\n",
        "\n",
        "def list_agg(df: pd.DataFrame) -> pd.DataFrame:\n",
        "    return df.groupby('user_id', as_index = False)[['movie_id']].agg(list)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "pNAF90tno0w4",
        "outputId": "e215a716-0a26-4156-b95f-6d6f9a4b7646"
      },
      "source": [
        "data = binarizer(ratings)\r\n",
        "data = make_warm(data)\r\n",
        "data.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1193</td>\n",
              "      <td>1</td>\n",
              "      <td>978300760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>3408</td>\n",
              "      <td>1</td>\n",
              "      <td>978300275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>2355</td>\n",
              "      <td>1</td>\n",
              "      <td>978824291</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1287</td>\n",
              "      <td>1</td>\n",
              "      <td>978302039</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>2804</td>\n",
              "      <td>1</td>\n",
              "      <td>978300719</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id  movie_id  rating  timestamp\n",
              "0        1      1193       1  978300760\n",
              "1        1      3408       1  978300275\n",
              "2        1      2355       1  978824291\n",
              "3        1      1287       1  978302039\n",
              "4        1      2804       1  978300719"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6px0uL8FoxvL"
      },
      "source": [
        "train, _, test = train_valid_test_split(data, size = 0.1)\r\n",
        "test_q, test_r = query_relev_split(test, size = 0.2)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KopcUAwxcLGR"
      },
      "source": [
        "## How to generate item features from user feedback\r\n",
        "\r\n",
        "- [Remark](https://github.com/JunsolKim/kakao-arena-melon-ysocio13)\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "For given user $u \\in U$ and arbitrary item $i \\in I$, define an item representative $\\mathbb{x}_i = \\mathbb{x}_i(u) \\in \\mathbb{R}^{|P|}$ as follows:\r\n",
        "\r\n",
        "$$\r\n",
        "\\begin{aligned}\r\n",
        "\\mathbb{x}_{iv} := \\left\\{ \\begin{array}{ll}\r\n",
        "\\frac{|I(u) \\cap I(v)|}{|I(v)|^{\\alpha}} & \\text{if } v \\in U(i) \\\\\r\n",
        "0 & \\text{o.w.}\\\\\r\n",
        "\\end{array} \\right.\r\n",
        "\\end{aligned}\r\n",
        "$$\r\n",
        "\r\n",
        "where\r\n",
        "- $U(i)$ is the set of users that include item $i$\r\n",
        "- $I(u)$ is the set of items included by user $u$\r\n",
        "- $\\alpha$ is an hyperparameter\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "Next, define the similarity $s_i = s_i(u) \\in \\mathbb{R}$ between two items $i\\in I(u)$ and $j\\in I$ as follows:\r\n",
        "\r\n",
        "$$ s_{ij} := \\frac{\\mathbb{x}_i^T\\mathbb{x}_j}{|U(i)|^{\\beta} |U(j)|^{1- \\beta}} $$\r\n",
        "\r\n",
        "where\r\n",
        "- $\\beta$ is an hyperparameter\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "Finally, predict $y_j \\in \\{0, 1\\}$ from the inputs $r_j \\in \\mathbb{R}^{|I(u)|}$ for each item $j \\in I$ using various classifier, where\r\n",
        "\r\n",
        "- $\\ r_{j} =\\big( s_{1j}, \\ s_{2j}, \\ldots, \\ s_{|I(u)|j}\\big)$\r\n",
        "\r\n",
        "- $\\begin{aligned}\r\n",
        "y_{j} = \\left\\{ \\begin{array}{ll}\r\n",
        "1 & \\text{if } j \\in I(u) \\\\\r\n",
        "0 & \\text{o.w.}\\\\\r\n",
        "\\end{array} \\right.\r\n",
        "\\end{aligned}\r\n",
        "$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GVjdMkqNwxKg"
      },
      "source": [
        "class RankSVM:\r\n",
        "    def __init__(self, alpha: float, beta: float, ranker_type = 'SVM'):\r\n",
        "        self.alpha = alpha\r\n",
        "        self.beta = beta\r\n",
        "        self.ranker_type = ranker_type\r\n",
        "        self.ranking_model = {\r\n",
        "            'LGB': LGBMRanker, \r\n",
        "            'XGB': XGBRanker, \r\n",
        "            'SVM': LinearSVC\r\n",
        "        }[self.ranker_type]\r\n",
        "\r\n",
        "    def fit(self, df: pd.DataFrame):\r\n",
        "        self.user_idx2id = df.user_id.unique() # list[index] = id\r\n",
        "        self.user_id2idx = {e: i for i, e in enumerate(self.user_idx2id)} # dict[id] = index\r\n",
        "        self.item_idx2id = df.movie_id.unique() # list[index] = id\r\n",
        "        self.item_id2idx = {e: i for i, e in enumerate(self.item_idx2id)} # dict[id] = index\r\n",
        "        self.n_item = len(self.item_idx2id)\r\n",
        "\r\n",
        "        df = df.assign(user_id = df.user_id.map(self.user_id2idx), \r\n",
        "                       movie_id = df.movie_id.map(self.item_id2idx))\r\n",
        "\r\n",
        "        row = df.user_id\r\n",
        "        col = df.movie_id\r\n",
        "        val = np.ones(shape = df.index.size)\r\n",
        "\r\n",
        "        self.mat = csr_matrix((val, (row, col)), dtype = np.float32)\r\n",
        "\r\n",
        "    def rerank(self, profile):\r\n",
        "        query = np.zeros(shape = (self.n_item, ), dtype = np.float32)\r\n",
        "        pos = np.array(profile)\r\n",
        "        query[pos] = 1.0\r\n",
        "\r\n",
        "        users_item = np.sum(self.mat, axis = 1)\r\n",
        "        items_user = np.sum(self.mat, axis = 0)\r\n",
        "\r\n",
        "        weight = self.mat[:, pos].multiply(np.power(users_item, -2*self.alpha)).sum(axis = 1)\r\n",
        "\r\n",
        "        value = self.mat[:, pos].multiply(weight).T.dot(self.mat)\r\n",
        "        value = value.multiply(np.power(1e-1 + items_user[:, pos], -self.beta).reshape(-1, 1))\r\n",
        "        value = value.multiply(np.power(1e-1 + items_user, self.beta-1))\r\n",
        "        sim = value.T.tocsr()\r\n",
        "\r\n",
        "        if self.ranker_type == 'SVM':\r\n",
        "            ranker = self.ranking_model(tol = 1e-6, max_iter = 500000)\r\n",
        "            ranker.fit(sim, query)\r\n",
        "            pred_batch = ranker.decision_function(sim)\r\n",
        "        else:\r\n",
        "            ranker = self.ranking_model(n_jobs = -1)\r\n",
        "            ranker.fit(sim, query)\r\n",
        "            pred_batch = ranker.predict(sim)\r\n",
        "        \r\n",
        "        pred_batch[pos] = -np.inf\r\n",
        "\r\n",
        "        rec = np.argsort(pred_batch)[:-self.N-1:-1]\r\n",
        "        rec = self.item_idx2id[rec]\r\n",
        "        return rec\r\n",
        "\r\n",
        "    def predict(self, df:pd.DataFrame, N: int) -> pd.DataFrame:\r\n",
        "        self.N = N\r\n",
        "        df = df.assign(movie_id = df.movie_id.map(self.item_id2idx)).dropna()\r\n",
        "        df = df.assign(movie_id = df.movie_id.astype(int))\r\n",
        "        df = list_agg(df)\r\n",
        "        profile = df['movie_id']\r\n",
        "        \r\n",
        "        pred = []\r\n",
        "        pool = Pool(25)\r\n",
        "        results = pool.map(self.rerank, profile)\r\n",
        "        pool.close()\r\n",
        "        pool.join()\r\n",
        "\r\n",
        "        for idx, rec in enumerate(results):\r\n",
        "            pred.append({'user_id': df.at[idx, 'user_id'],\r\n",
        "                         'movie_id': rec})\r\n",
        "\r\n",
        "        return pd.DataFrame(pred)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pP6v4dXR50fT"
      },
      "source": [
        "class evaluate:\n",
        "    def __init__(self, true: pd.DataFrame, pred: pd.DataFrame):\n",
        "        self.true = true\n",
        "        self.pred = pred\n",
        "        self.max_K = 10000\n",
        "        self.idcg = np.cumsum([1.0 / np.log2(i+2) for i in range(self.max_K)])\n",
        "\n",
        "    def _recall(self, gt: List, rec: List, K = None) -> float: # Recall\n",
        "        res = [r for r in rec[:K] if r in gt]\n",
        "        return len(res) / np.min([K, len(gt)])\n",
        "    \n",
        "    def _precision(self, gt: List, rec: List, K = None) -> float: # Precision\n",
        "        res = [r for r in rec[:K] if r in gt]\n",
        "        return len(res) / len(rec[:K])\n",
        "\n",
        "    def _AP(self, gt: List, rec: List, K = None) -> float: # Average Precision\n",
        "        res = 0.0\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                res += self._precision(gt, rec[:K], i+1)\n",
        "        return res / np.min([K, len(gt)])\n",
        "\n",
        "    def _HR(self, gt: List, rec: List, K = None) -> float: # Hit Rate\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                return  1.0\n",
        "        return 0\n",
        "\n",
        "    def _RR(self, gt: List, rec: List, K = None) -> float: # Reciprocal Rank\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                return  1.0 / (i+1)\n",
        "        return 0\n",
        "\n",
        "    def _nDCG(self, gt: List, rec: List, K = None) -> float: # normalized Discounted Cumulative Gain\n",
        "        dcg = 0.0\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                dcg += 1.0 / np.log2(i+2)\n",
        "        idcg = self.idcg[min([len(gt), K])-1]\n",
        "        return dcg / idcg\n",
        "\n",
        "    def _get_item(self, rec: List, K = None) -> float:\n",
        "        for r in rec[:K]:\n",
        "            self.uniq_item[r] = self.uniq_item.get(r, 0) + 1\n",
        "\n",
        "    def __call__(self, K = None):\n",
        "        self.K = K if K else self.max_K\n",
        "        self.recall = 0.0\n",
        "        self.precision = 0.0\n",
        "        self.MAP = 0.0\n",
        "        self.HR = 0.0\n",
        "        self.MRR = 0.0\n",
        "        self.nDCG = 0.0\n",
        "        self.uniq_item = {}\n",
        "        n = self.true.index.size\n",
        "        for gt, rec in zip(tqdm(self.true.movie_id), self.pred.movie_id):\n",
        "            self.recall += self._recall(gt, rec, K) / n\n",
        "            self.precision += self._precision(gt, rec, K) / n\n",
        "            self.MAP += self._AP(gt, rec, K) / n\n",
        "            self.HR += self._HR(gt, rec, K) / n\n",
        "            self.MRR += self._RR(gt, rec, K) / n\n",
        "            self.nDCG += self._nDCG(gt, rec, K) / n\n",
        "            self._get_item(rec, K)\n",
        "        self.CO = len(self.uniq_item) / (n*K) # Coverage\n",
        "        self.ED = -np.sum([self.uniq_item[i] * np.log(self.uniq_item[i] / (n*K)) for i in self.uniq_item]) / (n*K) # Entropy-Diversity\n",
        "\n",
        "    def print_all(self):\n",
        "        K = '@' + str(self.K) if self.K else ''\n",
        "        print(f'{\"Recall\":>12}{K} : {self.recall:.5f}',\n",
        "              f'\\n{\"Precision\":>12}{K} : {self.precision:.5f}',\n",
        "              f'\\n{\"MAP\":>12}{K} : {self.MAP:.5f}',\n",
        "              f'\\n{\"HR\":>12}{K} : {self.HR:.5f}', \n",
        "              f'\\n{\"MRR\":>12}{K} : {self.MRR:.5f}',\n",
        "              f'\\n{\"nDCG\":>12}{K} : {self.nDCG:.5f}',\n",
        "              f'\\n{\"CO\":>12}{K} : {self.CO:.5f}',\n",
        "              f'\\n{\"ED\":>12}{K} : {self.ED:.5f}')"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFu69sfZAxbT"
      },
      "source": [
        "model = RankSVM(alpha = 0.6, beta = 0.3, ranker_type = 'SVM')\r\n",
        "model.fit(train)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "zL3YxmyLApXp",
        "outputId": "8d21d764-f3c2-4eb4-9adb-df8794e4bdf6"
      },
      "source": [
        "true = list_agg(test_r)\r\n",
        "true.head(5)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>[1394, 104, 1079, 1259, 2355, 3552, 1304, 2081]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9</td>\n",
              "      <td>[2268, 1682, 2278, 590, 524, 529, 2294, 1921, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>42</td>\n",
              "      <td>[3421, 3868, 1257, 2997, 2134, 1265, 593, 2143...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>79</td>\n",
              "      <td>[3044, 3176, 2686, 2024]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>92</td>\n",
              "      <td>[2054, 2134, 733, 592, 2072, 3107, 2162, 1291,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id                                           movie_id\n",
              "0        3    [1394, 104, 1079, 1259, 2355, 3552, 1304, 2081]\n",
              "1        9  [2268, 1682, 2278, 590, 524, 529, 2294, 1921, ...\n",
              "2       42  [3421, 3868, 1257, 2997, 2134, 1265, 593, 2143...\n",
              "3       79                           [3044, 3176, 2686, 2024]\n",
              "4       92  [2054, 2134, 733, 592, 2072, 3107, 2162, 1291,..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "SvCyzVdgv3MG",
        "outputId": "d89f62e7-3e44-49a3-db52-bc6157a29476"
      },
      "source": [
        "pred = model.predict(test_q, N = 100)\r\n",
        "pred.head(5)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>[1240, 2571, 1036, 589, 110, 1270, 592, 2628, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9</td>\n",
              "      <td>[296, 2997, 2396, 110, 2599, 2329, 356, 3578, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>42</td>\n",
              "      <td>[2916, 2571, 2528, 780, 1527, 1580, 1676, 2968...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>79</td>\n",
              "      <td>[3300, 3555, 3354, 3578, 3785, 3753, 3623, 374...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>92</td>\n",
              "      <td>[2078, 594, 2087, 1032, 1029, 919, 3034, 2918,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id                                           movie_id\n",
              "0        3  [1240, 2571, 1036, 589, 110, 1270, 592, 2628, ...\n",
              "1        9  [296, 2997, 2396, 110, 2599, 2329, 356, 3578, ...\n",
              "2       42  [2916, 2571, 2528, 780, 1527, 1580, 1676, 2968...\n",
              "3       79  [3300, 3555, 3354, 3578, 3785, 3753, 3623, 374...\n",
              "4       92  [2078, 594, 2087, 1032, 1029, 919, 3034, 2918,..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "icmR-LqMyDTL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398,
          "referenced_widgets": [
            "032aa6615c714e00bdc277f7174b99e2",
            "a52cb29a33b040528bda2becdb61f96b",
            "ecc47f7f4b404f69ad92909c64a1566e",
            "1811ac8a0d1e47f7909b4b84c7f0d5b0",
            "c67b2176eb634a3a92a692d48fa20127",
            "54ac97e8c11844088253e5a2b3b0ee72",
            "98a1c89b230041c290453735261b6598",
            "f24a34a506db4561a796da820dee8677",
            "322def7939e34174867a3c87ba9cbdbe",
            "08756163141e4022978ded9b8524bc60",
            "a5ffba7d42c449ff8bd245f44ce35717",
            "0424fb23e42146fa9ad54830941e64f5",
            "eaa8b66be9af46748b69ec87759d96a7",
            "8a95c3a6bed645689d93b0141d516214",
            "177be703258549a98c7c25e5283b10cc",
            "0a5e57e8f60541aabbcc9aaa3b731c19"
          ]
        },
        "outputId": "e13cfa82-1f45-45af-864e-63f9c2a79dde"
      },
      "source": [
        "scores = evaluate(true, pred)\n",
        "\n",
        "scores(K = 10)\n",
        "scores.print_all()\n",
        "\n",
        "scores(K = 100)\n",
        "scores.print_all()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "032aa6615c714e00bdc277f7174b99e2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=604.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "      Recall@10 : 0.11874 \n",
            "   Precision@10 : 0.09470 \n",
            "         MAP@10 : 0.05153 \n",
            "          HR@10 : 0.50662 \n",
            "         MRR@10 : 0.20790 \n",
            "        nDCG@10 : 0.11021 \n",
            "          CO@10 : 0.12781 \n",
            "          ED@10 : 5.69472\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "322def7939e34174867a3c87ba9cbdbe",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=604.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "      Recall@100 : 0.41134 \n",
            "   Precision@100 : 0.05833 \n",
            "         MAP@100 : 0.06523 \n",
            "          HR@100 : 0.95530 \n",
            "         MRR@100 : 0.22744 \n",
            "        nDCG@100 : 0.22326 \n",
            "          CO@100 : 0.03303 \n",
            "          ED@100 : 6.49351\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}