04_AutoRec.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "04_AutoRec.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPWCyVT3Hc6ba2CD4cmU0Ut",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "aabf5662754c46f894c427b8e1dc2828": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c25d26f060b94e52a72da90c1e553e8f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_576e96debff9425891e6fce1d02325d4",
              "IPY_MODEL_e25bf164f48b4d86bbb97998a2a6d1c2"
            ]
          }
        },
        "c25d26f060b94e52a72da90c1e553e8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "576e96debff9425891e6fce1d02325d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5b485c98a972408cb18116e3c6a3dbfe",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 604,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 604,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2aaeda6eb9124c6c8a3a2168e8f86f26"
          }
        },
        "e25bf164f48b4d86bbb97998a2a6d1c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0fcaafd4095c45de96c78bdc4bf2e807",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 604/604 [00:01&lt;00:00, 597.28it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ab50d412df094667b128dc71d9484235"
          }
        },
        "5b485c98a972408cb18116e3c6a3dbfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2aaeda6eb9124c6c8a3a2168e8f86f26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0fcaafd4095c45de96c78bdc4bf2e807": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ab50d412df094667b128dc71d9484235": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dfa68c194495413d85e93c67fa0f3c2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9d8a661b62994e3e93859ed9d4b86a4f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_057d9197cb294810bcf142bf73dcb681",
              "IPY_MODEL_5357ddad2e034666a612cc0b32598124"
            ]
          }
        },
        "9d8a661b62994e3e93859ed9d4b86a4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "057d9197cb294810bcf142bf73dcb681": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fddf99322a414f808a8f2c3f0636a10c",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 604,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 604,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_db4e27e2921f4fb1909b750f8324e9d7"
          }
        },
        "5357ddad2e034666a612cc0b32598124": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6f5f0227aedf4aa7aaaee8f9f66aefce",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 604/604 [00:01&lt;00:00, 366.59it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_17898d31a8ab488e966b81c6325b47a6"
          }
        },
        "fddf99322a414f808a8f2c3f0636a10c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "db4e27e2921f4fb1909b750f8324e9d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f5f0227aedf4aa7aaaee8f9f66aefce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "17898d31a8ab488e966b81c6325b47a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ivoryRabbit/RecSys/blob/master/04_AutoRec.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qshJl-mq6dgf"
      },
      "source": [
        "# U-AutoRec\n",
        "\n",
        "- [AutoRec: Autoencoders Meet Collaborative Filtering](http://users.cecs.anu.edu.au/~u5098633/papers/www15.pdf)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fXc-WZ-k2mlW"
      },
      "source": [
        "import glob\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from typing import Callable, Tuple, List\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "plt.style.use('ggplot')\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras.backend as K\n",
        "\n",
        "from tensorflow.keras.layers import Input, Dense\n",
        "from tensorflow.keras.models import Model, load_model\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.regularizers import l2\n",
        "\n",
        "from tensorflow.keras.utils import get_file\n",
        "import zipfile\n",
        "\n",
        "np.random.seed(777)\n",
        "tf.random.set_seed(777)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cMJfOc0uALKr"
      },
      "source": [
        "def load_data(data_size : str) -> pd.DataFrame:\n",
        "    ''' load Movie Lens data '''\n",
        "\n",
        "    if data_size == '1m':\n",
        "        fname = 'ml-1m.zip'\n",
        "        data = 'ml-1m/ratings.dat'\n",
        "    elif data_size == '10m':\n",
        "        fname = 'ml-10m.zip'\n",
        "        data = 'ml-10M100K/ratings.dat'\n",
        "    elif data_size == '20m':\n",
        "        fname = 'ml-20m.zip'\n",
        "        data = 'ml-20m/ratings.csv'\n",
        "    elif data_size == '25m':\n",
        "        fname = 'ml-25m.zip'\n",
        "        data = 'ml-25m/ratings.csv'\n",
        "    if not glob.glob(data):\n",
        "        origin = f'http://files.grouplens.org/datasets/movielens/{fname}'\n",
        "        file = get_file(fname, origin)\n",
        "        zip_ref = zipfile.ZipFile(file, 'r')\n",
        "        zip_ref.extractall()\n",
        "\n",
        "    col_dtypes = {'user_id': np.int32, 'movie_id': np.int32, \n",
        "                  'rating': np.uint8, 'timestamp': np.int32}\n",
        "    col_names = list(col_dtypes.keys())\n",
        "    if data_size in ['20m', '25m']:\n",
        "        ratings = pd.read_csv(\n",
        "            data, names = col_names, dtype = col_dtypes, engine = 'python',\n",
        "            chucksize = 10000\n",
        "        )\n",
        "        \n",
        "    else:\n",
        "        ratings = pd.read_csv(\n",
        "            data, names = col_names, dtype = col_dtypes, engine = 'python',\n",
        "            sep = '|', delimiter = '::'\n",
        "        )\n",
        "    print(ratings.shape)\n",
        "    return ratings"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "nC7HYbguIrwR",
        "outputId": "0545e972-33f7-4c8f-d83d-141027640170"
      },
      "source": [
        "%%time\r\n",
        "ratings = load_data('1m')\r\n",
        "display(ratings.head(5))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from http://files.grouplens.org/datasets/movielens/ml-1m.zip\n",
            "5922816/5917549 [==============================] - 0s 0us/step\n",
            "(1000209, 4)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1193</td>\n",
              "      <td>5</td>\n",
              "      <td>978300760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>661</td>\n",
              "      <td>3</td>\n",
              "      <td>978302109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>914</td>\n",
              "      <td>3</td>\n",
              "      <td>978301968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>3408</td>\n",
              "      <td>4</td>\n",
              "      <td>978300275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>2355</td>\n",
              "      <td>5</td>\n",
              "      <td>978824291</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id  movie_id  rating  timestamp\n",
              "0        1      1193       5  978300760\n",
              "1        1       661       3  978302109\n",
              "2        1       914       3  978301968\n",
              "3        1      3408       4  978300275\n",
              "4        1      2355       5  978824291"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 4.5 s, sys: 223 ms, total: 4.73 s\n",
            "Wall time: 5.17 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O6DZEVacUnoP",
        "outputId": "226bba52-c1ac-4e4c-8360-90fa5579d62d"
      },
      "source": [
        "user_list = ratings.user_id.unique()\r\n",
        "user_idx_map = {e: i for i, e in enumerate(user_list)}\r\n",
        "n_user = len(user_list)\r\n",
        "print(f'# of users = {n_user}')\r\n",
        "\r\n",
        "item_list = ratings.movie_id.unique()\r\n",
        "item_idx_map = {e: i for i, e in enumerate(item_list)}\r\n",
        "n_item = len(item_list)\r\n",
        "print(f'# of items = {n_item}')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "# of users = 6040\n",
            "# of items = 3706\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_1qziMzbI_39"
      },
      "source": [
        "def binarizer(df: pd.DataFrame, threshold = 4) -> pd.DataFrame:\n",
        "    df = df.assign(rating = np.where(df.rating >= threshold, 1, 0))\n",
        "    return df[df.rating > 0.0].reset_index(drop = True)\n",
        "\n",
        "def make_warm(df: pd.DataFrame, threshold = 5) -> pd.DataFrame: # remove cold starters\n",
        "    positive = df.groupby('user_id')['movie_id'].count()\n",
        "    positive = positive.index[positive >= threshold]\n",
        "    return df[df.user_id.isin(positive)].reset_index(drop = True)\n",
        "\n",
        "def train_valid_test_split(df: pd.DataFrame, size: float) -> pd.DataFrame:\n",
        "    train_user, test_user = train_test_split(df.user_id.unique(), test_size = 2 * size, random_state = 777)\n",
        "    valid_user, test_user = train_test_split(test_user, test_size = 0.5, random_state = 777)\n",
        "    train, valid, test = map(lambda x: df[df.user_id.isin(x)], (train_user, valid_user, test_user))\n",
        "    train, valid, test = map(lambda df: df.reset_index(drop = True), (train, valid, test))\n",
        "    return train, valid, test\n",
        "\n",
        "def query_relev_split(df: pd.DataFrame, size: float) -> pd.DataFrame:\n",
        "    timeorder = df.groupby('user_id')['timestamp'].rank(method = 'first', ascending = True)\n",
        "    seen_cnts = df.groupby('user_id')['movie_id'].transform('count')\n",
        "    df = df.assign(seen_cnts = seen_cnts, timeorder = timeorder)\n",
        "    query = df[df.timeorder < df.seen_cnts * (1-size)]\n",
        "    relev = df[df.timeorder >= df.seen_cnts * (1-size)]\n",
        "    relev = relev[relev.user_id.isin(query.user_id.unique())]\n",
        "    query, relev = map(lambda df: df.drop(columns = ['timeorder', 'seen_cnts']), (query, relev))\n",
        "    query, relev = map(lambda df: df.reset_index(drop = True), (query, relev))\n",
        "    return query, relev\n",
        "\n",
        "def list_agg(df: pd.DataFrame) -> pd.DataFrame:\n",
        "    return df.groupby('user_id', as_index = False)[['movie_id']].agg(list)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "NEmDF0bfpXl7",
        "outputId": "ada3f24a-8379-46d5-c0bb-b8e49797492c"
      },
      "source": [
        "data = ratings.assign(user_id = lambda df : df.user_id.map(user_idx_map),\r\n",
        "                      movie_id = lambda df: df.movie_id.map(item_idx_map))\r\n",
        "data = binarizer(data)\r\n",
        "data = make_warm(data)\r\n",
        "data.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>978300760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>978300275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>978824291</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>978302039</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>978300719</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id  movie_id  rating  timestamp\n",
              "0        0         0       1  978300760\n",
              "1        0         3       1  978300275\n",
              "2        0         4       1  978824291\n",
              "3        0         6       1  978302039\n",
              "4        0         7       1  978300719"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CwE4nLcKvCVh"
      },
      "source": [
        "train, valid, test = train_valid_test_split(data, size = 0.1)\r\n",
        "valid_q, valid_r = query_relev_split(valid, size = 0.2)\r\n",
        "test_q, test_r = query_relev_split(test, size = 0.2)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MkLGRouPxvQ8"
      },
      "source": [
        "def train_generator(train: pd.DataFrame, \n",
        "                    n_item: int, \n",
        "                    batch_size: int) -> np.ndarray:\n",
        "\n",
        "    train = list_agg(train)\n",
        "    \n",
        "    n_data = train.index.size\n",
        "    ids = np.arange(n_data)\n",
        "    profile = train['movie_id'].to_numpy()\n",
        "\n",
        "    n_batch = int(np.ceil(n_data / batch_size))\n",
        "    while True:\n",
        "        np.random.shuffle(ids)\n",
        "        for batch_step in range(n_batch):     \n",
        "            lower = batch_size * batch_step\n",
        "            upper = batch_size + lower\n",
        "            \n",
        "            batch_id = ids[lower: upper]\n",
        "            batch = np.zeros(shape = (batch_id.size, n_item), dtype = np.float32)\n",
        "            for i, idx in enumerate(batch_id):\n",
        "                batch[i, profile[idx]] = 1.0\n",
        "            \n",
        "            yield batch"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pfKvNy9Ygix4"
      },
      "source": [
        "def valid_generator(query: pd.DataFrame,\n",
        "                    relev: pd.DataFrame,\n",
        "                    n_item: int,\n",
        "                    batch_size: int) -> Tuple[np.ndarray]:\n",
        "\n",
        "    query = list_agg(query)\n",
        "    relev = list_agg(relev)\n",
        "\n",
        "    n_data = query.index.size\n",
        "    ids = np.arange(n_data)\n",
        "    profile_q = query['movie_id'].to_numpy()\n",
        "    profile_r = relev['movie_id'].to_numpy()\n",
        "\n",
        "    n_batch = int(np.ceil(n_data / batch_size))\n",
        "    while True:\n",
        "        np.random.shuffle(ids)\n",
        "        for batch_step in range(n_batch):\n",
        "            lower = batch_size * batch_step\n",
        "            upper = batch_size + lower\n",
        "            \n",
        "            batch_id = ids[lower: upper]\n",
        "            batch_q = np.zeros(shape = (batch_id.size, n_item), dtype = np.float32)\n",
        "            batch_r = np.zeros(shape = (batch_id.size, n_item), dtype = np.float32)\n",
        "            for i, idx in enumerate(batch_id):\n",
        "                batch_q[i, profile_q[idx]] = 1.0 # seen\n",
        "                batch_r[i, profile_r[idx]] = 1.0 # unseen\n",
        "\n",
        "            yield batch_q, batch_r"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0U5vZc1jpYf"
      },
      "source": [
        "def binary_cross_entropy(true: tf.Tensor, pred: tf.Tensor) -> tf.Tensor:\n",
        "    ce = K.sum(K.binary_crossentropy(true, pred), axis = 1)\n",
        "    return K.mean(ce, axis = 0)\n",
        "    \n",
        "def batch_ndcg(query: tf.Tensor, \n",
        "               relev: tf.Tensor,\n",
        "               pred: tf.Tensor, \n",
        "               weight: tf.Tensor,\n",
        "               k = 100) -> tf.Tensor: # only for validation\n",
        "\n",
        "    rec = tf.where(query == 1.0, -1.0, pred) # leave only unseen-data\n",
        "    rec = tf.argsort(rec, axis = 1, direction = 'DESCENDING')[:, :k]\n",
        "    shape = tf.shape(rec)\n",
        "\n",
        "    row = tf.repeat(tf.range(shape[0]), shape[1])\n",
        "    row = tf.expand_dims(row, axis = 1)\n",
        "    col = tf.reshape(rec, shape = [-1])\n",
        "    col = tf.expand_dims(col, axis = 1)\n",
        "    loc = tf.concat([row, col], axis = 1)\n",
        "\n",
        "    top_k = tf.reshape(tf.gather_nd(relev, loc), shape = shape)\n",
        "    ideal = tf.sort(relev, axis = 1, direction = 'DESCENDING')[:, :k]\n",
        "    weight = weight[:shape[1]]\n",
        "\n",
        "    dcg = tf.reduce_sum(top_k * weight, axis = 1)\n",
        "    idcg = tf.reduce_sum(ideal * weight, axis = 1)\n",
        "    return tf.reduce_mean(dcg / idcg, axis = 0)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Duyx8XbFxK6v"
      },
      "source": [
        "class AutoRec(tf.keras.models.Model):\r\n",
        "    def __init__(self, input_dim, latent_dim):\r\n",
        "        super(AutoRec, self).__init__()\r\n",
        "        self.input_dim = input_dim\r\n",
        "        self.latent_dim = latent_dim\r\n",
        "        self.encoder = Dense(self.latent_dim, \r\n",
        "                             activation = 'linear',\r\n",
        "                             kernel_initializer = 'glorot_uniform',\r\n",
        "                             bias_initializer = 'zeros',\r\n",
        "                             kernel_regularizer = l2(1e-3),\r\n",
        "                             name = 'encoder')\r\n",
        "        self.decoder = Dense(self.input_dim,\r\n",
        "                             activation = 'sigmoid',\r\n",
        "                             kernel_initializer = 'glorot_uniform',\r\n",
        "                             bias_initializer = 'zeros',\r\n",
        "                             kernel_regularizer = l2(1e-3), \r\n",
        "                             name = 'decoder')\r\n",
        "        \r\n",
        "        _ = self.call(Input(shape = (self.input_dim, )))\r\n",
        "        self.build(input_shape = (None, self.input_dim))\r\n",
        "\r\n",
        "    def call(self, inputs):\r\n",
        "        h = self.encoder(inputs)\r\n",
        "        return self.decoder(h)\r\n",
        "\r\n",
        "    def train_step(self, inputs):\r\n",
        "        with tf.GradientTape() as tape:\r\n",
        "            outputs = self(inputs)\r\n",
        "            loss = self.loss(inputs, outputs)\r\n",
        "        grads = tape.gradient(loss, self.trainable_variables)\r\n",
        "        self.optimizer.apply_gradients(zip(grads, self.trainable_variables))\r\n",
        "        return {'loss': loss}\r\n",
        "\r\n",
        "    def test_step(self, data):\r\n",
        "        query, relev = data\r\n",
        "        pred = self(query)\r\n",
        "\r\n",
        "        weight = tf.constant(1.0 / np.log2(np.arange(2, 102)), dtype = tf.float32)\r\n",
        "        ndcg = batch_ndcg(query, relev, pred, weight)\r\n",
        "        return {'ndcg': ndcg}"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vc70c_-V0PfB"
      },
      "source": [
        "def show_history(hist, loss: str, metric: str) -> None:\r\n",
        "    fig, (ax1, ax2) = plt.subplots(2, 1, figsize = (8, 6))\r\n",
        "\r\n",
        "    ax1.plot(hist.history[f'{loss}'])\r\n",
        "    ax1.set_title(loss, fontsize = 20)\r\n",
        "    ax1.set_ylabel(loss)\r\n",
        "    ax1.set_xlabel('epoch')\r\n",
        "\r\n",
        "    ax2.plot(hist.history[f'val_{metric}'])\r\n",
        "    ax2.set_title(metric, fontsize = 20)\r\n",
        "    ax2.set_ylabel(metric)\r\n",
        "    ax2.set_xlabel('epoch')\r\n",
        "    plt.tight_layout()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NHoctUBox1Z4"
      },
      "source": [
        "def test_predictor(query: pd.DataFrame, \n",
        "                   model: Callable,\n",
        "                   n_item: int, \n",
        "                   batch_size: int, \n",
        "                   N = 100) -> float:\n",
        "\n",
        "    query = list_agg(query)\n",
        "\n",
        "    n_data = query.index.size\n",
        "    ids = np.arange(n_data)\n",
        "    profile = query['movie_id'].to_numpy()\n",
        "\n",
        "    pred = []\n",
        "    n_batch = int(np.ceil(n_data / batch_size))\n",
        "    for batch_step in range(n_batch):\n",
        "        lower = batch_size * batch_step\n",
        "        upper = batch_size + lower\n",
        "\n",
        "        batch_id = ids[lower: upper]\n",
        "        batch = np.zeros(shape = (batch_id.size, n_item))\n",
        "        for i, idx in enumerate(batch_id):\n",
        "            batch[i, profile[idx]] = 1.0\n",
        "\n",
        "        pred_batch = model.predict(batch, verbose = 0)\n",
        "        \n",
        "        rec = np.argsort(np.where(batch == 1.0, -1.0, pred_batch), axis = 1)[:, :-N-1:-1]\n",
        "        pred.append(rec)\n",
        "    pred = np.concatenate(pred)\n",
        "    pred = [{'user_id': query.at[i, 'user_id'], 'movie_id': pred[i]} for i in range(n_data)]\n",
        "    return pd.DataFrame(pred)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G05LGp9eAfE0"
      },
      "source": [
        "class evaluate:\n",
        "    def __init__(self, true: pd.DataFrame, pred: pd.DataFrame):\n",
        "        self.true = true\n",
        "        self.pred = pred\n",
        "        self.max_K = 10000\n",
        "        self.idcg = np.cumsum([1.0 / np.log2(i+2) for i in range(self.max_K)])\n",
        "\n",
        "    def _recall(self, gt: List, rec: List, K = None) -> float: # Recall\n",
        "        res = [r for r in rec[:K] if r in gt]\n",
        "        return len(res) / np.min([K, len(gt)])\n",
        "    \n",
        "    def _precision(self, gt: List, rec: List, K = None) -> float: # Precision\n",
        "        res = [r for r in rec[:K] if r in gt]\n",
        "        return len(res) / len(rec[:K])\n",
        "\n",
        "    def _AP(self, gt: List, rec: List, K = None) -> float: # Average Precision\n",
        "        res = 0.0\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                res += self._precision(gt, rec[:K], i+1)\n",
        "        return res / np.min([K, len(gt)])\n",
        "\n",
        "    def _HR(self, gt: List, rec: List, K = None) -> float: # Hit Rate\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                return  1.0\n",
        "        return 0\n",
        "\n",
        "    def _RR(self, gt: List, rec: List, K = None) -> float: # Reciprocal Rank\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                return  1.0 / (i+1)\n",
        "        return 0\n",
        "\n",
        "    def _nDCG(self, gt: List, rec: List, K = None) -> float: # normalized Discounted Cumulative Gain\n",
        "        dcg = 0.0\n",
        "        for i, r in enumerate(rec[:K]):\n",
        "            if r in gt:\n",
        "                dcg += 1.0 / np.log2(i+2)\n",
        "        idcg = self.idcg[min([len(gt), K])-1]\n",
        "        return dcg / idcg\n",
        "\n",
        "    def _get_item(self, rec: List, K = None) -> float:\n",
        "        for r in rec[:K]:\n",
        "            self.uniq_item[r] = self.uniq_item.get(r, 0) + 1\n",
        "\n",
        "    def __call__(self, K = None):\n",
        "        self.K = K if K else self.max_K\n",
        "        self.recall = 0.0\n",
        "        self.precision = 0.0\n",
        "        self.MAP = 0.0\n",
        "        self.HR = 0.0\n",
        "        self.MRR = 0.0\n",
        "        self.nDCG = 0.0\n",
        "        self.uniq_item = {}\n",
        "        n = self.true.index.size\n",
        "        for gt, rec in zip(tqdm(self.true.movie_id), self.pred.movie_id):\n",
        "            self.recall += self._recall(gt, rec, K) / n\n",
        "            self.precision += self._precision(gt, rec, K) / n\n",
        "            self.MAP += self._AP(gt, rec, K) / n\n",
        "            self.HR += self._HR(gt, rec, K) / n\n",
        "            self.MRR += self._RR(gt, rec, K) / n\n",
        "            self.nDCG += self._nDCG(gt, rec, K) / n\n",
        "            self._get_item(rec, K)\n",
        "        self.CO = len(self.uniq_item) / (n*K) # Coverage\n",
        "        self.ED = -np.sum([self.uniq_item[i] * np.log(self.uniq_item[i] / n) for i in self.uniq_item]) / n # Entropy-Diversity\n",
        "\n",
        "    def print_all(self):\n",
        "        K = '@' + str(self.K) if self.K else ''\n",
        "        print(f'{\"Recall\":>12}{K} : {self.recall:.5f}',\n",
        "              f'\\n{\"Precision\":>12}{K} : {self.precision:.5f}',\n",
        "              f'\\n{\"MAP\":>12}{K} : {self.MAP:.5f}',\n",
        "              f'\\n{\"HR\":>12}{K} : {self.HR:.5f}', \n",
        "              f'\\n{\"MRR\":>12}{K} : {self.MRR:.5f}',\n",
        "              f'\\n{\"nDCG\":>12}{K} : {self.nDCG:.5f}',\n",
        "              f'\\n{\"CO\":>12}{K} : {self.CO:.5f}',\n",
        "              f'\\n{\"ED\":>12}{K} : {self.ED:.5f}')"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqV0G-jz0HbN"
      },
      "source": [
        "epochs = 100\r\n",
        "batch_size = 128\r\n",
        "\r\n",
        "train_gen = train_generator(train, n_item, batch_size)\r\n",
        "valid_gen = valid_generator(valid_q, valid_r, n_item, batch_size)\r\n",
        "\r\n",
        "steps_per_epoch = train.user_id.nunique() // batch_size + 1\r\n",
        "validation_steps = valid.user_id.nunique() // batch_size + 1"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EfDSYBod6ZuU",
        "outputId": "039cd160-8c6d-4f8b-d571-8226aa97eee0"
      },
      "source": [
        "model = AutoRec(n_item, 512)\r\n",
        "model.compile(optimizer = Adam(lr = 1e-4), loss = binary_cross_entropy)\r\n",
        "\r\n",
        "model.summary()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"auto_rec\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "encoder (Dense)              (None, 512)               1897984   \n",
            "_________________________________________________________________\n",
            "decoder (Dense)              (None, 3706)              1901178   \n",
            "=================================================================\n",
            "Total params: 3,799,162\n",
            "Trainable params: 3,799,162\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "aOSh_Etb0Ldx",
        "outputId": "5244e691-8b78-49fe-aff1-87bfa2a8994c"
      },
      "source": [
        "%%time\r\n",
        "early_stopping = EarlyStopping(monitor = 'val_ndcg', mode = 'max', verbose = 1, patience = epochs // 10)\r\n",
        "model_checkpoint = ModelCheckpoint('U-AutoRec.h5', monitor = 'val_ndcg', mode = 'max', save_best_only = True)\r\n",
        "\r\n",
        "hist = model.fit(x = train_gen, validation_data = valid_gen, epochs = epochs,\r\n",
        "                 steps_per_epoch = steps_per_epoch, validation_steps = validation_steps, \r\n",
        "                 verbose = 1, callbacks = [early_stopping, model_cheWckpoint])\r\n",
        "show_history(hist, 'loss', 'ndcg')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "38/38 [==============================] - 2s 18ms/step - loss: 2258.7512 - val_ndcg: 0.0829\n",
            "Epoch 2/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 1403.3017 - val_ndcg: 0.1518\n",
            "Epoch 3/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 1068.0023 - val_ndcg: 0.1471\n",
            "Epoch 4/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 942.8892 - val_ndcg: 0.1450\n",
            "Epoch 5/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 873.9939 - val_ndcg: 0.1611\n",
            "Epoch 6/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 824.1758 - val_ndcg: 0.1590\n",
            "Epoch 7/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 778.7922 - val_ndcg: 0.1480\n",
            "Epoch 8/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 724.0481 - val_ndcg: 0.1745\n",
            "Epoch 9/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 675.2253 - val_ndcg: 0.1591\n",
            "Epoch 10/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 627.7527 - val_ndcg: 0.1660\n",
            "Epoch 11/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 591.2519 - val_ndcg: 0.1643\n",
            "Epoch 12/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 559.9562 - val_ndcg: 0.1766\n",
            "Epoch 13/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 539.1954 - val_ndcg: 0.1565\n",
            "Epoch 14/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 521.4718 - val_ndcg: 0.1542\n",
            "Epoch 15/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 503.9863 - val_ndcg: 0.1645\n",
            "Epoch 16/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 487.5365 - val_ndcg: 0.1725\n",
            "Epoch 17/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 472.5797 - val_ndcg: 0.1515\n",
            "Epoch 18/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 460.5465 - val_ndcg: 0.1896\n",
            "Epoch 19/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 447.5555 - val_ndcg: 0.1694\n",
            "Epoch 20/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 436.7304 - val_ndcg: 0.1656\n",
            "Epoch 21/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 425.9069 - val_ndcg: 0.1573\n",
            "Epoch 22/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 417.2971 - val_ndcg: 0.2063\n",
            "Epoch 23/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 409.6640 - val_ndcg: 0.1671\n",
            "Epoch 24/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 400.0016 - val_ndcg: 0.2011\n",
            "Epoch 25/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 391.2711 - val_ndcg: 0.1940\n",
            "Epoch 26/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 385.3082 - val_ndcg: 0.2000\n",
            "Epoch 27/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 377.0710 - val_ndcg: 0.1936\n",
            "Epoch 28/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 371.3633 - val_ndcg: 0.2128\n",
            "Epoch 29/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 364.2458 - val_ndcg: 0.1767\n",
            "Epoch 30/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 358.7575 - val_ndcg: 0.2097\n",
            "Epoch 31/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 353.0259 - val_ndcg: 0.2112\n",
            "Epoch 32/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 347.9354 - val_ndcg: 0.1893\n",
            "Epoch 33/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 341.6939 - val_ndcg: 0.1836\n",
            "Epoch 34/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 336.9227 - val_ndcg: 0.1978\n",
            "Epoch 35/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 332.7605 - val_ndcg: 0.1995\n",
            "Epoch 36/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 328.8246 - val_ndcg: 0.1860\n",
            "Epoch 37/100\n",
            "38/38 [==============================] - 0s 6ms/step - loss: 323.8293 - val_ndcg: 0.2062\n",
            "Epoch 38/100\n",
            "38/38 [==============================] - 0s 7ms/step - loss: 319.3358 - val_ndcg: 0.2071\n",
            "Epoch 00038: early stopping\n",
            "CPU times: user 13.3 s, sys: 1.35 s, total: 14.6 s\n",
            "Wall time: 11.9 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGoCAYAAABL+58oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hcxb3/8fesZMtyle21jeUi2ZYrBgwYY0rAlIAhhJZkKKElBEKAhLSbhJAEAuH+SEJJSLgQ08kFzEAIpl56DZhmOsbg3ovcqyxp5/fHHOG1LLlIu9rV7uf1PPvs2dlzZufrtaWv50wx3ntEREREckks0w0QERERSTUlOCIiIpJzlOCIiIhIzlGCIyIiIjlHCY6IiIjkHCU4IiIiknOU4IhIVjHGlBtjvDHmrky3RURaLyU4IiIiknOU4IiIiEjOUYIjIiIiOUcJjoi0CsaY3saYm4wxs40xm40xy4wxDxtj9m3g3LbGmB8ZY6YYY1YaYzZE100yxhxZ79yvGGMeM8bMN8ZUGWMWG2MmG2Mub7noRCTVCjPdABGRHTHGDABeA0qBF4D7gX7At4CvGWO+4b1/POmSu4DTgI+Be4CN0bUHA+OB56J6xwNPAGuAR4EFQDdgOHAh8Ps0hyYiaaIER0Rag1sICcpvvPdX1xUaY/4HeAW42xhT5r1fZ4zpApwKvAvs772vTa7IGNM96eV5hJ7scd77D+qdF09PKCLSEnSLSkSymjGmL3AUMBf4U/J73vvXCb053YCT64oBA1QBifr1ee+XN/AxGxs4r7JZDReRjFKCIyLZbu/o+VXvfXUD77+QfJ73fg3wGHAg8L4x5nfGmMOMMe0buPbe6PlNY8wtxphTooRKRFo5JTgiku26RM+LGnm/rrwkqewUwviZ4uj5BWC5MeafxphedSd57x8GjgPeA74LTATmGWPeMcZ8NXUhiEhLU4IjItludfS8WyPv9653Ht77jd77K7z3Q4D+wBmEQcpnAA8lX+y9f8J7fzjQFTgCuAHYHXjcGDMiZVGISItSgiMi2e696PlgY0xDEyMOi56nNHSx936e9/5e4GhgelRP9wbOW++9f8F7/1Pgv4G2wDHNbr2IZIQSHBHJat77+cCzQDnw4+T3jDH7A6cDK4F/R2U9jDF7NFBVB6AjUANsjs49pJGkqe421oYUhCAiGaBp4iLSGlwA/Af4szHmKOAdtqyDkwC+471fG53bB3jPGPMR8CEwD+hMGGuzG3Bj0rk3An2MMf8BZhMSn32Bw4E5hDE5ItIKGe99ptsgIvIlY0w5MAu423t/TlJ5H+A3wLGEcTdrCONqrvbev510XgnwI2AcMBSIAyuAacA/gIk++sFnjLHAScDoqM4EYTr6JOAv3vtlaQtURNJKCY6IiIjkHI3BERERkZyjBEdERERyjhIcERERyTlKcERERCTn5OM0cY2qFhERyS2mfkE+JjgsXLgwLfXG43EqK/NrA+J8jBkUdz7Jx5ghP+POx5ih9cddWlraYLluUYmIiEjOUYIjIiIiOUcJjoiIiOQcJTgiIiKSc5TgpIBfvpTaP/2KqvcmZ7opIiIighKc1OjYGaZ/RvW0jzPdEhEREUEJTkqYonbQuy/V0z/LdFNEREQEJTgpY8oqqJnxGdqdXUREJPOU4KRKeQWJVStg5fJMt0RERCTvZWQlY2vtHcBxwFLn3Mio7AFgaHRKCbDKOTfKWlsOTAWmRe9Nds5dEF2zL3AXUAw8CVzinMtIF4opqwh7QMydDt3imWiCiIiIRDK1VcNdwN+Be+oKnHOn1B1ba68DViedP8M5N6qBem4GzgPeJCQ444Gn0tDeHes3AGIF+NnTMaPGZqQJIiIiEmTkFpVz7hVgRUPvWWsNYIH7t1eHtbY30Nk5NznqtbkHODHVbd1Zpm0Rhf0H4OdMz1QTREREJJKNm21+BVjinPsiqWyAtfY9YA3wG+fcq0AfYH7SOfOjsm1Ya88HzgdwzhGPp+cW0trBw6l981W6d++OMdtsbJqTCgsL0/bnmc0Ud/7Ix5ghP+POx5ghd+POxgTnNLbuvVkE9HfOLY/G3Dxird19Vyp0zk0AJkQvfbp2TW0/cCj+2ceo/HwqpnvPtHxGtmntu9A2leLOH/kYM+Rn3PkYM7T+uBvbTTyrEhxrbSFwMrBvXZlzrgqoio7ftdbOAIYAC4C+SZf3jcoyps2gYeFgznTIkwRHREQkG2XbNPEjgc+cc1/eerLW9rDWFkTHA4HBwEzn3CJgjbV2bDRu5yxgUiYaXaewvAIKCvGzNQ5HREQkkzKS4Fhr7wfeAIZaa+dba8+N3jqVbQcXHwJ8aK19H3gIuMA5VzdA+ULgNmA6MINMzaCKmDZtoU9/DTQWERHJMJOHK+/6hQsXpqXieDzO0uuvwL/7OrG/3JsXA41b+73bplLc+SMfY4b8jDsfY4bWH3c0BmebX7jZdouq9SuvgA3roHJJplsiIiKSt5TgpJgpGxwOdJtKREQkY5TgpFqf/lCogcYiIiKZpAQnxUxhG+hTroHGIiIiGaQEJw1MeQXMmYFPJDLdFBERkbykBCcdyipg43pYtjjTLREREclLSnDSwJSHgca6TSUiIpIZSnDSoXc/KGyjmVQiIiIZogQnDUxhIfQboJlUIiIiGaIEJ01MeQXM1UBjERGRTFCCky5lg2HTRlianm0hREREpHFKcNLElFcA6DaViIhIBhRm4kOttXcAxwFLnXMjo7IrgPOAZdFpv3bOPRm9dylwLlAL/Mg593RUPh74K1AA3Oacu6Yl49iu3fpC27ZhoPHYcZlujYiISF7JSIID3AX8HbinXvkNzrlrkwustSOAU4HdgVLgOWvtkOjtm4CvAvOBt621jzrnPk1nw3eWKSiAfgPVgyMiIpIBGblF5Zx7BVixk6efAEx0zlU552YB04Ex0WO6c26mc24zMDE6N2uY8sHRQOPaTDdFREQkr2TbGJyLrbUfWmvvsNZ2jcr6APOSzpkflTVWnj3KKmBzFSxekOmWiIiI5JVM3aJqyM3AVYCPnq8DvpuKiq215wPnAzjniMfjqah2G4WFhVvVXTNqP5YDHZcvoXjPfdLymZlWP+Z8objzRz7GDPkZdz7GDLkbd9YkOM65JXXH1tpbgcejlwuAfkmn9o3K2E55/bonABOil76ysjIVTd5GPB4nuW5fVAxF7Vj78Xus32O/tHxmptWPOV8o7vyRjzFDfsadjzFD64+7tLS0wfKsSXCstb2dc4uilycBH0fHjwL3WWuvJwwyHgy8BRhgsLV2ACGxORU4vWVbvX0mVgD9B2pPKhERkRaWqWni9wPjgLi1dj5wOTDOWjuKcItqNvB9AOfcJ9ZaB3wK1AAXOedqo3ouBp4mTBO/wzn3SQuHskOmbDD+lafwtbVhZpWIiIikXUYSHOfcaQ0U376d868Grm6g/EngyRQ2LfXKK+C5zbBoHvQtz3RrRERE8kK2zaLKOaYsWtFYt6lERERajBKcdOvZG9oVgxb8ExERaTFKcNLMxGJQVqEeHBERkRakBKcFmLIKmDcLX1OT6aaIiIjkBSU4LaG8AmqqYeHcTLdEREQkLyjBaQGmbBCggcYiIiItRQlOS+jRG4o7aKCxiIhIC1GC0wKMMVCugcYiIiItRQlOCzFlFTB/Nr66OtNNERERyXlKcFqIKa+A2hpYOCfTTREREcl5SnBaSv9ooLHG4YiIiKSdEpyWEu8FHTqBxuGIiIiknRKcFmKMCSsaz/4i000RERHJeRnZTdxaewdwHLDUOTcyKvsz8HVgMzAD+I5zbpW1thyYCkyLLp/snLsgumZf4C6gmLCr+CXOOd+CoewSU16Bf/phfPVmTJu2mW6OiIhIzspUD85dwPh6Zc8CI51zewKfA5cmvTfDOTcqelyQVH4zcB4wOHrUrzOrmLIKqK2F+bMz3RQREZGclpEExzn3CrCiXtkzzrm6zZomA323V4e1tjfQ2Tk3Oeq1uQc4MR3tTZmyCkADjUVERNItI7eodsJ3gQeSXg+w1r4HrAF+45x7FegDzE86Z35Utg1r7fnA+QDOOeLxeFoaXVhYuN26fffuLOtcQtHieXRJUxta2o5izlWKO3/kY8yQn3HnY8yQu3FnXYJjrb0MqAHujYoWAf2dc8ujMTePWGt335U6nXMTgAnRS19ZWZmy9iaLx+PsqG7ffxCbpn1MdZra0NJ2JuZcpLjzRz7GDPkZdz7GDK0/7tLS0gbLs2oWlbX2HMLg42/XDRZ2zlU555ZHx+8SBiAPARaw9W2svlFZVjPlFbBoHr6qKtNNERERyVlZk+BYa8cDvwCOd85tSCrvYa0tiI4HEgYTz3TOLQLWWGvHWmsNcBYwKQNN3yWmrAISCZg/K9NNERERyVmZmiZ+PzAOiFtr5wOXE2ZNFQHPWmthy3TwQ4ArrbXVQAK4wDlXN0D5QrZME38qemS3pIHGZtCwDDdGREQkN2UkwXHOndZA8e2NnPsv4F+NvPcOMDKFTUu/km7QpSvM0YJ/IiIi6ZI1t6jyxZYVjTVVXEREJF2U4GSAKauAxQvwmzZmuikiIiI5qcm3qKy1hwGznXOzokX3riGMkbnUObc4VQ3MRaasAu8TMG8WDB6R6eaIiIjknOb04PwPUBsdXwe0ISQ4Exq9QoKyQQB4jcMRERFJi+YMMu7jnJtrrS0EjgbKCBtlLkxJy3KYKekGJd3xb72K3/dgTNfumW6SiIhITmlOD84aa20v4FDgU+fcuqi8TfOblfvM10+BebNI/PYHJJ5w+OrNmW6SiIhIzmhOD87fgLeBtsCPo7KDgM+a26h8EDtkPH74KBIP3oF/5H/xrz1L7Fvfhb3HhplWIiIi0mRN7sFxzv0ROBI4yDk3MSpeAHwvFQ3LB6bHbhRc+GtiP70K2haRuPn/kbjhd/gFczLdNBERkVatWQv9Oec+rzuOZlUlnHMvN7tVecYM34vY7/6Kf/kp/KT7SFx5CebQYzAnnI7p0CnTzRMREWl1mtyDY6192Vp7UHT8S2AicJ+19tepalw+MQUFxA4/jtjVt2AOGY9/6SkSl11A4sUn8bW1O65AREREvtScQcYjgcnR8XnAYcBY4ILmNiqfmY6diX37AmK/uwH6luPvu4XEH36Cn/ZRppsmIiLSajQnwYkB3lo7CDDOuU+dc/OArqlpWn4zfQcQ+9kfiF3wK9i4gcS1l5G49Vp8TU2mmyYiIpL1mjMG5zXg70Bv4N8AUbJTuaMLrbV3AMcBS51zI6OybsADQDkwG7DOuZXWWgP8FTgW2ACc45ybEl1zNvCbqNo/OOfubkY8WccYA/seSGyPffFPPYR//AHosRvmxDMy3TQREZGs1pwenHOAVcCHwBVR2TBCMrIjdwHj65X9CnjeOTcYeD56DXAMMDh6nA/cDF8mRJcD+wNjgMuttTnZe2TaFhE74duYA4/AP/kQ/vNPMt0kERGRrNbkHhzn3HLg1/XKntjJa1+x1pbXKz4BGBcd3w28BPwyKr/HOeeBydbakmjvq3HAs865FQDW2mcJSdP9TQinVTCnnYf/4hMSt19P7PK/Ytp3zHSTREREslJzNttsQ7g9dCZQStii4Z/A1c65pizL28s5tyg6Xgz0io77APOSzpsflTVW3lBbzyf0/uCcIx6PN6F5O1ZYWJi2uutU//wqVlx6AW0fupPOP7ki44sCtkTM2Uhx5498jBnyM+58jBlyN+7mjMH5E+HW0AXAHMJeVL8FOgM/aU6jnHPeWuubU0e9+iawZRNQX1m5w2FCTRKPx0lX3V/q1gvz9VPZNOleqoaMJDb2sPR+3g60SMxZSHHnj3yMGfIz7nyMGVp/3KWlpQ2WN2cMzreA451zzzjnpjnnngFOAmwT61sS3Xoiel4alS8A+iWd1zcqa6w855ljvwkVI/D33oJftjjTzREREck6zUlwGrs30tR7Jo8CZ0fHZwOTksrPstYaa+1YYHV0K+tp4ChrbddocPFRUVnOM7ECYt/7KRhD4vbrtRCgiIhIPc25RfUg8Ji19vfAXMItqt9E5dtlrb2fMEg4bq2dT5gNdQ3grLXnEm551fUEPUmYIj6dME38OwDOuRXW2qsIG34CXFk34DgfmO49Md/+Af626/BPOMzxp2W6SSIiIlmjOQnOLwgJzU2EQcYLCNs1XLWjC51zjf02PqKBcz1wUSP13AHcsZPtzTmx/Q8l8fEU/OMP4EeMwlQMz3STREREssIuJTjW2sPrFb0UPQxQNyj4YOCF5jZMdo45/fv46Z+SuO06YpffiClun+kmiYiIZNyu9uDc3kh5XXJTl+gMbHKLZJeY4vbEzv0piT9fir/vFsy5P810k0RERDJulxIc59yAdDVEms5UDMd87RT8Y/eTGLkvsf0PzXSTREREMqo5s6gki5ivWRg0DH/vzfjKJZlujoiISEYpwckRpqCA2Lk/Be9J3H6Dpo6LiEheU4KTQ0yP3TDfvgCmf4p/6qFMN0dERCRjlODkGLP/OMyYQ/CP3Y+fOS3TzREREckIJTg5xhgTenG6xkncdh1+VfasfejXr8V/8alun4mISNo1Z6E/yVKmfccwdfy6y0j86nuYfQ7AHHoMDNm9xXYf997DkgX4GZ/B9KnheVHY/N0c803MyWe1SDtERCQ/KcHJUWbwCGJX/B3/8v/hX38e//ar0Lsf5tBjMAcchmnfIaWf56uqYM4X+Bmf4adPhZmfwbq14c32HWHQMMz+h+LnzsA//TB+9MGY/louSURE0kMJTg4zu/XBnHIu/qQz8G+/hn/5KfzECfiH78bsf2hIdsoG7XK9vroaFs3Dz5/N2spF1H40BebNhLpbT7v1wew1BgYND9tH9OqDiYW7oX79WhLTp5K4+2/Efn0tpqAglSGLiIgASnDygmlbhDnoCDjoCPyc6aFX582X8a8+A+WDMeOOwYz+CqaoaKvrfCIBy5fCgjn4BXNg/uzwvGQBJBIAbGjbNtRx1ImYQcNh4DBMp86Nt6VDJ2KnnU/iH3/CPzcJc/TJaY1dRETykxKcPGPKKjBnXYz/5jn4N14KvTp33Yh3t2MOPAJ67BYSmvmzYcFcqNq45eJ4L+hThtn7AOhbhulTRnzEnixftWrXGrHvQTBqLH7SffhRYzG9SlMao4iISFYlONbaocADSUUDgd8BJcB5wLKo/NfOuSejay4FzgVqgR85555uuRa3XqZ9R8wRx+EP/xp8/klIdF58EmprwpiZvuWYAw+PEply6NMf027bjTxN4a7/FTLGEPv290n87mIS/7yJ2M/+0GKDn0VEJD9kVYLjnJsGjAKw1hYAC4B/A98BbnDOXZt8vrV2BHAqsDtQCjxnrR3inNM85J1kjIGhIzFDR+LXrYHqaijplvaEw5R0x3zzHPw/b8K/+gzmkKPT+nkiIpJfsnkdnCOAGc65Ods55wRgonOuyjk3C5gOjGmR1uUg07Ezpmv3FutNMV85CobugX/oTvzK5S3ymSIikh+yqgennlOB+5NeX2ytPQt4B/iZc24l0AeYnHTO/KhsK9ba84HzAZxzxOPxtDS4sLAwbXVnq+bGXHPJb1n+4zNo89AddPnVNa3mVlU+fteQn3HnY8yQn3HnY8yQu3FnZYJjrW0LHA9cGhXdDFwF+Oj5OuC7O1ufc24CMCF66SsrK1PX2CTxeJx01Z2tmh1zm3aY40+n6qG7qHx6Emb0walrXBrl43cN+Rl3PsYM+Rl3PsYMrT/u0tKGJ6pkZYIDHANMcc4tAah7BrDW3go8Hr1cAPRLuq5vVCatiDnyBPzbr5G47x/Ehu+F6dAp000SEZFWLlvH4JxG0u0pa23vpPdOAj6Ojh8FTrXWFllrBwCDgbdarJWSEqaggNjZP4QN6/AP3J7p5oiISA7IugTHWtsB+CrwcFLxn6y1H1lrPwQOA34C4Jz7BHDAp8D/ARdpBlXrZPoNwBx9Mv6NF/CfvJfp5oiISCtnvPeZbkNL8wsXLkxLxa39PmZTpDJmX72ZxJWXQHU1sSv+hmlXnJJ60yEfv2vIz7jzMWbIz7jzMWZo/XFHY3C2maGSdT04kr9Mm7bEzvohLF+Kn3RvppsjIiKtmBIcySpm8AjMuGPxzz+Gn/FZppsjIiKtlBIcyTrm5LOgpDuJe/6Or6nOdHNERKQVUoIjWccUtyd2xg9g4Vz8kw9lujkiItIKZes6OJLnzJ77YcYcin/yQfyAwdCpS1jm0SegbmC894BPKie8LmwDPXpBp5KUrozsN1fB0kWwbDG1e+4DBW1TVreIiKSWEhzJWubU7+E/fY/EjVc2rYKidhDvBT16Y3rUPe8GPXeDbj0whW22ucRXV0PlYliyEL90ISxZFD0vhJVbZhksb1cMZ11MbL+vNDU8ERFJIyU4krVMpy7EfvsXmDsDjAFMmAhoYtGEQBPK6x4QnjdX4SuXwLLF+GWLYckC/CdToHozXy6KYGLQLQ49e2NKuuNXr4SlC2H5stAbVKdjJ+hZihm6B/TqHY5LulPw2P1UT/gziZnTMN84B1Oof0oiItlEP5Ulq5lu8ZCI7Op19V77RALWrISli/GVi2HZ4nC8bBF+6gfQpStm4FAYexj06o3pWQq9ShvdNqLrVX9n2S1/xj/3KH72dGLf/wWmpFsTIhQRkXRQgiN5wcRiUNIdSrpjhuze/PoKC4mdeh6JgUPxd/+NxFU/DknOkJEpaK2IiDSXZlGJNENszCHEfn0dFHcgcd1vSDzzbzKxOriv2qQp9SIiSdSDI9JMpk9/YpddR+LOv+AfvBM/cxqxc36Eadc+pZ+z1QDoJQvD2KIl0QDo1SugoBBK+2H6DYR+A6Lnckz7jilth4hIa6AERyQFTHF7Yj+4FP/Mv/H/uofEgrnELrwU07vfLtfl162BuTPxSxZEyUx4pnLp1gOgO3UJ44RG7g09S2HTBvy8WWFA9evPbxlQ3b0n9BsYNjTtPwD6DYJu8ZROoRcRyTZZl+BYa2cDa4FaoMY5N9pa2w14ACgHZgPWObfSWmuAvwLHAhuAc5xzUzLRbhFjTNgRvayCxIQ/k7j658TO+SFm9MGNXuM3rIM5M/BzpsPs6eG5csmWE4qKQxJTPhj2HxeOe5WG2VwdGu+Z8atXwryZ+HmzYN4s/LyZ+A/e3HL7rH1HKB+M2Xt/zN4HYLp0Tc0fgohIlsi6BCdymHMueWvTXwHPO+eusdb+Knr9S+AYYHD02B+4OXoWyRgzbE9iv/0LiX/8kcQ//oSZOQ1z8tlQszn0zMyeDnOmh+elSTvb99gNUz4Yxh2D6T8IevcLs7ua0NNiunSFLvtiRu77ZZmv2gTzZ29JeqZ9hL/3Fvx9/4CK4Zh9DsTscwCmW48U/CmIiGRWtiY49Z0AjIuO7wZeIiQ4JwD3OOc8MNlaW2Kt7e2cW5SRVopETNfuxH5+Nd7dgX92En7yS7BuzZZVmLvFoawCc+DhmPLBUDYI07FzettU1A4GDcMMGgYQenMWzsW/+zp+yuv4B27DP3AbDBgSEp19DsT07J3WNomIpEs2JjgeeMZa64F/OOcmAL2SkpbFQK/ouA8wL+na+VHZVgmOtfZ84HwA5xzx+K6vq7IzCgsL01Z3tsrHmGEX4v7RZWzaewybJr9MYf+BtKkYRuGgYRRky5o5PXrAXvvCd39IzYK5VE1+iU1vvETNv+7G/+tuCgcMpuiAcbQbexiF/crz8vvOx5ghP+POx5ghd+POxgTnYOfcAmttT+BZa+1nyW8653yU/Oy0KEmaEL30lZWV2zu9yeLxOOmqO1vlY8ywi3EP3xuG700NsAmgJgHZ+GdW1B4OPRYOPZZY5RL8lDeomfI6Nffdyvr7boXe/Sjee3829R2IGTwibxY21N/x/JGPMUPrj7u0tLTB8qxLcJxzC6LnpdbafwNjgCV1t56stb2BpdHpC4DkaSp9ozIRaQYT74U56kQ46kT8quX49ybj35vMxucfh6pNYYZWj90wFSNg8AjM4N3DAOhmzszy3mt2l4ikRFYlONbaDkDMObc2Oj4KuBJ4FDgbuCZ6nhRd8ihwsbV2ImFw8WqNvxFJLVPSHXPY1+Cwr9G9pITK997Gf/EJ/otP8R+9A2+8EBKeziVQMQIzeHhIePoOwBQU4GtrYe3qsFXG6pVhhlf08GtWwupVYR2fNavCNPh+AzHlg8Msr/LBYb+wWO6sSepXr4S5M/BzZuDnzoB5s6Brd8zog8O4pzzpGRNJt6xKcAhja/5trYXQtvucc/9nrX0bcNbac4E5gI3Of5IwRXw6YZr4d1q+ySL5wxQWYgYMxgwYHHp3vIfF8/FffApffBoSnymvh4SnqBiKimDtmq3X76nTvgN07hpmig0YEo59Aj9nOv7Vp+H5x0I9xR2gvAJTXkFd4kPX7F/Hx3sPK5fD3On4OTNDMjN3BqxaseWkXn0wZRX4xfPx90/AT7wVhozEjD4oJDudSzIXgEgrZzKxrHyG+YULF+74rCZo7fcxmyIfYwbFvT1+RSX+i09gxlSoqYEu3aBLCSZKZujSFTqXYNoWNV5HbS0smouf9UVYH2j2F7BgNtTWhhM6l4QenrJBITHq0DGs2Ny+Q1jjp30HKO6Qkl3ed/a79qtWwKzP8bM+j5KZmaHnCsLu9b37hun/ZQPDc7+BmOItq137hXPx77yGf/s1WDw/XDNsj5Ds7H0gplN6Z9l92Y6NG2D6p5T06c+qknhO9Z7tiP5dt07RGJxt/sejBCeFWvtfkqbIx5hBcWeCr94c1u+Z9QXM/iIkPUsWbJl635CidlsSnij5MV27h4USe5ZCz97QoxemsE2jVTQUs9+0ISzQGCU0zPoCVkbnFBRA7/4h+SobFJKZvuVhmv7OxOk9LJizJdlZuhBiMRi2Z7iNtffYlC4p4GtqQmI29X38p+/DrM8hEfW4demK2WM0Zq8xMHwUpqjxpDQX6N9166QEZwslOCmUjzGD4s4WvnozrF8HG+oe68Pq0BvWf/maDevw69fDxvWwfi2sWBbK65gYdO8RJT29oVfvKPkphXhP4j16UPnBlCiR+TwkVgvnbkmseuwWbrENGBKe+w3Ybu/ULsXnfVic8e1X8e+8BssWhyRK/MUAACAASURBVARq0DBMrz7Qo3doc8/eoR3tineuzsXz8Z++j5/6AUz7CDZtDH8O5RWY4Xthhu1Jx9pq1v7nBfzH74b327QNSdZeYzB77hcSxabGlUjApg2hly2LbjVm29/vltLa424swcm2MTgiIjvNtGkLJd3Co65sB9d470Ois2Qhfumi0EOydBF+yUL8m5/DxvVb9vGKxVhaUAjVm8Prjp2gfEgYHzNgCAwYnNYFGo0x0capA/AnnRlWwn771TDW6f03Ye1qtvovaueSkOh8mfRECVDHzvgZn8HUD0Ivzarl4fweu2H2PxQzfFRIXpK2/yiOx1k/cnTYpf7zT/Afvo3/4C38R++EzyyrCInOXmOg/8CtEhW/aWNIJFcsw69YBssrYeUy/IrKqLwSamvCfmrlgzFlYYwVZRUaZC0pox6cFGrtWXBT5GPMoLhzlfc+rDgdJTwsXUhxQYyNPftgBg6FeK+s6nHwGzfAskWhvcsWh+eli0JPz8oGvqcOnTDD9oQRe2GGj8L02K3Ruhu8NVe3+vUHb+E/fBtmTgs9WSXdoW95GEC9YlnoPUsWi4VzuvXAdItDtx7QoSMsnBf2X1s0f8tA9JJuIdEpr8CURUnPLgy29okEbK6CzZtCr1SHTjs9jijX/343prXHrR4cEZEdMMaEXoVOXb7c0qJTPE5Vlv7wN8Xtof8g6D9om5/ufnNV2Lh16SL86pVhTFD/gZhYQdM/zxjoU4bpUwbHfgu/ZhX+o3fxH74VkqpucUzF8JDAdItjuvcIx126YQoa/1xftSlsDpu0T5v/8O0tm8N2i0P/Ckxxe/zmTVAVJTBVVSGZqdq05XVdb1udgoJocHu30I6SbuF1SXRcEsrp0GmX/iy89+ATzfrzlPRSgiMikoNM2yIo7Q+l/Xd4267Jn9G5BHPQEXDQEc2rp6hdWEOpYsSXZX7jhnBLbk6U9MyZEW6XtS0Kj6J2W2bj1b0uKoK2Sc+JRFhjafXKMMtt2aIww2/92vAZyY0oKGRZlxISiZC4kEh6+AaO65Kvnr0xI6ONbYeOTNn4K2k+JTgiIpJ1THH7kDAMHZnyun315rDY5KoVsHoFftVKWL2ctpurqNq0KdxS+/JREG51JZcZE54h9Da99gz+hcfDQOyhIzEjR2P22CcMVpeMUYIjIiJ5xbRpC/Fe4cGWwRtdmjgWxW+uCgOxP34X//EU/MQJ+IlkvHfH19ZC1cYwC65qU3jetBGqNoaB4FH5hi4l+K49wurjSQPNWzslOCIiIs1g2hbByH0wI/cBwC9dtCXZqd+7M2AotGsHbaJba23bhuvbtIW2bUNZ0nsUFMLGDVuWQli/Fr8+WgJh/dotyyOsX7tlyYS65KX+eKRGrE1+0a1HmLnXtxzTbyD0K4f4bq1ywUclOCIiIilkevbGHH4cHH5cuB027eMtCc/HU7Y5v8lzmQsKooUsO4ZZaZ26hPWRiouhXfQo2nJsito1WN6tfTErPngXP38WzJuNnzczLAdQt+BjUTH0LcP0GxB6efqUhcHZO1iRPNOU4IiIiKSJadN2696dmprQs1JdtWXWV/XmLcebq8Itr+iYmhoobh9uHXXolJTQdICi4pQsW1DQtTsmqY0QjVNaOBc/d2ZYbHL+LPybr8BLT22dkBUVQ+cuYQ2mTl3ClP5OXaBTSUiA6t4r6b7V1iQtQQmOiIhICzGFhVBYCNv5ZZ8NKy2ZNm3DGkRlFV+Wee9h+dKQ+KxZBWtWhf3W1qzGr10FyxbjZ07baoPdumTIHHkC5pRzWzSGrEpwrLX9gHsIu4p7YIJz7q/W2iuA84Bl0am/ds49GV1zKXAuUAv8yDn3dIs3XEREJMcZY74cnL29JMwnasN4oDWrYc1K/NrVYUXtFpZVCQ5QA/zMOTfFWtsJeNda+2z03g3OuWuTT7bWjgBOBXYHSoHnrLVDnHO1LdpqERERAQiLH0YLZtInfesw7UhWDYt2zi1yzk2JjtcCU4E+27nkBGCic67KOTcLmA6MSX9LRUREJJtlWw/Ol6y15cDewJvAQcDF1tqzgHcIvTwrCcnP5KTL5tNAQmStPR84H8A5RzweT0ubCwsL01Z3tsrHmEFx55N8jBnyM+58jBlyN+6sTHCstR2BfwE/ds6tsdbeDFxFGJdzFXAd8N2drc85NwGYEL306dpUrLVvWNYU+RgzKO58ko8xQ37GnY8xQ+uPO9pscxtZl+BYa9sQkpt7nXMPAzjnliS9fyvwePRyAdAv6fK+UZmIiIjksaxKcKy1BrgdmOqcuz6pvLdzblH08iTg4+j4UeA+a+31hEHGg4G3dvQ5jWV7qZDOurNVPsYMijuf5GPMkJ9x52PMkJtxZ9UgY8JYmzOBw62170ePY4E/WWs/stZ+CBwG/ATAOfcJ4IBPgf8DLtqJGVQmXQ9r7bvprD8bH/kYs+LOfDsUs+JWzIq73mMbWdWD45x7jYYb+uR2rrkauDptjRIREZFWJ9t6cERERESaTQlOak3Y8Sk5Jx9jBsWdT/IxZsjPuPMxZsjRuI33Td7HVERERCQrqQdHREREco4SHBEREck5WTWLqrWy1o4H/goUALc5567JcJNahLV2NrCWsJN7jXNudGZblB7W2juA44ClzrmRUVk34AGgHJgN2Gj7kJzQSMxXAOcBy6LTfu2ca3SGY2tkre0H3AP0IqycPsE599dc/r63E/MV5PD3ba1tB7wCFBF+Fz7knLvcWjsAmAh0B94FznTObc5cS1NrO3HfBRwKrI5OPcc5935mWpka6sFpJmttAXATcAwwAjgt2uU8XxzmnBuVq8lN5C5gfL2yXwHPO+cGA89Hr3PJXWwbM8AN0fc9Kpd+2SWpIex1NwIYC1wU/XvO5e+7sZght7/vKuBw59xewChgvLV2LPBHQtwVwErg3Ay2MR0aixvgv5K+71ad3IASnFQYA0x3zs2MsvyJhF3OJUc4514BVtQrPgG4Ozq+GzixRRuVZo3EnPOcc4ucc1Oi47XAVMIGvjn7fW8n5pzmnPPOuXXRyzbRwwOHAw9F5Tn1XcN24845ukXVfH2AeUmv5wP7Z6gtLc0Dz1hrPfCPaFPTfNErafuQxYTu/XxwsbX2LOAdwv/6c+I2TUOsteXA3sCb5Mn3XS/mg8jx7zvqgX8XqCD0xM8AVjnnaqJT5pODyV79uJ1zb1prfwBcba39HVEvpXOuKpPtbC714EhzHOyc24dwe+4ia+0hmW5QJjjnPDn6P6B6bgYGEbq1FwHXZbY56WOt7UjY9PfHzrk1ye/l6vfdQMw5/30752qdc6MIGzWPAYZluEkton7c1tqRwKWE+PcDugG/zGATU0IJTvPl7Y7mzrkF0fNS4N+EHxD5Yom1tjeEzWCBpRluT9o555ZEPxgTwK3k6PdtrW1D+EV/r3Pu4ag4p7/vhmLOl+8bwDm3CngROAAosdbW3d3I6Z/nSXGPj25V+qjX5k5y4PtWgtN8bwODrbUDrLVtgVMJu5znNGttB2ttp7pj4Ci27PKeDx4Fzo6OzwYmZbAtLaLuF3zkJHLw+7bWGuB2YKpz7vqkt3L2+24s5lz/vq21Pay1JdFxMfBVwvijF4FvRqfl1HcNjcb9WVICbwjjjlr9962VjFMg2vH8L4Rp4ndEG4DmNGvtQEKvDYSxXPflatzW2vuBcUAcWAJcDjxC2Mm+PzCHMG04ZwblNhLzOMLtCk+YKv39pHEpOcFaezDwKvARkIiKf00Yk5KT3/d2Yj6NHP6+rbV7EgYRFxD+s++cc1dGP9smEm7TvAec0drHoiTbTtwvAD0IG16/D1yQNBi5VVKCIyIiIjlHt6hEREQk5yjBERERkZyjBEdERERyjhIcERERyTlKcERERCTnaKsGEclr0fYEs4A2SUv0i0grpx4cERERyTlKcERERCTnaKE/Eck61tpS4G/AIcA64Abn3I3W2iuAkUAtcCzwBfAd59wH0XXDCZtEjiLsIXSpc+7R6L1i4A+EZfhLCCv3fpWwM/gs4BzgKqB99Hk5uTK3SL5QgiMiWcVaGyPs8TYJuIaw4eFzwA8ImyFeRthGYBJwCXARMCS6fCpwB3AtcHB0zmjn3DRr7U3A7sC3gcXA/sC7QG9CgnMb8KOorreAUc65qWkOV0TSRAmOiGQVa+3+wIPOuf5JZZcSEo85hJ2Px0blMUJPjY1OfRAojXbArttTaxpwJbAeGFvX25NUdzkhwennnJsflb0FXO+cm5iuOEUkvTSLSkSyTRlQaq1dlVRWQNgQcg4wr67QOZew1s4HSqOieXXJTWQO0IewaWg7YMZ2Pndx0vEGoGOTIxCRjFOCIyLZZh4wyzk3uP4b0RicfkmvY4RbWAujon7W2lhSktMf+ByoBDYBg4CtenBEJDcpwRGRbPMWsNZa+0vgRmAzMBwojt7f11p7MvAoYcxMFTAZMISel19Ya68DDgK+DuwX9fTcAVxvrT0TWAKMAaa0XFgi0pI0TVxEsopzrhY4jjATahah9+U2oEt0yiTgFGAlcCZwsnOu2jm3mZDQHBNd8z/AWc65z6Lrfk6YOfU2sAL4I/oZKJKzNMhYRFqN6BZVhXPujEy3RUSym/73IiIiIjlHCY6IiIjkHN2iEhERkZyjHhwRERHJOUpwREREJOcowREREZGcowRHREREco4SHBEREck5SnBEREQk5yjBERERkZyjBEdERERyjhIcERERyTlKcERERCTnKMERkZxgjCk3xnhjzF2ZbouIZJ4SHBEREck5SnBEREQk5yjBERERkZyjBEdEMiZ53Ex0PNEYU2mM2WSMeccYc1wD13QyxlxvjJkfnfeZMeanbOfnmTGmvTHml1Gda40x64wxU40xNxpjetU7d4gx5l/GmJXGmPXGmNeNMV8zxpwTtfWc1P9JiEiqFWa6ASIiQBnwFjAT+CfQDTgFmGSMOdJ7/yKAMaYIeB7YD/gAuBcoAX4LHNpQxcaYrsCLwF7ANOAOYDMwCPgO8DCwJDp3GPA60BV4AvgQGAj8G3gyxTGLSBopwRGRbDAOuMJ7//u6AmPMfcD/Af9FSFAAfkZIbh4GvuW9T0TnXgO820jdNxGSm1uAi+quia7rCBTUO7crcKH3/uak845BCY5Iq6JbVCKSDeYAf0gu8N4/DcwFxiQVfwdIAL9ITlS897OAG+tXaozpSegJWgT8PPma6Lp13vvV0bn9gMOB6cA/6p33FPBcU4MTkZanBEdEssH73vvaBsrnEXpUMMZ0AiqABd77GQ2c+1IDZfsRfs694r1fv4M2jIqe36ifCEVe28H1IpJFlOCISDZY1Uh5DVt+TnWJnpc0cu7iBspKoucFO9GGHdXfWLmIZCElOCLSWqyOnns18v5uDZTVJU59dqL+NTuov7FyEclCSnBEpFXw3q8ljI/pY4wZ1MAp4xooe4swZucQY0yHHXzE+9HzAcaYhn42HryzbRWRzFOCIyKtyZ2En1t/TE5CjDEDgB/VP9l7vwyYCPQGrq2fuBhjOhpjukTnziWM46kAvl/vvPHAkSmNRETSStPERaQ1uQ44EfgGMMUY8zRhnI0FXgGOb+Cai4GRwAXAuOiazcAA4Ojompeicy8C/gP8jzHmWLasg/MNYBJwAqFHSESynHpwRKTV8N5XEXpSbgB6AJcQFvj7A/CTRq5ZCRwI/AaoBs4HfgDsTlj079Okcz8FDiAs7PcV4MdAOXASW2ZR1Y3VEZEsZrz3mW6DiEjWM8bcC5wODPPeT8t0e0Rk+9SDIyISMcbEjDHbzMYyxhxBWDDwUyU3Iq2DxuCIiGzRFphnjHkR+IywDs/uwFcJ43YuymDbRGQX6BaViEjEGFMA/IWwZUNfoD1QSRjAfI33/r0MNk9EdoESHBEREck5+XiLShmdiIhIbjH1C/IxwWHhwoVpqTcej1NZWZmWurNVPsYMijuf5GPMkJ9x52PM0PrjLi0tbbBcs6hEREQk5yjBERERkZyjBEdERERyjhIcERFpcX6tdryQ9FKCIyIiLcq/+x8SPzsT//nHmW6K5LC8nEUlIiKZ4devI3HfP8B7/OSXMENGZq4tmzZC5RKoXIJfvpR1ePzBR2GK2mWsTZI6SnBERKTF+H/dBevWQPlg/JQ38KdfgClMz68iX70ZKpfC8iX4KJGhcmk4Xr4E1q3d6vz1gFm7BnPiGWlpj7QsJTgiItIi/Ocf4199BnPUSZjBI0jcdDV89iGM3Cfln5V46l/4h+/eurCwELr3gu49MWUVEO8J8V6YeChr8+97qHr2Efwh4zHd4ilvk7QsJTgiIpJ2vrqaxD9vCsnF8adBLAbF7fHvvIZJcYLjE7X4Fx6DAUMwh38tSmB6QZeumFjjQ087nfkDqia/hJ90L+Y7l6S0TdLyNMhYRETSzj/1ICxeQOyMCzFF7TBt2mL22h//3hv4murUftjUD2HVCmJHn0Rs7GGYihGYrt23m9wAFPTsjTn86/g3XsDPm5XaNuUJ7z1+wzr8ssX42V/gP3mPxFuv4OdMb/G2qAdHRCRFfE01/sUnMYccrYGqSfzCufgnH8Lsf+hWvTVmv4Pxk1+EqR/AHqNT93mvvwDtO8CeY3b5WnPst/D/eY7Eg3cQ+8mVGLPNFkd5zVdV4V96ElZWwvq1+PXrYP1aqHvesA4SiW0v/OoJ4bZgC1KCIyKSKh++g3e3Q5s2mHHHZro1WcEnEuHWVLtijD136zdHjILiDvi3X8OkKMHxGzfg338Dc8DhmDZtdvl606Ej5rhT8A/cBp9MgZH7pqRducL/+x78849BcQfo0BE6dIIOHTHde4bX7TtBx6gseo8OnaBL1xZvqxIcEZEUqVvXxb/3JijBAcC/+gxMn4o55xJM55Kt3jOFbTB7j8W/NxlfXd2khGSbz3v3P7B5M+aAw5tchxl3DP6Fx0k8eCex4aMwBQXNblcu8PNm4V94AnPoeGJnXJjp5uyQxuCIiKSInxYtXDftI/yG9ZltTBbwq5aHaeFD98Ac2HDCYUYfDBvXw6fvpeYz33gRevWBgUObXIcpbEPsG2fDwrn4159PSbtaO+89iftugQ4dMCedmenm7JQW68Gx1o4H/goUALc5566p9/5Pge8BNcAy4LvOuTnRe2cDv4lO/YNz7u6ofF/gLqAYeBK4xDnn0x+NiMjW/Pq1sGB2mPL88RT8x+9ixhyS6WZlVGLirVBdTezMixofyzJ8L2jfMcym2mvXx8wk88sWw+cfY048o/ljZ/Y5EAYNw0+6F7/fVzDtiptXXyvn33gx9MSddXG49dQKtEgPjrW2ALgJOAYYAZxmrR1R77T3gNHOuT2Bh4A/Rdd2Ay4H9gfGAJdba+tu5t0MnAcMjh7j0xyKiEjDvvgEvCc2/pvQqQu8/2ZaPsZv3IBfMCctdaeSf/9NePd1zHGnYHqVNnqeKSzE7HMA/v03w8J8zfnMN18KdY49rFn1ABhjiH3ru7B6Jf6ZR5pdX2vmN6zDP3RnmHZ/0JGZbs5Oa6lbVGOA6c65mc65zcBE4ITkE5xzLzrnNkQvJwN9o+OjgWedcyuccyuBZ4Hx1treQGfn3OSo1+Ye4MSWCEZEpD4/7WNo0xYGDsXsNQb/8bupn/4M+AduJXHFD6m95Rr88qUprx/AV23C19Q0/fqNG8J2DH3KMEefvMPzzeiDYdPGMKi3qZ/pfehlGLoHpnuPJtezVbsGDcPsexD+6Yfxq1akpM7WyE+6D9atIfbtH+xwqn02aamW9gHmJb2eH5U15lzgqR1c2yc63tk6RUTSxn/+cUhu2rTBjNofNm6AaandTNJXVeHfeR1K+8NH75D47YUkHr0PX1WVmvqXLyNx7y0kfvxtKi/8FonXnsXX1u56PY/8L6xaHm5N7cw2DMP2hI6d8G+/1oRWR2Z8BksXNTrWp6nMyWdBbS3+0ftSWm+q+PXr8LO+IPHmyyQen0ji7r/hp36QuvrnzgxLHxx6DKZsUMrqbQlZN4vKWnsGMBo4NIV1ng+cD+CcIx5PzxLchYWFaas7W+VjzKC488nOxJxYt4Zl82bR4ZRz6RiP4w8+gqW3XkvRZ+/T+dCvpqwtm159ltVVG+n6g19Q0KuUtffcRNVjEzGTX6Tj2T+k6MDDmjT2pHbJQtb/6x42vvgkAO0OOYraebOpvvtvxJ6dRMfTvkfRgYfv1P/eqz//hBUvPkHxMd+g8/4H73Qb1hx4OJteeYbunTphiop2OYY1D77OxqJ2xL96HLHiDrt8PTTyXcfjrD32G2x44kFKvnEmhS38S957j1+zippF86ldPJ/aRQuoXTw/vF40H79uzVbnm3bFJN56ma5X3Ejb4Xvu1Gc09nfcJxKsvO4yfKfOxM/9EbGOnVMSU0tpqQRnAdAv6XXfqGwr1tojgcuAQ51zVUnXjqt37UtRed965dvUCeCcmwBMiF76ysrKXQ5gZ8TjcdJVd7bKx5hBceeTnYnZv/8meM/GvgPZVHfuiFFsnPwKVSednbJu/dpnH4OucVb37IsxMTj7EmJjjyAx8VZWX/sbGDKS2GnnYfoO2Kn6/JKF+CcfDIvtxWKYg4/CjP8G1d170L17dyqfe4LaR/6X1df9Dh64k9hJZ8AeoxtNonxNDYkb/wBdulE1/pu79HfFjxyNf2YSlS8/jdnnwJ2+DsKmmolXn8PsfQAr1m+E9Rt36fo6jX3X/vCvw/OPs/y2v1BwyeVNqrsp/NQPSNx6LaxdvaXQxKB7D+jZGzP6IEyP3pievaFnb4j3gqpN+Gt+yco//IzYL/6I6dN/h5/TWNyJ/zyH/+wjzDmXsGLTZtiUnf/2S0sbHuPVUgnO28Bga+0AQhJyKnB68gnW2r2BfwDjnXPJN5afBv47aWDxUcClzrkV1to11tqxwJvAWcDf0hyHiMg2/OcfQ2EbGDjkyzIzKqzvwpwZMGBw8z9j7Wr49D3MkSdslTCZoSOJ/fZ6/CvP4B/5XxJX/gRz6NGYE76NaeR/3H7hXPwTD+LffhUKCzGHfQ1z9MmYrt231GsMZu+xxPbaD//Wq/hH7yPxt6tg0DBiJ52JGbrHtvU++wgsmEPsol9jitvvWoBDRkKnLmHRv11NcN5/CzauxxzQ/MHFDTEdO2O+ZvEP3on/9D3MiL3T8jnJ/KJ5JG6+Bkq6YY79Vhio3aM3xHtiCrezXlDbImI/voLEH39J4q9XEPvVHzHddn1Mkl+/Dv/QXTBoWNr+XNOtRcbgOOdqgIsJycrUUOQ+sdZeaa09Pjrtz0BH4EFr7fvW2keja1cAVxGSpLeBK6MygAuB24DpwAy2jNsREWkxflrd+Ju2X5aZPUdDLBZ6d1LxGe+8BrW1mLHb3r03sQJi444hdvUtYZG6V54mcdkFJF58YqsxNH7+LBK3/JHEFT/Ef/Am5qgTiF1zK7FTz9squdmm7rHjiF35P5gzLoTly0hcexm11/8WP+vzLXUvXYh/bCLscwBm1Nhdjs8UFGD2PRD/4dv4qk27dK1/4wXoGodh2yZdqWIOOw669yTx4J34xK6PS9oVfs0qEjdeCW3aELvkcmJHHo/ZYzRmtz7bT27q2tpjN2KXXAGbNpD4yxVhCYNdbcMj/wvr1xE7/YJWNbA4mfE+75aN8QsXLkxLxeq+zx+KO3/sKGa/YR2JH5+BOc4SO36rjmlqr70M1q6m4Pd/b3Y7aq/5BWzaSMEVO+6o9gvmhDVoPvswzGQafzL+3dfD1PXi9pjDjsMceTymU+NjKhq9XbO5Cv/SU2HzzHVrYdRYYiecTsLdDrO/IHblTZiShpOlHbZ72kckrr2M2Pd/EWZW7cw1a1aS+K/vYI4+idjJZzfpc+vs6LtOvPUK/tZrMedcQuygI5r1WY3x1ZtJXPcbmDeT2M//GzNgyI4vaqyuaR+R+MvlUFZB7CdXNTq2qX7cfs50Elf/DHPY14iddn6TP7+lRLeotrlv2jrTMhGRbPHFVPAJzJCR27xlRo0Jq+Eubd5/qvzSRTDjM8z+43bqfNOnjNhPryJ2wa9g00b87TeEBfC+fhqx/3cbsZPO2G5ys9262xYRO+pEYv99K+b402HahyR+/yOY+gHm5LObnNwAMHgEdC4h8c7Oz6byk1+GRKJZWzPsLLPfV2DAEPwj/0zZzLVkPpHA3/lXmPEZse/+tFnJDYAZ+v/bu+/wqor0gePfOQmhIyXU0BFFpClVRURdFXufxYqua9ld21rW3kVxdy3oDwtiXxHHjq4uKjZUmgICiiJCAknoHUJLzvz+mBMMIeXe5Lbc+36eJw/JuafMcCF5M/POOz3x/nw9LP4Ff+w/Q1oRZ30f/9WnoeE+qFPPrfT8RCYBjhBCVIPLv0kvc2sA1XugO6ea01R2+pfufgNDr4yslEL1PRTv3jFummPUc3innIOq36Babdl9/7r18E4ejvfAWNSwM90O6kOOq949PTdNxbzvsNtDSxS2Uz+Hjl1RrdtVfnI1KaXwzroYNqzDfvpexO9v3xuPnTkFdeYI9/cQAarvYahzLoe5M7GvjKGyWRv7zaewZCHqrItR9SLzbyVeJMARQohqsL/Mc/k3GXsP/6vmraBtx2oFONZaF+Ds37NKyaIqozaqR9/wk35DvX+DRnhnjnA1byKQq6H6HQ47d2Lnzqz0XLtsCeQuiXjtm4qo/Q6EgwZhP3oLu2l9xO7rfzMZ+6FBHX5sSMURw+EdeYLbIf2bT11uTTns1s3Yt1+CfbujBg2NaBviQQIcIYSoIrutAJYuLnN6qpjqMwgW/exWQVVF9iJYmYcaGLHSYIlt3wNgn6YuqboSdupnkJbupo5iyDtjBBTuxE58LSL3s7/Mw74yBg7ojTr3iurvo1UGdcq5qMOPxX74Bv7kD8pux9uvQMFWvPMuj0obYk0CHCFEQrBLF+N//G6lQ+gJZdFP5ebfFFN9BoL1QxqRKIud/oVbyh2hKYtEpzwP1e8wmPe9YvTOagAAIABJREFUCyDLYYuK3MhWr37lLoePFtUqCzVkGHbKx9XeF8wuz8V/8gFo0RrviptCq/xcBUop1Hl/gd4D3HYfpapG2yW/YqdMQh11Ush1lBKdBDhCiITgv/8a9o3nYfa0eDclZPaX+ZCWDp27lX9S+87QNNPVxAn3/kVF2BlfQa/+NT4fIhyq32Ao3IX9YUb5J/00GzZtwItBcnFZ1MnDoU5d/JHX47/6tNvJPEx280b8J+6FtHS8q+6I+nus0tLwLrvR7ZL+/CO7t3SwRUX4rz4FjRq7xPEkIQGOECLu7PZt8ONsAPzXx4VdByVe7ML5boflCrYWUEq5ZOMFc8JfebPgB9i8ES/E1VNJo/P+0CSzwmkq++1n0KAh9Owbw4b9TjXcB++2h1GDhmKnfIx/+xX4zz6MzV0S0vV21078MSNhwzq8K293+VoxoDJq4115O7Rog//kA9ilv7Ft8vuQswh19p+ilqsVDxLgCCHi78dZsGunW5a6bjX2wzfi3aJK2e0F7odCBdNTxdRBg2DnTjfqEM4zpn8B9epDz35VbGXNpDwP1fcw+HEWtmDrXq/bgi3YOdNR/YeEVPguWlSLNngXXon34LOoP5yK/WEG/j3XUDT6HuzC+eVOt1prsS8+7paDX/J3VBkr8KLa7voNXSHAevXxR9/Dlleehv16oAaEvkqvJpAARwgRd3bWNGjQCHX82ahBR2InvYNdkRvvZlVs0QJXf2X/ygMcuh4I9eqHtZrK7tiOnT3NLfOtFb8f4vGi+h0GhYVl/p3Z776Gwl0xXT1VEdWkGd7ZF+M99BzqtPMhZxH+v27Ff+gm7JzpWN/f43w7cTx2xleoMy50gVw82tw0E+/ae9zfccFWV7E4CRKLS5IARwgRV3bXLuy8mag+A125/rMvgowM/NfGJnTCsV04H9LSoEsF+TcBlZ6O6tkPO3dGSMXWIKids2N7UizXrZLO+0PT5mVOU9mpn0PrdtBh3zg0rHyqfgO8EzXeqHGo866Ajevxx4zEv/sq/G8nYwt34X/7GfaD11GDj0ENOzO+7W3dDu/mh2h85yMhbcpZ00iAI4SIr5/nwrYC1MGHAKAaNUGdej78NAdmfRvnxpXP/jLfFZirXSek81WfgW5rg98WhHb/aV9A00zYt3s1WllzKaVcsvFPc7Bbt+w+blflw6IFqEOOStgRB5VRG2/oCXj3P4368/WQloZ9YTT+rZdjX/4/6NYLdd5fEqL9qnU7avfuH+9mRIUEOEKIuLKzp0KdutCt9+5jaujx0K4T/uvPhVzRNpbs9m0h59/s1uNgSE/Hzq58mspu2uB2Dh94RI3d6DASVL/BULTnNJWd+gUoVSPqAqm0NLyBR+DdORrv6rugRWto1wnvipujthxc/C51/+cIIeLO+kUuWbRnvz3yTFRaGt65V8D6NdgPXo9jC8vx289uZ+/9Q9+9WtWpB916Y3+YXnm5/Jlfu/yeVFs9VVrHfSGzJfa7KUCwV9PUz1xBvKaZcW5c6JRSqJ59SbthJGm3PRyx7TJExSTAEULEz6IFsHnj7umpktS+B6AOOxr76XvY5cvi0Ljy2V/mgeeFlH9TkjpoIKxeAZUUh7PTv4C2nVBZHarRyppv9zTVgh+wWza5woprV6EOOTLeTRM1gAQ4Qoi4sbOmQnot6FF2LRN1xgioXQd//DMJlXBsFwb5N3XqhnWd6jXAXV/Baiq7Mt9tdpiqycWluGmqIuzsaa72Te26qIP2DoiFKE0CHCFEXFhrXXXfAw8qN1BQjRqjTrsAfp4b0t5EsWB3bIfsX0NbHl6KatwUOu9fcYAz/UuXY5JkNUmqrH1naN4K++1n2O+/QfU9NOTEbpHaJMARQsTH0t9g3epKfxtXRxwH7btgzXOuuF68FeffhJNgXILqMxByFmHXrd7rNbdz+Bdu5/AmzarZ0OSglHKbaS76CbZvS5jaNyLxxSyNW2s9DBgNpAHjjDGjSr0+BHgM6AUMN8a8GRw/Eni0xKndgtff1Vq/CBwBFG/Te5ExZk5UOyJEmOym9bA8N6yE1FRgZ00Fz0NVskRVeWl4512B/+CN2PcnoM7+U4xaWDb7y3yXf7PvAVW6XvUZhH37ZewPM1BHnrjni0sWwqrlqOPPikBLk4fqN9hVt27WwhVNFCIEMQlwtNZpwBjgGCAXmKm1nmiM+anEaUuBi4AbSl5rjPkc6BPcpymwCPi4xCk3FgdDQiQau241/r9vg9Ur8G4ahUrRmiZlsbOmuvLwIewErTrvjzr8WOynE7GHHh3X5Fu7cD502NetiqoC1bottMxy01SlAhw7/UtIr4U6ODV2Dg9Z247QewCqx8EpvWxehCdW/1IGAIuMMYuNMTuBCcCpJU8wxmQbY+YCflk3CJwFfGSMSYBxahELdlU+dsPaeDejSuy6NS642bIJ9mmK/5+nQq5im+zs8mWwIrfM1VPlUadfCHXrxzXh2O7Y4RKAqzg9VUz1GQi/zMMWlChgV1iInTkF1XsAql796jY1qSilSLvydryhJ8S7KaIGidUUVRZQcp1nLjCwCvcZDjxS6thIrfWdwGTgZmPMXtv1aq0vAy4DMMaQmRmd+gnp6elRu3eiimaf7Y4drLnhFkhPp8m/XyCtcdOoPKcqKut30drVrH/sLtSWTTS5ezRF69aw8aFbqDf9c+qfMjyGLY2sSL3fW7/4L1uAZkedQFqzEO+XmUnBhX9h81P/pMFPs6h7xHHVbkcoSvZ559zvWF9UyD79D6V2Nf4edg49jvWT3qZBzkLqHn4sADu+n8qGzRtpdMzJ1EmA7yPy/Sx1JGu/a0wpRa11a6AnMKnE4VuAFUAGMBa4Cbi39LXGmLHB6wB2zZo1UWljZmYm0bp3oopmn/0vPsJuXA9paax58Ga86+5DpaVF5VnhqqjfdsNa/H/fDhvX4V17DxubtsQ2aQE9+rLltWcp6H4QqnHNSiC11sLKfDIP7MXatdUfUSv6ejJ02o/1VkEY/35sn0OgY1c2vfA4Wzp1i8lIR8n32p/5DSiPTS3aoqrx7942bQGNGrP5q0/ZesDB7t4fT4T6DdncoStbEuD7iHw/Sx01vd9t2rQp83ispqjygHYlvm4bHAuHBt4xxuwqPmCMWW6MscGozQu4qTCRBKxfhP3kXVdrZMTVsHA+9q0X492sStkN6/Afvh02rMO75m5UUAhOKYV3zmVu517zfJxbGR67fi3+/92Pf8df2PbRW9W/39rVbpuDMKanihUnHLNpA/b916rdlnDZhfOhfWdU3arl3xRTXhqq9wDs/O/dZqPbt2HnBDuHp6fezuFCREOsApyZQFetdSetdQZuqmlimPc4B9jjO1owqoPWWgGnAfMj0FaRCGZPh1XL8YadgXfIkagjT8B+8h7+zCnxblm57Mb1LrhZvxbvmrtQpVbZqBatUcefhZ05Bbvghzi1MnTWWvwpH+Pf9Tf4+QdolcWW8c9iN2+q3n1nTwWocrE21bErashx2M8+wOYuqVZbwmF37oDFv0RsNZzqMxC2b4Nf5mLnTIOdO6S4nxARFJMAxxhTCFyJm15a4A6ZH7XW92qtTwHQWvfXWucCZwPPaK1/LL5ea90RNwL0Zalbv6q1ngfMAzKB+6PeGRF11lr8SW9D81Zw0CAAlL4EunTDvvQEtpIy9/FgNxUHN2vwrr4L1bXs1VLq+DOheSv88U9jd+0q85xEYFevwH/0TrfzcfsueHc9jnfFLdhtBdiJr1bv3rOnQlYHVMuyh5VDoU6/AOrVx3/1Gaxf0bqECFqyEAoLq51gvNsBvaF2Heyc6W7n8GYtwt76QQhRPpVI5c9jxObn50flxjV9HrMqotFnu/BH/H/dgjr3Crwjf181YTesxb//OqhdF++2h+O60qRkv+2mDS64WbPSBTeVVLi1877Hf/we1OkX4J1wdiyaGzLr+9jPP8S+/ZKrUXPmRaghx+1empvx7sts+/BtvDseRbXrFP79N23Av2EE6sQ/4p16brXa6k/52AVgjZuh+gx0+zzt1yPiUzzF77U/cTz2A4P32H9Q9SKzWWLRU6Ng4Two2Ioadibe6RdE5L6RIN/PUkdN73eQg6NKH5eCAiLh+JPehgaNUIcevcdx1bgZ3mX/gLUr8Z9/NHa/uVfAbt6I/8gdsGYF3lV3hFS+X/XsCwcNwv73dezaVTFoZWjsilz8f92CnTAW9jsQ7+7/wxt6/B51RxoM/zPUr4//+rgqLdW2P8wAa6uUf1OaGnwM6rIbofN+2G8n4z96F/51F+I/+zD2+2+w27dV+xkl2YU/uvybCAU3EExTbdkc7Bx+RMTuK4SoQauoRGqw+Uth7kzUyeegatfe63W134Gos/+EnfAs9qM3USfqOLTSsZs3ueBm1XIX3HTrFfK13h8vxb/zr/gTxpH2t1uj2MrK2aIi7MfvYieOh4wM1MXXoA45CqX2+oUIr0Ej1KnnY199CmZ9C30PC+9Zs6a6qce2Havd7t0l/Psf7vJjfpqDnTMN+8MM7AxXMI8DeqMOGuRqyzRqXOVn2V074befUUdGtg6L6tUP63nQtiOqTfuI3luIVCcBjkgo9uN33Q/Z0iXsS1BHnQSLF2LfexXboQuqnJ2oQ3reogXYye9DnbrQojWqRRto0dp9XsGGfv6mYORmZT7elbejDugd1nNVs+aok4Zj334JO3cmqlfF2xVEi83Nxn/xcchZBAcNwjv3CrchZAXUkGOxX36E/8YLeD37oTL2DkTLfFbBVljwA+rok8sMnqpDZdSGPgNRfQa6Yoq/LXC7T8+ehp33HVYp6HIA6qCBqEFDUY2ahPeAxQuhcFfEt9tQ9Ruihl/mqhsLISJKAhyRMOyGtdhpX6CGHItqWH75fqUUXPg3bF42/rMP493+CKp5q/CetXa1Cy5mfAUNGoFSsHkje0y67NMUWrTaHfSolm2geWto0JD1T4+CFbkuuOnep0r9Vcecgp36Gf5rY/G69Qo5UIgEW7gL++Gbbn+fevXxLv8H9D0spMBDeWl4wy/F//dt2I/fQZ0UWuFCO+87KCpEBYnj0aLS0lwuzn49sPoSyM3+Pdh54wXsf99AnXcFXhi7dduF892/kXKSx6vDi/CokBDCkQBHJAw7+QOXi3DMaZWeq2rXwfvrrfgjr8N/6kG8m/8ZUoBgd2zH/u8t7KR33H1OGo4adgaqdh3stgJYvRy7cjmsyodVy7GrlrsfzJs27BH8+Om18P52K+rAg6raXVR6LbxzL8d/+HbsR2+hqpl0Gyq7eRP+kw/Aop9QA45ADb+0woCyLGr/ntD3UOxHb7q9oZo2r/y5s6fCPk2g8/5VbXrYlFLQrpNLiD7lHGzeUvyXHsc++2/82dNQ514RUt/tL/PcfSKYfyOEiC4JcERCsNsKsF/+D3XwISGPxqgWrfEuuQ7/ifuw/3kSLr623BEI6/vY6V+61UEb1qEGDEGdMQLV7PcfzKpuPWjfBdW+y97Xby+AVStgVT529QoaDxjMpmbhjRqV2YduvVADjsD+703soKHVWjodCrt8Gf4T98H6tag/X49XjcRW76yL8ed+h33rJdSlN1R4rt25A+Z9jzr0qLhulqiy2uPd9BB20tvYia9hF87Hu/BKVO/ya4TaXTtd/ZsjhsWwpUKI6pJVVCIh2CmTYNtW1HFnhHWd6tUfdfJw7NTPsV98VPa9Fy3Af/BG7POPQuNmeDc9hHfpDXsEN5U+p049VPvOqH6D8Y4/i4wQVkuFfO+zL4ZaGfivRXcTSfvTHPwH/wHbt+HdMLJawQ2AymyJOu507IyvsL/+VPHJP812heyiPD0VCpWWhnfC2Xi3PQyNGrsqzS+OdiN4Zdj160+wa2dIK+SEEIlDRnDEbv5/DeQvhTbt3YqONu2heUuUF939n2xhIfbT92H/nqhOXcO+Xp00HJu9CPv6OGz7zru3R9gjz6ZxM9Qlf3dTMnEcQSiLatwUdcq52NfHwayp0PfQiD/D//J/2PFPQ+t2bsVXsxYRua8adib2m8n4E8a62kTl/Fuxs6ZBvfqwX2STdKtDteuEd9vD2PcnYD96C7tgLt5FV++VML7zx9lB/s2BcWqpEKIqJMARAPjTPse++x+XcDvjq9/zTWplQKssF/C0bofKCgKfzMgFPnbmFFcB+IK/Vul65XluqmrkdfhPj8K78UHs1M/KzLNJVOrIE12g8Po4vAMPQtWpG5H7Wr/IJdZ+OhF69MW77MZq76NUkqpdB3XmCOy4h7HfTEYFO2Pv0YbCQuwPM9xS7fTE+paj0muhTr8A23sA/vOP4T9yB+qok9z0ZVCmYNf82ZDVEVW/YZxbK4QIR2J9txFxYZcvw77yJHTtjnf9SNi1A5bnupo0+Uux+Uuxv/4I07/8PfDJyIBWbdnUsy/2eF1mzZqQnm0tdtLbkNUBqrHcW9VvgPfXW/AfvBH/9itcMbkBR6DOuDCsqah4UWlpeOf/BX/UP7AfTECddXG172m3F+A/+7CrK3T0yaiz/xSV3djVgCHYLz7CvvMKtu9he1eYXjgfCrZEpLhftKjO++Pd8Rj2nZexk9/Hzp+F96droUMXdv48r8zATQiR2CTASXF2x3b8px+C2nXwLr3R/QBMqwed9kN12m/Pc7cV7A54yF+Gzc9h24dvws/z3LRHVX7D/XEW5OW44nLVrI2i2nbCu+R6/K8/wTtR756qqilUl26ow/6A/XQi9pCj3WhZFdl1q/GfuB/yc/ba8iLSlFJu2fjI61xwpi/Zsy2zp0JGbehe9RVnsaBq10YNvxTbZyD+C6PxH7rZBWU7d0Ru/ykhRMyEHOBorY8q56UdQK4xJvF2QBSVsuOfgeXL8K69G9WkWYXnqrr1oEu3PQKHBr/OY+Mjd+E/dLO7RwjLhUvyJ73j8mPCqElSYRsPPoS0BB4pqIw68yLsnOn445/Gu2FklYI+u+RX/DH3w84deFfdiepxcBRauifVoQtq8DFuh+/Dj9tduM76Pnb2dOjRN6Z1fqpDdeuFd/cTWPMc9utP3MH9JP9GiJomnBGc54DiNaxrgeKfhquAVlrrucBwY8yvEWyfiCL/m8nYbyejTvojqoq/Xdc55Eg2Xwv+mJH4o27C+/s9qNbtQrrW5iyCn+eizro44hsk1lSqYSPUGRdgX3kS/9pzoV1nVLvObg+k9l1cPlQF00z2+2/wn38UGjbG+/t91RoFCrvtp52P/e5rfPMcadfc5Q4uWQgb1yX09FRZVN16qBFXYfseSv2d2yhoEF6dICFE/IUb4OwD3GmM2aa1rgvcDWwCHgMeBp4Ejol0I0Xk2bwc7Pin3Mqlk0OrRFsetX9PvBsewB99txvJueqOkKaH7KR3oG491JDjqvX8ZKMGHwsZdeDXn7DLFmO//Ah27XT5T7Uy3L5Fu4Oezi5/qVaGK7r3zivQpRveX2+t1t5LVWp3o8ZuRdsbz+/efsLOmgpp6aie/WLalkhRPfpSLzOTghq807IQqSqcAOcaoLUxphAgCHJuB/KNMSO11tcDudFopIgsu32by7upU8/Vg4nAaijVvjPezf/Ef/RO/Edux7v8pgr3V7KrV2C/+wZ17KkRXdWTDJTnoQYNhUFDAbcZJivysEt/g6WLXdAzcwp89T8X9HgeNMmEtatcAcOLrkbVyohP2486ETtlEv7rz+F17+Pybw7otXfisRBCRFk4Ac5WoD8wtcSxvkBxdSw/Uo0S0WOtdVV/V+a76aR9wtx0sAKqeSu8mx/CH30v/piRqBFX4x1aduqW/XQieB7q6FMi9vxkpdLSIKu9m2465EjAvY+sWQnLFmOXLsbmZqOOPBF17GkR38gyrLam18LTf8Z//B7sy/8Hq1eghp0Zt/YIIVJXOAHOncDHWuuJwDKgLXAycFXw+tHAm+VdrLUeBowG0oBxxphRpV4fgpvq6oXL5XmzxGtFwLzgy6XGmFOC452ACbh8oO+BC4wxO8PoU9zYDWth166wN4ms9nOnfIyd/iXqlHPD3gE7FKpRE7wbRuI/9SD2hcfwN2/AK1Wd2G7ZhP36E9TAIypNbBZlU0pB81bQvBXq4MgXBqwO1bMv9OyHnfo5KIXqMzDeTRJCpKCQS7oaY14GBgI/43JxFgKHBMcxxnxgjLm0rGu11mnAGOB4oDtwjta69La8S4GLgPFl3GKbMaZP8FHyV/6HgEeNMfsC64FLyrg2IflPPoh/62UU/fNml+y7Y3vUn2mXLcG+Nha690GdeHbUnqPq1nOrd/oNxr75Iv4bz2P93wf47BcfuqW3x54etTaI+PL0JZCWDvseEPNcICGEgPCWidcGfjXG3FfiWC2tdW1jzI5KLh8ALDLGLA6umwCcCuzewMYYkx28FtJUl9ZaAUcBxVswv4RLen4qlOvjyRYWwrLF0Gk/2LQR++Jo7GtjUQMORx32B+i8f8SnGey2Apd306Ah3iXXRX37BVWrFlx6AzTcB/vxu7BpA4y4Gvwi7Gf/hZ79YrrCR8SWapWFd+Vt0FhG6IQQ8RHOFNUnwD+AaSWO9QVGAUMruTYLN61VLBc3GhSqOlrr74BCYJQx5l3ctNSG4qTn4J5ZZV2stb4MuAzAGENmZmYYjw5denp6SPcuXLaEtYWFNDrlj9Q5Yhi7Fsxl2+QP2PHtZ/hTPiYtqwN1jz6JOkOHkRaBKRxrLRsfGc2ONStocu8TZHTet9r3LFZZn+1Vt7K1dRZbx4+l1o7tZPTux5bNG2miLyYjSu9DLIT6XiebsPo9NDlWx8l7nTpSsc+QvP0OJ8DpCUwvdWwGEPlEjr11MMbkaa07A59precBG0O92BgzFhgbfGnXRGnJZ2ZmJqHc258/B4AtjZqxde1aaJEF51yOOv0C+O4bir75lC0vj2HLf56Cnv3wBv8BevSr8j4+/hcfYr+ejDr9Aja1bAcR7H9IfT7yJFR6BjtfeZKds6dBx65sbNkWVYOX3ob6XiebVOx3KvYZUrPfqdhnqPn9btOmTZnHw/mJuRFoCawocawlbnVVZfKAktXf2gbHQmKMyQv+XKy1/gI4CHgLaKy1Tg9GccK6Z1zlZrulvaUK4qk69VCDj4HBx2BX5GK/mYyd+hn+DzOg4T6o/odDh31RbTu4jS9DWApsc35zu1T36BvX1Sze4cdiGzbCf3kM3innxnWljxBCiOQXToDzFjBea301sBjoAjwCmBCunQl0DVY95QHD+T13pkJa6yZAgTFmh9Y6EzgM+KcxxmqtPwfOwq2kGgG8F0Z/4sbm5UDLLJenUg7Vqq3bpfm082H+LPxvPsF+NQkKP/i99knLLFRWB8jq4IKerI7QrAXKc7njtmAr/jMPuaq2f/r77uPxovoMwus9UIIbIYQQURdOgHMbrlrxDKA2sB14Abi1sguNMYVa6yuBSbhl4s8bY37UWt8LfGeMmai17g+8AzQBTtZa32OMORA4AHgmSD72cDk4xcnJNwETtNb3A7Nx1ZYTX242qvP+IZ2q0tKgd3/Sevd3Bd9W5WNzcyAv21UjzlkE3339+y7ftesGNVM6YFevgHWr8W54ANUwMUrNS3AjhBAiFpS1tvKzSghWL2UCa4wx4V2cGGx+fn5UbhzKPKbdVoB/9XDUaefjnagj8ly7vQDylrqRobyc4M9s2LIZdfaf8I49LSLPKUtNn7utKul36kjFPkNq9jsV+ww1v99BDs5evz1XOIITJPWWp6HW7gd08fJvEYI8t+m6atsxYrdUdfbe5dtaC9u2ouo1iNhzhBBCiJqisimqRYDFRUbFozXFUVLJ0ZvoFlVJIjYIcMjqENXnKKVAghshhBApqsIAxxizOytVa30x8AdcMb0coANu+4bJUWxf8snLhjp1oVmLeLdECCGESFrhJBnfB3Q1xmwLvv5Va305bsuGFyPdsGRl83LcqidJthVCCCGiJpx1wx7QsdSxDsj0VMistZCbg8rqGO+mCCGEEEktnBGcR3FVhF/AbbvQDrgYtwO4CMX6tVCwBdpGN/9GCCGESHXh7Cb+L9xu3y2Bk4M/LzLGPBSdpiWh4hVUUU4wFkIIIVJdyAGO1joDNyW1E1gL1ALO01q/HKW2JR2bl+0+kSkqIYQQIqrCmaJ6Cbex5vvsuR+VCFVeDjRuhqovy7eFEEKIaAonwBkGdDLGbIhWY5Kdzc2GCBb4E0IIIUTZwllFtRS3B5WoAltYCMtzJf9GCCGEiIFwRnBeBt7TWo8GVpZ8wRjzWURblYxW5kNRoaygEkIIIWIgnADnyuDPB0odt0BFe1YJfk8wlho4QgghRPSFHOAYYzpFsyFJLzcb0tKgVdt4t0QIIYRIeuHk4IhqsHk50DILVatWvJsihBBCJD0JcGIlL0cSjIUQQogYCScHp1q01sOA0bi9q8YZY0aVen0IbtuHXsBwY8ybwfE+wFNAI6AIGGmMeT147UXgCGBjcJuLjDFzot+b8NiCrbB2FRx+bLybIoQQQqSEmIzgaK3TgDHA8UB34BytdfdSpy3FbQUxvtTxAuBCY8yBuFo8j2mtG5d4/UZjTJ/gI+GCGwDygy0a2koakxBCCBELsRrBGQAsMsYsBtBaTwBOBX4qPsEYkx285pe80BizsMTn+VrrVUBzoMYUHLS5LsCRJeJCCCFEbMQqwMnC7UBeLBcYGO5NtNYDgAzgtxKHR2qt7wQmAzcbY3ZUp6FRkZcDdetB0+bxbokQQgiREmKWg1NdWuvWwCvACGNM8SjPLbh9sTKAscBNwL1lXHsZcBmAMYbMzMyotDE9Pb3Me69blQcdutC0efIFOOX1OdlJv1NHKvYZUrPfqdhnSN5+xyrAyQPalfi6bXAsJFrrRsB/gduMMdOKjxtjlgef7tBavwDcUNb1xpixuAAIwK5ZsyaMpocuMzOT0ve21uJnL0L1P3yv15JBWX1OBdLv1JGKfYbU7Hcq9hlqfr/btGlT5vFYLROfCXSCUSIQAAAMEUlEQVTVWnfSWmcAw4GJoVwYnP8O8HLxyqoSr7UO/lTAacD8iLY6EtavgYKtIBWMhRBCiJiJSYBjjCnEbfUwCVjgDpkftdb3aq1PAdBa99da5wJnA89orX8MLtfAEOAirfWc4KNP8NqrWut5wDwgE7g/Fv0JS16wgkpq4AghhBAxo6y18W5DrNn8/Pyo3LisYT7/o7ewb7+E99h4VP0GUXluPNX0oc2qkn6njlTsM6Rmv1Oxz1Dz+x1MUanSx6WScbTlZUPTzKQMboQQQohEJQFOlNncbMm/EUIIIWJMApwosoWFsCJP8m+EEEKIGJMAJ5pW5kFRIUiAI4QQQsSUBDhRZHOzAVBtO8a1HUIIIUSqkQAnmvKyIS0NWmXFuyVCCCFESpEAJ4psbg60aotKrxXvpgghhBApRQKcaMrLkQRjIYQQIg4kwIkSW7AV1q0Gyb8RQgghYk4CnGjJly0ahBBCiHiRACdKildQSZE/IYQQIvYkwImWvByoWx+aZsa7JUIIIUTKkQAnSmxuDmR1QKm99v8SQgghRJRJgBMF1lq3gqqt5N8IIYQQ8SABTjSsWwPbtsoWDUIIIUScSIATDXnZAChJMBZCCCHiQgKcKLB5bom4jOAIIYQQ8ZEeqwdprYcBo4E0YJwxZlSp14cAjwG9gOHGmDdLvDYCuD348n5jzEvB8b7Ai0Bd4EPgGmOMjXJXKpebA02bo+rVj3dLhBBCiJQUkxEcrXUaMAY4HugOnKO17l7qtKXARcD4Utc2Be4CBgIDgLu01k2Cl58CLgW6Bh/DotSFsNi8bBm9EUIIIeIoVlNUA4BFxpjFxpidwATg1JInGGOyjTFzAb/UtccBnxhj1hlj1gOfAMO01q2BRsaYacGozcvAaVHvSSVs4S5YkSsrqIQQQog4itUUVRawrMTXubgRmapemxV85JZxfC9a68uAywCMMWRmRqf4Xnp6Oo13FLCuqIiG3XpSN0rPSSTp6elR+/tMZNLv1JGKfYbU7Hcq9hmSt98xy8GJJ2PMWGBs8KVds2ZNVJ6TmZnJ+vlzANiyTzO2Ruk5iSQzM5No/X0mMul36kjFPkNq9jsV+ww1v99t2rQp83ispqjygHYlvm4bHKvOtXnB51W5Z/TkZkNaOrQsczBJCCGEEDEQqxGcmUBXrXUnXBAyHDg3xGsnAQ+USCw+FrjFGLNOa71Jaz0ImA5cCDwR4XaHzeblQKssVHpKDI4JIYQQCSkmIzjGmELgSlywssAdMj9qre/VWp8CoLXur7XOBc4GntFa/xhcuw64DxckzQTuDY4B/BUYBywCfgM+ikV/KpSXLQX+hBBCiDhT1sa/bEyM2fz8/KjcuGnd2qw+/zjUGSPwjj8zKs9INDV97raqpN+pIxX7DKnZ71TsM9T8fgc5OHvtbC2VjCOoMGcxgCwRF0IIIeJMApwIKsxZ5D6RIn9CCCFEXEmAE0GFOYuhbn1oknz1BIQQQoiaRAKcCCrM+Q3adkCpvaYChRBCCBFDEuBEiLWWwqW/yQoqIYQQIgFIgBMp61ZjC7ZK/o0QQgiRACTAiZTcHEBWUAkhhBCJQAKcCLF52e6TNhLgCCGEEPEmAU6k5OXgNW+Fqlc/3i0RQgghUp4EOBFic7NJ79Al3s0QQgghBBLgRIQt3AUr8yTAEUIIIRKEBDiRsG41pNeilgQ4QgghREJIj3cDkoFq0Qbv8QnUbtqULRs2xLs5QgghRMqTEZwIUZ6HSpd4UQghhEgEEuAIIYQQIulIgCOEEEKIpCMBjhBCCCGSjgQ4QgghhEg6ylob7zbEWsp1WAghhEhyqvSBVBzBUdH60Fp/H837J+JHKvZZ+h3/dkifpd/SZ+l3qY+9pGKAI4QQQogkJwGOEEIIIZKOBDiRNTbeDYiDVOwzSL9TSSr2GVKz36nYZ0jSfqdikrEQQgghkpyM4AghhBAi6UiAI4QQQoikI7tDRoDWehgwGkgDxhljRsW5STGhtc4GNgNFQKExpl98WxQdWuvngZOAVcaYHsGxpsDrQEcgG9DGmPXxamOkldPnu4FLgdXBabcaYz6MTwujQ2vdDngZaImrmTXWGDM6md/vCvp8N0n8fmut6wBfAbVxPwvfNMbcpbXuBEwAmgHfAxcYY3bGr6WRVUG/XwSOADYGp15kjJkTn1ZGhozgVJPWOg0YAxwPdAfO0Vp3j2+rYupIY0yfZA1uAi8Cw0oduxmYbIzpCkwOvk4mL7J3nwEeDd7vPsn0w66EQuB6Y0x3YBDwt+D/czK/3+X1GZL7/d4BHGWM6Q30AYZprQcBD+H6vS+wHrgkjm2MhvL6DXBjife7Rgc3IAFOJAwAFhljFgdR/gTg1Di3SUSQMeYrYF2pw6cCLwWfvwScFtNGRVk5fU56xpjlxphZweebgQVAFkn8flfQ56RmjLHGmC3Bl7WCDwscBbwZHE+q9xoq7HfSkSmq6ssClpX4OhcYGKe2xJoFPtZaW+AZY0xSLjUsR0tjzPLg8xW44f1UcKXW+kLgO9xv/UkxTVMWrXVH4CBgOinyfpfq82Ek+fsdjMB/D+yLG4n/DdhgjCkMTsklCYO90v02xkzXWv8FGKm1vpNglNIYsyOe7awuGcER1THYGHMwbnrub1rrIfFuUDwYYyxJ+htQKU8BXXDD2suBh+PbnOjRWjcA3gKuNcZsKvlasr7fZfQ56d9vY0yRMaYP0BY3Gt8tzk2KidL91lr3AG7B9b8/0BS4KY5NjAgJcKovD2hX4uu2wbGkZ4zJC/5cBbyD+waRKlZqrVsDBH+uinN7os4YszL4xugDz5Kk77fWuhbuB/2rxpi3g8NJ/X6X1edUeb8BjDEbgM+BQ4DGWuvi2Y2k/n5eot/DgqlKG4zavEASvN8S4FTfTKCr1rqT1joDGA5MjHObok5rXV9r3bD4c+BYYH58WxVTE4ERwecjgPfi2JaYKP4BHzidJHy/tdYKeA5YYIx5pMRLSft+l9fnZH+/tdbNtdaNg8/rAsfg8o8+B84KTkuq9xrK7ffPJQJ4hcs7qvHvt1QyjgCt9QnAY7hl4s8bY0bGuUlRp7XujBu1AZfLNT5Z+621fg0YCmQCK4G7gHcBA7QHcnDLhpMmKbecPg/FTVdY3FLpy0vkpSQFrfVgYAowD/CDw7ficlKS8v2uoM/nkMTvt9a6Fy6JOA33y74xxtwbfG+bgJummQ2cX9NzUUqqoN+fAc1xO3PPAa4okYxcI0mAI4QQQoikI1NUQgghhEg6EuAIIYQQIulIgCOEEEKIpCMBjhBCCCGSjgQ4QgghhEg6slWDECKlBdsTLAFqlSjRL4So4WQERwghhBBJRwIcIYQQQiQdKfQnhEg4Wus2wBPAEGAL8Kgx5nGt9d1AD6AIOAH4FbjYGPNDcN0BuE0i++D2ELrFGDMxeK0ucD+uDH9jXOXeY3A7gy8BLgLuA+oFz0vKytxCpAoJcIQQCUVr7eH2eHsPGIXb8PBT4C+4zRBvw20j8B5wDfA3YL/g8gXA88C/gcHBOf2MMb9orccABwLnASuAgcD3QGtcgDMOuDq41wygjzFmQZS7K4SIEglwhBAJRWs9EHjDGNO+xLFbcIFHDm7n40HBcQ83UqODU98A2gQ7YBfvqfULcC+wFRhUPNpT4t4dcQFOO2NMbnBsBvCIMWZCtPophIguWUUlhEg0HYA2WusNJY6l4TaEzAGWFR80xvha61ygTXBoWXFwE8gBsnCbhtYBfqvguStKfF4ANKhyD4QQcScBjhAi0SwDlhhjupZ+IcjBaVfiaw83hZUfHGqntfZKBDntgYXAGmA70AXYYwRHCJGcJMARQiSaGcBmrfVNwOPATuAAoG7wel+t9RnARFzOzA5gGqBwIy//0Fo/DBwGnAz0D0Z6ngce0VpfAKwEBgCzYtctIUQsyTJxIURCMcYUASfhVkItwY2+jAP2CU55D/gjsB64ADjDGLPLGLMTF9AcH1zzJHChMebn4LobcCunZgLrgIeQ74FCJC1JMhZC1BjBFNW+xpjz490WIURik99ehBBCCJF0JMARQgghRNKRKSohhBBCJB0ZwRFCCCFE0pEARwghhBBJRwIcIYQQQiQdCXCEEEIIkXQkwBFCCCFE0vl/KKqwcdxI8r4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 576x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "-4tP0-a-zSxv",
        "outputId": "92a04133-60c1-4bdf-f8c6-b704120eb86d"
      },
      "source": [
        "true = list_agg(test_r)\r\n",
        "true.head(5)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>[177, 179, 183, 84, 4, 198, 199, 207]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>8</td>\n",
              "      <td>[59, 438, 93, 123, 463, 525, 30, 288, 534, 322...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>41</td>\n",
              "      <td>[175, 182, 563, 184, 1176, 116, 128, 1037, 104...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>78</td>\n",
              "      <td>[1258, 305, 500, 2230]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91</td>\n",
              "      <td>[558, 1176, 195, 587, 592, 74, 1904, 186, 647,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id                                           movie_id\n",
              "0        2              [177, 179, 183, 84, 4, 198, 199, 207]\n",
              "1        8  [59, 438, 93, 123, 463, 525, 30, 288, 534, 322...\n",
              "2       41  [175, 182, 563, 184, 1176, 116, 128, 1037, 104...\n",
              "3       78                             [1258, 305, 500, 2230]\n",
              "4       91  [558, 1176, 195, 587, 592, 74, 1904, 186, 647,..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fgw83iYFysnj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "bf177f22-49d7-46df-f9f6-6d673c2c4c96"
      },
      "source": [
        "pred = test_predictor(test_q, model, n_item, batch_size, N = 100)\n",
        "pred.head(5)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>movie_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>[92, 48, 97, 22, 38, 132, 217, 23, 171, 68, 26...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>8</td>\n",
              "      <td>[184, 244, 134, 68, 92, 171, 5, 97, 315, 44, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>41</td>\n",
              "      <td>[132, 62, 160, 189, 155, 758, 740, 775, 856, 5...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>78</td>\n",
              "      <td>[104, 38, 48, 128, 23, 97, 68, 124, 132, 92, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>91</td>\n",
              "      <td>[64, 44, 9, 124, 733, 116, 104, 189, 13, 741, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id                                           movie_id\n",
              "0        2  [92, 48, 97, 22, 38, 132, 217, 23, 171, 68, 26...\n",
              "1        8  [184, 244, 134, 68, 92, 171, 5, 97, 315, 44, 1...\n",
              "2       41  [132, 62, 160, 189, 155, 758, 740, 775, 856, 5...\n",
              "3       78  [104, 38, 48, 128, 23, 97, 68, 124, 132, 92, 1...\n",
              "4       91  [64, 44, 9, 124, 733, 116, 104, 189, 13, 741, ..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKQgNhpY0Bjl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398,
          "referenced_widgets": [
            "aabf5662754c46f894c427b8e1dc2828",
            "c25d26f060b94e52a72da90c1e553e8f",
            "576e96debff9425891e6fce1d02325d4",
            "e25bf164f48b4d86bbb97998a2a6d1c2",
            "5b485c98a972408cb18116e3c6a3dbfe",
            "2aaeda6eb9124c6c8a3a2168e8f86f26",
            "0fcaafd4095c45de96c78bdc4bf2e807",
            "ab50d412df094667b128dc71d9484235",
            "dfa68c194495413d85e93c67fa0f3c2c",
            "9d8a661b62994e3e93859ed9d4b86a4f",
            "057d9197cb294810bcf142bf73dcb681",
            "5357ddad2e034666a612cc0b32598124",
            "fddf99322a414f808a8f2c3f0636a10c",
            "db4e27e2921f4fb1909b750f8324e9d7",
            "6f5f0227aedf4aa7aaaee8f9f66aefce",
            "17898d31a8ab488e966b81c6325b47a6"
          ]
        },
        "outputId": "1cc5d929-0bd7-4cfd-ea40-8d9a2c2ed807"
      },
      "source": [
        "scores = evaluate(true, pred)\n",
        "\n",
        "scores(K = 10)\n",
        "scores.print_all()\n",
        "\n",
        "scores(K = 100)\n",
        "scores.print_all()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aabf5662754c46f894c427b8e1dc2828",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=604.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "      Recall@10 : 0.10649 \n",
            "   Precision@10 : 0.09123 \n",
            "         MAP@10 : 0.05127 \n",
            "          HR@10 : 0.49007 \n",
            "         MRR@10 : 0.22880 \n",
            "        nDCG@10 : 0.10883 \n",
            "          CO@10 : 0.05977 \n",
            "          ED@10 : 21.91308\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dfa68c194495413d85e93c67fa0f3c2c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=604.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "      Recall@100 : 0.34341 \n",
            "   Precision@100 : 0.05407 \n",
            "         MAP@100 : 0.05590 \n",
            "          HR@100 : 0.91225 \n",
            "         MRR@100 : 0.24676 \n",
            "        nDCG@100 : 0.19868 \n",
            "          CO@100 : 0.02109 \n",
            "          ED@100 : 125.42148\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}