Similarity_Selection_ML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# imports\n",
    "from __future__ import division\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "from polara.recommender.models import RecommenderModel\n",
    "from polara.recommender.data import RecommenderData\n",
    "from polara.tools.movielens import get_movielens_data\n",
    "from polara.lib.similarity import combine_similarity_data\n",
    "\n",
    "from models_sparse import FeatureSimilarityMixin\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimilarityDataModel(FeatureSimilarityMixin, RecommenderData): pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ContentBased(RecommenderModel):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super(ContentBased, self).__init__(*args, **kwargs)\n",
    "        self.method = 'CB'\n",
    "\n",
    "    def build(self, *args, **kwargs):\n",
    "        # use copy to prevent contaminating original data\n",
    "        self.item_similarity_matrix = self.data.item_similarity.copy()\n",
    "        self.item_similarity_matrix.setdiag(0) # exclude self-links\n",
    "        self.item_similarity_matrix.eliminate_zeros()\n",
    "        \n",
    "    def slice_recommendations(self, test_data, shape, start, stop, test_users=None):\n",
    "        test_matrix, slice_data = self.get_test_matrix(test_data, shape, (start, stop))\n",
    "        # test matrix is in CSR format\n",
    "        scores = self.item_similarity_matrix.dot(test_matrix.T).T\n",
    "        return scores, slice_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ml_path = \"datasets/ml-1m.zip\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "data, genres = get_movielens_data(ml_path, get_genres=True, split_genres=False)\n",
    "genres = genres.assign(movienm=lambda x: x.movienm.str.decode('unicode_escape'))\n",
    "\n",
    "id_fix = pd.read_csv('datasets/ml1m_sub.csv')\n",
    "\n",
    "genres.movieid.replace(id_fix.set_index('ml1mid').movieid, inplace=True)\n",
    "\n",
    "genres = genres.drop_duplicates(subset='movieid').copy()\n",
    "\n",
    "data.movieid.replace(id_fix.set_index('ml1mid').movieid, inplace=True)\n",
    "data = data.drop_duplicates().copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "meta_info = pd.read_csv('datasets/meta_info_ml1m.csv', sep=';', na_filter=False).set_index('movieid')\n",
    "meta_info.loc[:, meta_info.columns] = meta_info.loc[:, meta_info.columns].applymap(lambda x: x.split(',') if x else [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ml_data = data.query('rating>=4').assign(rating=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userid     6038\n",
       "movieid    3532\n",
       "rating        1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_stats = ml_data.apply(pd.Series.nunique, axis=0)\n",
    "data_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.02697524961765369"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_data.shape[0] / np.prod(data_stats[['userid', 'movieid']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95.27658164955284"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_data.userid.value_counts().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_data.userid.value_counts().median()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_categorical(data, x, y, c=None, figsize=(3.7, 2.2)):\n",
    "    markers = [\n",
    "    'o', # circle\n",
    "    'v', # triangle down\n",
    "    's', # square\n",
    "    'D', # diamond\n",
    "    ]\n",
    "    \n",
    "    plt.figure(figsize=figsize)\n",
    "    \n",
    "    x_map = defaultdict(lambda: len(x_map))\n",
    "    x_ = data[x].map(x_map.__getitem__)\n",
    "\n",
    "    if c is not None:\n",
    "        c_map = defaultdict(lambda: len(c_map))\n",
    "        c_ = data[c].map(c_map.__getitem__)\n",
    "        for label, value in c_map.iteritems():\n",
    "            sel = c_== value\n",
    "            plt.scatter(x_[sel], data.loc[sel, y], marker=markers[value], label=label)\n",
    "    else:\n",
    "        plt.scatter(x_, data[y])\n",
    "        \n",
    "    plt.xticks(*reversed(zip(*x_map.iteritems())))\n",
    "    plt.tight_layout()\n",
    "    plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "jw = 'jaccard-weighted'\n",
    "jd = 'jaccard'\n",
    "cs = 'cosine'\n",
    "tc = 'tfidf-cosine'\n",
    "\n",
    "features = ['director', 'writer', 'cast', 'genres']\n",
    "sim_types = [jd, jw, cs, tc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_sel_res = {}\n",
    "folds = [1, 2, 3, 4, 5]\n",
    "\n",
    "for feature in tqdm_notebook(features, desc='feat'):\n",
    "    for stype in tqdm_notebook(sim_types, desc='sim', leave=False):\n",
    "        item_similarity = combine_similarity_data(meta_info[[feature]], similarity_type=stype)\n",
    "        similarities = {'userid': None, 'movieid': item_similarity}\n",
    "        sim_indices = {'userid': None, 'movieid': meta_info.index}\n",
    "        \n",
    "        data_model = SimilarityDataModel(similarities, sim_indices, ml_data, 'userid', 'movieid', 'rating', seed=0)\n",
    "        data_model.random_holdout = True\n",
    "        data_model.holdout_size = 1\n",
    "        data_model.verbose = False\n",
    "        \n",
    "        cb = ContentBased(data_model)\n",
    "        \n",
    "        for fold in folds:\n",
    "            data_model.test_fold = fold\n",
    "            data_model.prepare()\n",
    "            cb.build()\n",
    "\n",
    "            key = '{}--{}'.format(feature, stype)\n",
    "            nusers = data_model.test.evalset[data_model.fields.userid].nunique()\n",
    "            feat_sel_res.setdefault(key, []).append(cb.evaluate().true_positive / nusers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>feature</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.036108</td>\n",
       "      <td>cast</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.021698</td>\n",
       "      <td>cast</td>\n",
       "      <td>jaccard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.038759</td>\n",
       "      <td>cast</td>\n",
       "      <td>jaccard-weighted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.037268</td>\n",
       "      <td>cast</td>\n",
       "      <td>tfidf-cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.022197</td>\n",
       "      <td>director</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.022197</td>\n",
       "      <td>director</td>\n",
       "      <td>jaccard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.022031</td>\n",
       "      <td>director</td>\n",
       "      <td>jaccard-weighted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.024019</td>\n",
       "      <td>director</td>\n",
       "      <td>tfidf-cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.011098</td>\n",
       "      <td>genres</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.004970</td>\n",
       "      <td>genres</td>\n",
       "      <td>jaccard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.016564</td>\n",
       "      <td>genres</td>\n",
       "      <td>jaccard-weighted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.013584</td>\n",
       "      <td>genres</td>\n",
       "      <td>tfidf-cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.011595</td>\n",
       "      <td>writer</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.012423</td>\n",
       "      <td>writer</td>\n",
       "      <td>jaccard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.011264</td>\n",
       "      <td>writer</td>\n",
       "      <td>jaccard-weighted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.012920</td>\n",
       "      <td>writer</td>\n",
       "      <td>tfidf-cosine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       score   feature            metric\n",
       "0   0.036108      cast            cosine\n",
       "1   0.021698      cast           jaccard\n",
       "2   0.038759      cast  jaccard-weighted\n",
       "3   0.037268      cast      tfidf-cosine\n",
       "4   0.022197  director            cosine\n",
       "5   0.022197  director           jaccard\n",
       "6   0.022031  director  jaccard-weighted\n",
       "7   0.024019  director      tfidf-cosine\n",
       "8   0.011098    genres            cosine\n",
       "9   0.004970    genres           jaccard\n",
       "10  0.016564    genres  jaccard-weighted\n",
       "11  0.013584    genres      tfidf-cosine\n",
       "12  0.011595    writer            cosine\n",
       "13  0.012423    writer           jaccard\n",
       "14  0.011264    writer  jaccard-weighted\n",
       "15  0.012920    writer      tfidf-cosine"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_scores = pd.DataFrame(pd.Series(feat_sel_res)\n",
    "                              .reset_index()\n",
    "                              .rename(columns={'index':'feature', 0:'score'})\n",
    "                              .assign(feature = lambda x: x.feature.str.split('--'))\n",
    "                              .assign(score = lambda x: x.score.apply(np.mean))\n",
    "                             )\n",
    "feature_scores = pd.concat([feature_scores.drop('feature', axis=1),\n",
    "                            pd.DataFrame(feature_scores.feature.tolist(),\n",
    "                                         columns=['feature', 'metric'])], axis=1)\n",
    "feature_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAegAAAEpCAYAAABY5LEdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl8VNXdx/HPSYCwJYGwCyEByiaoGCiEALIEI2IFVKQqKhQFlwcVsVr1wYKIYGuxCAhaF6LIg1SKILYCssoiIlBEIKiIgEgAJRBZw5Lz/HFnYpZJMkkmmRn4vl+veU3mbPfcuZn5zTl3M9ZaREREJLCE+LsDIiIikpcCtIiISABSgBYREQlACtAiIiIBSAFaREQkAClAi4iIBCAFaBERkQCkAC0iIhKAFKBFREQCkAK0iIhIAFKAFhERCUAK0CIiIgFIAVpERCQAKUCLiIgEIAVoERGRAFTO3x0IdMaY74EIYI+fuyIiIoErFvjFWtvIVw0qQBcuolKlSlEtW7aM8ndHREQkMKWkpHD69GmftqkAXbg9LVu2jNq0aZO/+yEiIgGqbdu2bN68eY8v29Q+aBERkQCkAC0iIhKAFKBFREQCkAK0iIhIAFKAFhERCUAK0CIiIgFIAVpERCQA6TxoER/LzMwkLS2N48ePk5GRgbXW310SkSIwxhAWFkZ4eDhRUVGEhPhnLKsALeJDmZmZ/PDDD5w6dcrfXRGRYrLWcubMGc6cOcPJkyeJjo72S5BWgBbxobS0NE6dOkW5cuWoW7cuVapU8duvbxEpnszMTE6ePMnBgwc5deoUaWlp1KxZs8z7oW8OER86fvw4AHXr1iU8PFzBWSQIhYSEEB4eTt26dYFfP9dl3g+/LFXkIpWRkQFAlSpV/NwTESkp9+fY/bkuawrQIj7kPiBMI2eR4GeMAfDbgZ76FhEREfHAHaD9xWcB2hjTwBjzljHmgDEmwxizxxgzyRhTvYjtRLnq7XG1c8DVboNC6nUxxvzLGJPqqpdqjFlijOldsjUTEREpez45itsY0wRYB9QGFgA7gfbAI0AvY0wna+0RL9qp4WqnGbAceA9oAfwBuMEY09Fau9tDvVHAc8DPwEdAKlATuBroBvynhKsoIiJSpnx1mtU0nOD8sLV2ijvRGPMS8CjwPHC/F+2MxwnOf7fWjszWzsPAy67l9MpewRhzK05wXgrcbK09niu/fHFWSERExJ9KPMVtjGkMJAF7gFdyZY8GTgJ3GWMKPKzVlX+Xq/zoXNlTXe1f51qeu04I8BfgFHBH7uAMYK09V4TVEQkqszfsI2HCMmZv2OfvrhTJuXPnGD16NE2bNiUsLAxjDPPnz2fPnj0YYxg8eLDXbSUnJ2OMITk5OU/ekiVLSEhIoHr16hhj6Nevn+9WopQ1aNCA3/zmN/7uhviRL0bQPVzPS6y1mdkzrLXHjTFrcQJ4PLCsgHY6ApVc7eQItNbaTGPMEmAY0B1wT3MnAI2AucBRY8wNQGvgDLDBWvuZtythjNmUT1YLb9sQKUvvrt/LqPnbAHhq3ldcyLTcGR/j51459uzZQ6NGjRg0aJDHwDlx4kTGjh3LNddcw4ABAyhfvjwtWvj2o7Znzx769u1LtWrV+MMf/kBERITPlyFSmnwRoJu7nr/JJ/9bnADdjIIDtDft4GrH7beu50PAZuCK7BWMMZ8C/a21PxWwXJGgkz04u7lfB0qQLshHH31E1apV+eSTT6hQoUJW+rlz50hJSSEyMrLEy1i6dClnzpxh4sSJ3HHHHSVur6ytWrXK70cRi3/5IkC7P0np+eS706uVQju1Xc/3A98DPYHPgRhgInAd8D7OgWIFsta29ZTuGlnHFVZfpKx4Cs5uwRKkDxw4QI0aNXIEZ8CnI+kDBw4AcNlll/mkvbLWpEkTf3dB/KwszoN2/wQs6ZnentoJzZbX31q7zFp7wlq7HbgJ2A90NcZ0LOGyRQJCQcHZbdT8bby7fm8Z9SivMWPG0KhRIwDefvttjDFZj9jYWIwxfP/99+zduzdHOlDgPuhdu3Zx6623Ur16dapUqUJCQgL//ve/85RbuXIlxhhGj3YOZenevXvWclauXOnVOpw/f55p06aRkJBAZGQklStXpmnTpgwdOpTvvvsuR9ljx47xpz/9iWbNmlGxYkWioqLo1asXy5cvz9OutZYZM2bQsWNHatWqRcWKFYmOjqZXr17MnTs3R1lP+6DfeOMNjDG8++67LFu2jK5du1K1alUiIyO58cYb+frrrz2uz8mTJxk/fjxXXXUVVapUoWrVqiQkJDBnzhyv3g/xD1+MoN0j2/zmpCJylfNlO0ddz7uttV9mL2ytPW2MWQzcg3PKl9f7o0UCkTfB2c2fI+lu3bpx7NgxXn75Za666qocB2ZVq1aNY8eOMWnSJABGjBiRlV6Qb7/9lo4dO3LkyBGuv/562rRpw65du+jXrx/XX399jrKxsbGMHj2alStXsmrVKgYNGpT1A8D9XJCMjAyuv/56VqxYQUxMDAMHDiQ8PJw9e/bwr3/9i65du2aNbtPS0ujUqRM7d+6kffv23Hzzzfz000/885//pGfPnvzjH//g3nvvzWr7T3/6Ey+++CJNmjRhwIABREREkJqayhdffMHcuXPp379/of0DmD9/PgsWLKB379488MADbNu2jY8++ogvvviCHTt2EBUVlVX26NGjdO/enS+//JK2bdsyZMgQMjMzWbRoEbfddhspKSmMGTPGq+VKGbPWlugB3Iszqn0tn/zFrvzEQtrp6Sq3OJ/811z592RLu9mV9kU+dV505T9ZgvXbFBcXZ0W8sWPHDrtjxw6ftzvzsz025k8fFfkx87M9Pu+LN77//nsL2EGDBnnMj4mJsTExMV7Xu/baay1gJ02alCN9/vz51vUZtzNmzMiRN3r0aAvYFStWFKnvjz/+uAVsv379bEZGRo6806dP259++inr9ZAhQyxgH3jggRzlUlJSbNWqVW1YWJjdt29fVnpERISNjo62p06dyrPc7O1aa239+vVtkyZNcqS9/vrrFrDlypXLs15//OMfLWAnTpyYI33gwIEe00+dOmV79uxpjTF269at+bwb4u1nOi4uzgKbbAljavaHL6a4V7iek1ynPWUxxoQDnYDTwPpC2lnvKtfJVS97OyE4B5plXx7Ap8B5oKkxJufOLEdr1/OeQpYtErCKMnLOzd/T3b6wf/9+PvnkExo1asTw4cNz5PXt25euXbv6bFnnzp3j1VdfpUqVKkyfPj3PPvKKFStm3XYwIyOD//u//yMiIoLx48fnKNeiRQuGDx9ORkYGM2fOzJFXoUIFQkNDya0otzMcOHAg3bp1y5E2bNgwADZs2JCVdvjwYWbPnk18fDwjR47MUb5SpUq88MILWGuZPXu218uWslPiKW5r7XeuU6CSgP8BpmTLfhaogjO6PulONMa0cNXdma2dE8aYmTinUo0BHsvWznAgFmd0vTtbnZ+NMXOAgcCfgVHZlnEtzkFi6cCikq6niD/M3rCv2MHZbdT8bYSGGG5v39BHvSpb//3vfwHo3Lmzx8DWrVs3Vq1a5XV78+bNY+vWrTnS4uLi6NOnDzt27OD48eN06tQp61aD+dmxYwdnzpyhQ4cOHqfoe/TowQsvvJDVf3AC6/Tp02nVqhUDBgzgmmuuoWPHjkREROSpX5B27drlSYuOjgacKW23DRs2kJmZibXW4zS2+y5NKSkpRVq+lA1fXUnsQZxLdE42xiQCKUAHnHOWvwH+N1d5939D7nMInsY54nqkMaYNsAFoCfQFDuP8AMhtpGtZ/2uMucZVJwbnILELwFBr7bGSrJyIv0xZ9m3hhbwwdfmuoA3Q6enOYSd16tTxmF9YIM1t3rx5zJo1K0faPffcQ58+fTh2zPmqqF+/vtf9qlevnsd8d7q7TYDJkyfzm9/8huTkZMaPH8/48eMpV64cN9xwAy+99BKNGzf22FZunn4QlCvnfJ1fuHAhK+3IEecKy59//jmff/55vu2dOHHCq+VK2fLJUdzW2u+AdkAyTrB8DGgCTAY6Wi+uw+1q5wjOBUsmA79xtdMBmAG0dS0nd53DrjJ/B6KBh3EunvJvoIu19v2SrJuIPz2U2NQn7QzvEbxXpHKfE33o0CGP+QcPHixSe++++26efX1vvPEG8Gvg+/HHH73uV37LT01NzVEOnCA6cuRItm7dysGDB5k7dy59+vRhwYIFXH/99Zw759sLH7qX/fjjjxe4r/OTTz7x6XLFN3w1gsZa+wPOTS28KZvv2ffW2jScm2w8UoRlp+GMpEcWVlYkmNzeviEXMm2JprnH9Wtd5qNn91R09tFccV199dUArFmzhgsXLuSZ5vb21ClvXH755YSHh7NlyxYOHjxY4Oj88ssvp2LFivz3v//ll19+yTNNvWKFc7hMXJznyyjUqVOHW265hVtuuYWuXbvy6aefsmPHDq666iqfrU+HDh0wxrB69WqftSllR/eDFglwd8bHMK5f68ILejCuX2u/nGrlvvb1vn0lv0Z4gwYNuPbaa/n++++ZOnVqjrwFCxYUaf9zYcqXL88DDzzAyZMneeCBBzh79myO/IyMDH7++WcAwsLCuP3220lPT+fPf/5zjnLffvstU6dOpUKFCtx5550AnD59muXLl7vPDsly7ty5rP3GlStX9tm6gDPNftttt7F+/XomTJjg8QfTrl272Ls3uA8kvFj5bAQtIqXHHWSLMpL2V3AGqFq1Kh06dGD16tUMHDiQZs2aERoaSp8+fbjyyiuL3N4rr7xCx44dGTFiBEuWLOGqq65i165dfPDBB9x4440sXLjQZ31/9tln2bBhA/Pnz6dZs2b87ne/o2rVqvzwww8sXryYSZMmZQXdF198kTVr1vDyyy+zYcMGunbtmnUe9IkTJ5g+fToNGzqzFydPniQxMZFGjRrRoUMHGjZsyJkzZ1iyZAk7d+7kpptuomlT3+zSyG769Ons2rWLp59+muTkZDp37kytWrVITU1lx44dbNy4kffff5+YmMC++tylSAFaJEgUJUj7Mzi7zZw5k0cffZRFixYxe/ZsrLU0aNCgWAG6adOmrF+/nieffJKlS5eycuVKrrzySubPn89PP/3k0wBdsWJFlixZwvTp05k5cyYzZswAnAPHbrnlFhISErLK1qhRg88//5zx48fzwQcf8NJLL1G5cmU6duzI448/Ts+ePbPKRkRE8MILL7BixQrWrl3LBx98QEREBE2aNOG1117jD3/wag9hkUVGRrJ69Wpee+01Zs+ezdy5c8nIyKBOnTo0bdqUSZMm0aNHj8IbkjJnck+3SE7GmE1xcXFxmzbld7MrkV+5T1dp2bJlqS2jsPOiAyE4i1wsvP1Mt23bls2bN2+2+dzXoTi0D1okyBS0T1rBWeTioQAtEoQ8BWkFZ5GLi/ZBiwSpO+NjCA0xTF2+i+E9fhO0FyIREc8UoEWC2O3tGyowi1ykNMUtIiISgBSgRUREApACtIiISABSgBYREQlACtAiIiIBSAFaREQkAClAi4iIBCAFaBERkQCkAC0iIhKAFKBF5KI0ZswYjDGsXLnS310RKRYFaBERkQCka3GLBKvDKbB7FWQch7BwaNwVapfefaiDzfDhw7ntttto2FDXKpfgpAAtEmx2r4RVf4W9a/PmxXSCrk9A425l3KnAU7NmTWrWrOnvbogUm6a4RYLJ5ndg5k2egzM46TNvgs0zy7Zf+diwYQO///3vqV+/PmFhYdSrV4+kpCT++c9/5ij3z3/+k2uuuYbIyEgqVarEFVdcwYQJE8jIyMjT5tatW7n99tuJjY0lLCyMWrVqERcXx4gRIzh37lxWufz2QRtj6NatGz///DPDhg2jXr16hIWF0apVK2bMmJHvuixevJjevXtTs2ZNwsLCaNKkCY8//jjHjh0r2Zskkg+NoEWCxe6VsPARsJkFl7OZsPBhqBbt15H066+/zgMPPEBoaCh9+vShadOmHD58mI0bNzJt2jQGDBgAwNNPP82ECROoWbMmd9xxB1WrVuXjjz/m6aefZvHixXzyySeUL18ecIJzhw4dMMbQp08fGjVqxC+//MKuXbuYNm0a48aNyypbkGPHjtGpUycqVKhA//79OXPmDHPnzmXIkCGEhIQwaNCgHOXHjh3L6NGjiYqK4ne/+x21a9dm69at/O1vf+M///kPn332GREREb5/E+XSZq3Vo4AHsCkuLs6KeGPHjh12x44dpdP4W9dbOzrC+8dbvUunH17Yvn27LVeunK1evbrdtm1bnvwffvjBWmvtunXrLGCjo6NtampqVv65c+fs7373OwvY559/Pit95MiRFrDz58/P02ZaWpq9cOFC1uvRo0dbwK5YsSJHOcAC9p577rHnz5/P0efQ0FDbsmXLHOWXL19uAduxY0d79OjRHHkzZsywgB0xYoQX74oEI28/03FxcRbYZH0YfzTFLRIMDqfkP62dn71rnHp+MH36dM6fP88zzzxDq1at8uQ3aNAAgLfeeguAUaNGUbdu3az8cuXKMXHiREJCQnjjjTfy1K9UqVKetOrVqxMS4t1XWuXKlXnppZcIDQ3NSrv88svp1KkTKSkpHD9+PCt98uTJgDMjUK1atRztDB48mDZt2jBr1iyvlitSFJriFgkGu1cVv54fjuxev349ANdff32B5TZv3gxAjx498uQ1a9aMBg0a8P3333Ps2DGqVavG73//e15++WX69etH//796dmzJ506daJJkyZF6l/Tpk09TklHR0cDzhR4eHg4AJ999hnly5fn/fff5/33389T5+zZs/z0008cOXKEGjVqFKkfIgVRgBYJBhnHCy/jy3ol5D5wqn79+gWWS09PB6BevXoe8+vVq8e+fftIT0+nWrVqtG/fntWrV/P8888zd+5cZs50DoZr3rw5o0eP5vbbb/eqf7lHwm7lyjlfiRcuXMhKO3LkCOfPn+fZZ58tsM0TJ04oQItPaYpbJBiEhZdtvRJyB8Aff/yxwHKRkZEAHDx40GN+ampqjnIAHTt25KOPPuLo0aOsXbuWZ555hkOHDnHHHXewdOlSX3Q/Tx+rV69e6P7CmJgYny9bLm0K0CLBoHHXsq1XQvHx8QB8/PHHBZa7+uqrATxejnPXrl3s37+fRo0aeRzxhoWFkZCQwNixY7P2Ey9YsKCEPc8rPj6eo0ePsn37dp+3LVIQBWiRYFC7pXMRkqKI6ey3K4s98MADlCtXjueee44dO3bkyd+/fz8AQ4YMAWDcuHH89NNPWfkXLlzgj3/8I5mZmdxzzz1Z6atXr86aFs/u0KFDgHPwl689+uijAAwdOpQDBw7kyT958mTWPncRX9I+aJFg0fUJ5yIkhZ0HDWBCoOvjpd+nfFx++eVMmzaN+++/n6uvvpq+ffvStGlTjhw5wsaNGwkPD2fFihUkJCTwxBNP8Ne//pXWrVvTv39/qlSpwscff8y2bdvo3Lkzjz/+63pMnDiRJUuW0K1bNxo3bkzVqlXZvn07H3/8MdWrV2fYsGE+X5fExEReeOEFnnrqKZo2bUrv3r1p1KgRJ06cYO/evaxatYrOnTuzaNEiny9bLm0K0CLBonE3uPHlwi9WYkLgxsl+v9zn0KFDad26NX/7299YuXIl8+fPp2bNmlx55ZXce++9WeX+8pe/cPXVVzN16lTeeecdzp07R5MmTRg3bhyPPfYYFSpUyCr74IMPUr16dT7//HPWrl3L+fPnadCgAQ8++CCPPfZYqe0H/tOf/kSnTp2YPHkya9asYcGCBURGRlK/fn2GDRvGHXfcUSrLlUubsc7FOCQfxphNcXFxcZs2bfJ3VyQIpKQ45x23bFmKU8u7V8KqF53znHOL6eyMnBt3K73li1xCvP1Mt23bls2bN2+21rb11bI1ghYJNo27OQ/dzUrkoqYALRKsardUQBa5iOkobhERkQCkAC0iIhKAFKBFREQCkAK0iIhIAFKAFhERCUAK0CIiIgFIAVpERCQAKUCLiIgEIAVoERGRAKQALSIiEoAUoEVERAKQArSI+NyePXswxjB48GB/d8XvVq5ciTGGMWPG+LsrBerWrRvGmBK3M2bMGIwxrFy5suSdKgXBsj1AAVpERPzMVz8OLja6m5VIEHhw6YOs/nF1kep0qd+FaT2nlVKPCla/fn1SUlKIjIz0y/Kl6N555x1OnTrl725INgrQIkGgqMG5uHV8pXz58rRo0cJvy5eia9iwob+7ILloiltEfM7TPuhvvvmGJ598knbt2lGrVi3CwsKIiYlh2LBh7N+/P9+2lixZwo033kjt2rUJCwsjOjqavn37snTp0mKVPXv2LFOnTqV3797ExMQQFhZGVFQUPXv25OOPP/bYh9jYWGJjY/nll18YOXIksbGxlC9fPsd+zEOHDnHPPfdQp04dKlWqRJs2bXj77beL/N699tprGGN4/fXXc6S/9dZbGGOoXLkyGRkZOfLat29PxYoVOX36dI70zz//nP79+1O3bl0qVKhAdHQ09913HwcOHMiz3PymmTMyMhgzZgyNGzcmLCyMRo0aMWrUKDIyMjDG0K1bt3zXZe7cubRv357KlSsTFRXFbbfdxo8//piV7/4/WbVqFQDGmKxH7nb379/P8OHDs/pRo0YN+vTpwxdffOFx2b7aHv6kEbSIlIl58+bx6quv0r17dxISEqhQoQLbt2/njTfeYOHChWzcuJH69evnqDN69GjGjh1L1apV6devH9HR0Rw4cIB169bx7rvv0rNnzyKXTUtL45FHHiEhIYFrr72WWrVqkZqaysKFC+nduzevv/469957b57+nz17lh49epCWlkZSUhIRERE0atQIgCNHjpCQkMDu3bvp3LkznTt3JjU1lfvvv5+kpKQivU+JiYkALFu2jKFDh2alL1++HIDTp0/z2WefZQWw9PR0Nm/eTJcuXahUqVJW+RkzZjB06FDCwsLo06cP0dHRfPvtt1nv9/r16wsdNVtrueWWW/j3v/9N06ZNGT58OOfOnSM5OZnt27cXWHfatGl8+OGH9OnTh65du/L5558zZ84cvvzyS7Zs2UJYWBjVqlVj9OjRJCcns3fvXkaPHp1VPzY2NuvvzZs3k5SURFpaGtdddx0333wzP//8M/Pnz6dz58588MEH9O7dO6u8L7eHX1lr9SjgAWyKi4uzIt7YsWOH3bFjh8/bbZ3culgPf/n+++8tYAcNGpSVtn//fnvmzJk8ZRcvXmxDQkLs/fffnycdsI0aNbL79+/PU++HH34oVtkzZ87keO127Ngx26pVK1u9enV76tSpHHkxMTEWsImJifbEiRN56g4dOtQCdsSIETnSv/jiC1uuXDkL2NGjR+epl5+GDRvaWrVq2czMzKy0evXq2R49etiQkBA7atSorPT58+dbwI4dOzYr7euvv7bly5e3TZo0yfN+LFu2zIaEhNh+/frlSO/atat1QsKv3nnnHQvYLl262IyMjKz0o0eP2ubNm1vAdu3aNUed0aNHW8CGh4fbrVu35si7/fbbLWDnzJlT6LLdzp07Z5s0aWLDwsLsypUrc+T9+OOP9rLLLrN169bN8b/ly+3h7Wc6Li7OApusD+OPprhFpEzUr1+fsLCwPOlJSUm0atWKxYsX50ifMmUKABMnTswzsgZo0KBBscqGhYXleO0WGRnJkCFDOHr0aL7TphMnTqRKlSo50s6dO8esWbMIDw/Pc+pOu3btGDhwoMe2CtKjRw9++uknvvrqKwB27NhBamoq/fv3Jy4ujmXLlmWVdf/tHnkDTJ8+nXPnzvHyyy/neT969OhBnz59WLhwIcePHy+wH+4p4XHjxlGhQoWs9GrVqvHMM88UWPfhhx/miiuuyJHmnhHYsGFDgXWz+/e//813333HQw89RNeuXXPkXXbZZTzxxBMcPHgw630oje3hL5riFpEyYa1l1qxZJCcn8+WXX3L06FEuXLiQlZ89AACsX78eYwy9evUqtO2ilAXYvn07L774Ip9++impqamcOXMmR372/aRuFStW5Morr8yTvnPnTk6dOkWXLl08HrXerVu3PPs+58+fz5YtW3KktWnThn79+gFOEE1OTmbZsmVceeWVWdPbiYmJ7Nmzh5deeonjx48THh7O8uXLqVq1Ku3bt89q67PPPgNg1apVHn9sHD58mAsXLvDNN9/Qtm1bj+8RwH//+19CQkJISEjIk9e5c+d864ETDHOLjo4G4OjRowXWzc69Lnv37vV47vK3334LQEpKCr179y7W9ghUPgvQxpgGwFigF1ADSAXmA89aa73eGsaYKODPQD+gHnAEWAT82Vqb/5Ekv9a/C3jH9XKotfaNoqyHiJSOkSNHMmnSJOrVq8d1111H/fr1s/aZuvdBZnfs2DGqV6+eY79qfopSdv369fTo0YPz58+TmJhInz59iIiIICQkhC1btrBgwYI8B2EB1K5d2+NBVOnp6QDUqVPH4/Lq1q2bJ23+/Pl5gsSgQYOyAnT2/dCPPvooy5Yto0GDBjRr1ozExET++te/smrVKn7729+yfft2evfuTblyv36dHzlyBIAXX3yxwPfixIkTBeanp6cTFRWVo223/NbXrVq1annS3O1k/2FWGPe6vP/++wWWc69LcbZHoPJJgDbGNAHWAbWBBcBOoD3wCNDLGNPJWnvEi3ZquNppBiwH3gNaAH8AbjDGdLTW7i6gfjQwBTgBVC3RSomIzxw+fJjJkyfTunVr1q1bR3h4eI782bNn56lTrVo1jhw5wunTpwsNvEUpO27cOE6fPs2KFSvyHCk8YcIEFixY4LFefhfScI/SDh065DH/4MGDedKSk5NJTk7Ot4+XXXYZzZs3Z9WqVWRkZLBy5Ur69u0LOCPXChUqsHTpUn755RfAGXF76lN6ejoRERH5LqcwERERpKWlcf78+TxBOr/19TX3uixYsIA+ffp4Xb4o2yNQ+Wof9DSc4PywtbaftfZJa20P4O9Ac+B5L9sZjxOc/26tTXS10w8n0Nd2Lccj43x6ZuCMuF8t/qqIiK/t3r2bzMxMkpKS8gTn/fv3s3t33t/d8fHxWGtZtGhRoe0XpeyuXbuIioryeHqQ+3SfomjRogWavn2rAAAgAElEQVSVK1dmy5YtWaO37Ip7ycvExEROnDjB9OnTOXbsWNaounLlysTHx7Ns2bIcU9/ZxcfHA7B6dcnOhb/66qvJzMxk3bp1efLWrFlTorazCw0NBTyPrIu6LqW1PfyhxAHaGNMYSAL2AK/kyh4NnATuMsZUoQCu/Ltc5Ufnyp7qav861/I8eRjogTPaPun9GpSNB5c+yBVvX1Gkx4NLH/R3t0V8wn3KzJo1a3J8CZ84cYKhQ4dy/vz5PHUeeughAB577DGP+4SzpxWlbGxsLGlpaWzdujVHmTfffDPPgWreKF++PAMHDuT48eN59pFu3LiRWbNmFblN+HVUPGHChByv3X9v27aNDz/8kBo1anDVVVflqDt8+HDKly/Po48+yjfffJOn7bNnz3oV8O6++24ARo0axdmzZ7PS09PTee6554q+UvmoUaMGAPv27cuT17dvX5o0acIrr7zCf/7zH4/1P/vss6yroJXW9vAHX0xxu/9rllhrM7NnWGuPG2PW4gTweGBZ7srZdAQqudrJcWihtTbTGLMEGAZ0B3L83DbGtAReAF621n5qjMk53xMAgu1KUCK+VLduXW677Tbee+892rRpQ1JSEunp6XzyySdUrFiRNm3a5DloKikpiWeeeYbnnnuOli1bZp3bfOjQIdasWUN8fHzWNHFRyo4YMYLFixfTuXNnBgwYQGRkJBs3bmTNmjX079+fuXPnFnn9xo8fz7Jly5g0aRIbN27MOu92zpw59O7dmw8//LDIbXbv3p2QkBAOHz5MixYtchyNnZiYyJgxY/jpp5/o379/nun3Fi1a8NZbbzFkyBBatWpFr169aNasGefOnWPfvn2sXr2aWrVqsXPnzgL7cPfdd/Pee++xaNEiWrduTZ8+fTh37hz/+te/aNeuHV9//TUhISWfiE1MTOT999/n5ptvpnfv3lSqVImYmBjuuusuypcvz7x587juuuu44YYbSEhIoE2bNlSuXJkffviBL774gt27d5OamkrlypWB0tke/uCLAN3c9Zz3Z5rjW5wA3YyCA7Q37eBqJ4sxphwwE9gHPF1YZ/NjjNmUT5auVyjiA2+++SaNGzdmzpw5vPLKK9SqVYs+ffowduxYbrnlFo91xo4dS3x8PJMnT+ajjz7i5MmT1K5dm3bt2mWN7opatlevXixcuJBx48YxZ84cQkNDad++PStWrGD37t3FCtA1a9Zk7dq1PP3001kXXWnevDnTp08nNja2WAEhKiqKNm3asHnz5jz7mDt06ECVKlU4efJknjy3O++8k6uuuoqJEyeyYsUKlixZQpUqVbjsssvo378/v//97wvtgzGGDz74gPHjxzNz5kymTJlCvXr1GDRoEA8++CALFiwo0T5ut3vvvZe9e/fy3nvv8de//pXz58/TtWtX7rrrLgCuvPJKvvzyS1566SU++ugjZsyYQUhICPXq1ePqq6/m2WefpWbNmlntlcb28AdjnYtxFL8BY/4BDCWfI6aNMc/jBM6nrbUTCmjnaZx91c9ba0d5yB8K/AP4h7X2vmzpY4H/BTpbaz9zpY3BmSb3+ijuggJ0XFxc5U2b8sv2zhVvX1F4IQ++GvRViZYrZSslJQWAli1b+rTdYLtZxs6dO2nZsiXDhg3jtdde80sfpHR98sknJCUl8eSTT2ZNw1+MvP1Mt23bls2bN2+21uZ/3loRlcV50O65l5L9EvDQjjGmPU7wn+gOzsWV35vqCtxxJWlbpKT8FWiLy73f09MFQSS4HDhwgMsuuyxH2pEjR3jyyScBuOmmm/zRrUuCLwK0+zC5/O4rF5GrnE/ayTa1/Q1Q8CVtRKRMbN26lVmzZjFr1ixCQkL05X0RGDlyJF9++SUJCQnUqlWL/fv38/HHH5OWlsZ9992X4wIp4lu+CNBfu56b5ZPf1PWc377l4rZTNVvZM/mco/i6MeZ1nIPHRhSyfBEpoc2bNzNlyhRatGjBq6++SuvWrf3dJSmhm2++mUOHDrFw4UKOHTtGxYoVadWqFUOGDPF4UxHxHV8E6BWu5yRjTEj2I7mNMeFAJ+A0sL6Qdta7ynUyxoRnP5LbGBOCc6BZ9uVlAG/m01YccDWwBifwl2j6W0S8M3jw4By3mJTgN2DAAAYMGODvblySShygrbXfuU6BSgL+B+dKXm7PAlWA16y1WecmG2NauOruzNbOCWPMTJxTqcYAj2VrZzgQCyx2X0nMWnsa8PjzzXWQ2NXA27rUp4iIBCNfHST2IM4lOicbYxKBFKADzjnL3+AcZZ1dius597z000A3YKQxpg2wAWgJ9AUO4/wAuKS8u34vd8bH+LsbIiJSxnxyqU9r7XdAOyAZJzA/BjQBJgMdvbkOt6udIzgXLJkM/MbVTgecS3i2dS0n6Ly7fm/hhfIxav62EtUXEZHg5LPTrKy1P+BcZtObsp6vOu/kpeFce/uREvRlDM40ud/N3rCPUfO3EV6C02JHzd9GaIjh9vYNfdcxEREJaL66WYbkY8qybwsv5IWpy3f5pB0REQkOCtCl7KFE5+yw8yeaF1IyrytOhmb9PbzHb3zWJxERCXxlcSWxS9rt7RtyIdPy9oJr+STsE6yFfG4rm4O73LVmP4P69tL0tojIJUYBugzcGR9D8z1HYKd3wRl+LTf+qiP8Vkdxi4hccjTFXUZ+W698mdYTEZHgpgBdVsLCy7aeiB/t2bMHY4yuKgasXLkSYwxjxozxd1ckyChAl5XGXcu2noiIBDXtgy4rtVtCTCfYu9b7OjGdnXpyybuQns6xuXPJPHXa6zohlStR7dZbCY2IKLywj9WvX5+UlBQiI/O7OZ2IFEYBuix1fQJm3gS/3k8kfyYEuj5e+n2SoJC+8CMOv/i3ItczYRWJunNgKfSoYOXLl6dFixZlvlyRi4mmuMtS425w48tO8C2ICYEbJzvlRYCq13SB0NDCC2YXGurU8wNP+6C/+eYbnnzySdq1a0etWrUICwsjJiaGYcOGsX///nzbWrJkCTfeeCO1a9cmLCyM6Oho+vbty9KlS4tV9uzZs0ydOpXevXsTExNDWFgYUVFR9OzZk48//thjH2JjY4mNjeWXX35h5MiRxMbGUr58+Rz7lQ8dOsQ999xDnTp1qFSpEm3atOHtt98u+psn4qIRdFmLuxuqNYRVL8LeNXnzYzo7I+fG3cq6ZxLAKjRsSGTfvqTPm+d1nci+fanQMHDOn583bx6vvvoq3bt3JyEhgQoVKrB9+3beeOMNFi5cyMaNG6lfv36OOqNHj2bs2LFUrVqVfv36ER0dzYEDB1i3bh3vvvsuPXv2LHLZtLQ0HnnkERISErj22mupVasWqampLFy4kN69e/P66697vM/x2bNn6dGjB2lpaSQlJREREUGjRo0AOHLkCAkJCezevZvOnTvTuXNnUlNTuf/++0lKSsrTlog3FKD9oXE353E4BXavgozjztHajbtqn7Pkq+b995G+YAFcuFB44dBQat5/X+l3qgjuuusuHn30UcLCwnKkL1myhOuvv55x48Yxffr0HOljx46lUaNGrF69Ok/wzj7qLkrZ6tWrs3fvXho0aJCjTHp6Op06deKJJ55g4MCBVKpUKUd+amoql19+OatWraJKlSo58p566il2797NiBEj+Pvf/56VPnz4cDp27OjN2yOSh6a4/al2S4i/3xkxx9+v4CwFco+ivRFoo2dwDhzLHZwBkpKSaNWqFYsXL86RPmWKc2v5iRMn5gm4QI4AW5SyYWFheYIzQGRkJEOGDOHo0aN88cUXHtdh4sSJeYLzuXPnmDVrFuHh4XlOpWrXrh0DB5b9MQBycVCAFgkiNe+/r/B90QE4egaw1mZNNdeqVYty5cphjMEYw1dffcWPP/6Yo/z69esxxtCrV69C2y5KWYDt27czePBgGjduTKVKlbL68dhjjwHk6QtAxYoVufLKK/Ok79y5k1OnTtGmTRuPR61369bNqz6J5KYpbpEg4s2+6EAcPQOMHDmSSZMmUa9ePa677jrq16+fNY2cnJzM3r0573t+7Ngxqlevnmeq2ZOilF2/fj09evTg/PnzJCYm0qdPHyIiIggJCWHLli0sWLCAjIyMPPVq166N8XCt3vT0dADq1KnjcXl169YttE8inihAiwSZAvdFB+jo+fDhw0yePJnWrVuzbt06wsNzXiFv9uzZeepUq1aNI0eOcPr06UIDb1HKjhs3jtOnT7NixYo8o9sJEyawYMECj/U8BWcga9R86NAhj/kHDx4ssD8i+dEUt0iQKWhfdKCOnnfv3k1mZiZJSUl5gvP+/fvZvXt3njrx8fFYa1m0aFGh7Rel7K5du4iKivI49bxq1apC6+fWokULKleuzJYtW7JG09mtXLmyyG2KgAK0SFDyuC86QEfP4JxHDLBmzRouZBv5nzhxgqFDh3L+/Pk8dR566CEAHnvsMY/7hLOnFaVsbGwsaWlpbN26NUeZN998M8+Bat4oX748AwcO5Pjx43kOEtu4cSOzZs0qcpsioClukaDkaV90oI6ewdkPe9ttt/Hee+/Rpk0bkpKSSE9P55NPPqFixYq0adOGLVu25KiTlJTEM888w3PPPUfLli2zzm0+dOgQa9asIT4+nuTk5CKXHTFiBIsXL6Zz584MGDCAyMhINm7cyJo1a+jfvz9z584t8vqNHz+eZcuWMWnSJDZu3Jh1HvScOXPo3bs3H374YUnfQrkEKUCLBKkc+6IDePTs9uabb9K4cWPmzJnDK6+8Qq1atejTpw9jx47llltu8Vhn7NixxMfHM3nyZD766CNOnjxJ7dq1adeuHXfffXexyvbq1YuFCxcybtw45syZQ2hoKO3bt2fFihXs3r27WAG6Zs2arF27lqeffjrroivNmzdn+vTpxMbGKkBLsRhrrb/7ENCMMZvi4uLiNm3a5O+uSBBISUkBoGXLsjmn/cDT/0v6vHlE3nwzl41/vkyW6Y2dO3fSsmVLhg0bxmuvvebv7ogUm7ef6bZt27J58+bN1tq2vlq2RtAiQazO009RsdXlRPbt5++u5PDNN98AeLwgiIh4RwFaJIiFVq1KVABdqWrr1q3MmjWLWbNmERISwk033eTvLokELR3FLSI+s3nzZqZMmULt2rVZsGABrVu39neXRIKWRtAi4jODBw/OcYtJESk+jaBFREQCkAK0iIiIB/4+y0kBWsSH3NdrzszM9HNPRKSk3AE6v+uwlzYFaBEfct/v+OTJk37uiYiUlPtz7Ok+5mVBAVrEh9w3gjh48CDHjx8nMzPT79NkIuI9ay2ZmZkcP348605kuW/wUlZ0FLeID0VFRXHy5ElOnTrF/v37/d0dESmhypUrExUV5ZdlK0CL+FBISAjR0dGkpaVx/PhxMjIyNIIWCTLGGMLCwggPDycqKoqQEP9MNitAi/hYSEgINWvWpGbNmv7uiogEMe2DFhERCUAK0CIiIgFIAVpERCQAKUCLiIgEIAVoERGRAKQALeJDszfsI2HCMmZv2OfvrohIkNNpViI+8u76vYyavw2Ap+Z9xYVMy53xMX7ulYgEK42gRXwge3B2GzV/G++u3+unHolIsFOAFikhT8HZTUFaRIpLAVqkBAoKzm4K0iJSHArQIsXkTXB2U5AWkaJSgBYphqIEZzcFaREpCgVokSIqTnB2U5AWEW/pNCuRIpi9YR+j5m+j6tlTXLf3cyqeP5uVV8P8QrQ5TAXOc5Zy/GBrc8RGZOWfKVeBRTHxjJq/jdAQw+3tG/pjFUQkSChAixTBlGXfApB0Yhr3bj9YYNnWfJ8n7WxoeRY27szU5bsUoEWkQArQZeRCejrH5s4l89Rpr+uEVK5EtVtvJTQiovDCUiYeSmzKU/O+YuvlP3FhNYRa7+teMLCxTgsAhvf4TSn1UEQuFgrQZSR94UccfvFvRa5nwioSdefAUuiRFMft7RtyIdPyl68Nn15h6L7V+wj96RWG1Co1GdevtUbPIlIoHSRWRqpe0wVCQ4tWKTTUqScBxX35znkJIVwwUFiItjij53kJIYzr11qX/xQRryhAl5EKDRsS2bdvkepE9u1LhYYaaQWqQ9WdUbQppJzBGT0fqm64s/GpsuiaiFwEFKDLUM377/N+FB0a6pSXgFbYKDr76BmA3avKqmsiEuQUoMtQUUbRGj0Hh8JG0dlHzwBkHC+rrolIkFOALmNejaI1eg4q+Y2i84yeAcLCy7JrIhLEFKDLmDejaI2eg0t+o+g8o2eAxl3LsmsiEsR8FqCNMQ2MMW8ZYw4YYzKMMXuMMZOMMdWL2E6Uq94eVzsHXO028FC2hjHmXmPMB8aYXcaY08aYdGPMGmPMPcaYgPwBUuAoWqPnoJR7FO1x9AxQu2VZd01EgpRPApgxpgmwCfgDsAH4O7AbeAT4zBhTw8t2agCfuep952png6vdTcaYxrmq3Aq8DnQAPgcmAf8CWgNvAP80xhR2kG2ZK2gUrdFzcMo9ivY4ehYRKQJfjTCnAbWBh621/ay1T1pre+AE2ObA8162Mx5oBvzdWpvoaqcfTsCu7VpOdt8AfYAG1tqB1tqnrLVDgBbAD8AtwM0lXbnS4HEUrdFzUHOPoiGf0bOISBGU+BvENapNAvYAr+TKHg2cBO4yxlQppJ0qwF2u8qNzZU91tX9d9lG0tXa5tXahtTYze2Fr7UHgVdfLbkVYnTLjaRSt0XNwc4+iQaNnESk5X/zE7+F6XuIhUB4H1gKVgfhC2ukIVALWuuplbycTWOJ62d3Lfp1zPZ/3prAxZpOnB85ovFTkGEVr9BxUutT3fIW35J4hvJkUwoyeeT9a+dUREfHEF9fibu56/iaf/G9xRtjNgGUlbAdXOwUyxpQD7na9XFRYeX9xj6LT583T6DnITOuZe2+LiIhv+SJAR7qe0/PJd6dXK6N2AF7AOVDsP9baxV6Ux1rb1lO6axQd500bxVHn6aeo2OpyIvv2K61FiIhIECqLu1m5d8QV4cZ8xW/HGPMw8BiwE2efdkALrVqVqIG6W5WIiOTki33Q7pFtZD75EbnKlVo7xpj/AV4GdgDdrbVphSxTREQkIPkiQH/tes5v33BT13N++5Z90o4xZgTO0d7bcILzwUKWJyIiErB8McW9wvWcZIwJyX4ktzEmHOgEnAbWF9LOele5TsaY8OxHcruuCJaUa3lky/8Tzn7nLcC11tqfi7syInLpeXDpg6z+cXWR6nSp30UHC0qpKvEI2lr7Hc4pULHA/+TKfhaoArxjrT3pTjTGtDDG5Dh9yVp7ApjpKj8mVzvDXe0vttbuzp5hjHkGJzhvAhIVnEWkqIoanItbR6QofHWQ2IPAOmCyMSYRSMG5/GZ3nCnp/81VPsX1nPtKDk/jXFhkpDGmDc5lPlsCfYHD5PoBYIwZBIwFLgCrgYc9XNlzj7U2uZjrJSIi4hc+CdDW2u+MMe1wgmUvoDeQCkwGnvX2YC1r7RFjTEecK4n1A7oAR4AZwJ+ttftzVWnkeg4FRuTT7Cog2fu1ERER8T+fnWZlrf0B56YW3pTN9xqIrmD+iOtRWDtjyDsdLiIiEvR0NX8REZEApAAtIiISgBSgRUREApACtIiISABSgBYREQlACtAiIiIBSAFaREQuerM37CNhwjJmb9jn7654rSxuNykiIuI3767fy6j52wB4at5XXMi03Bkf4+deFU4jaBG55HWp36VM6kjZyx6c3UbN38a76/f6qUfe0whaRC5503pO8/hFXpCE5q1LsUfiCwVtU3d6II+kFaBF5JJX1OAMwfEFfym6kJ7Osblz2fJNKik7DzOwgLIpKYtZ1qI2bZrVo9qttxIaEVFm/fSGArSIXNKKE5zdFKQDT/rCjzj84t+4DLjTmwpfO7dKNGEVibqzoHBe9rQPWkQuWbM37Ct2cHYbNX9bUB0ZfLFbGtGYC6Zooe2CCWFpRONS6lHxKUCLyCVryrJvfdLO1OW7fNKOlMy76/fyxLo0lkW3LVK9ZdFteWJdWsAdOKYALSKXrIcSm+ab19TsZ3DoIoaHfsDg0EU0NblvR/+r4T1+UxrdkyJwz4ZUip7Bwt6buWDAFlLHAhcMfNDtGBB4syHaBy0il6zb2zfkQqbNMc2dELKNR8rNo0PIzjzlP89swcvnb2Zd5q9HcI/r15rb2zcsk/5K/tyzIeWqfs0hDJ9eYei+teAQbYBPrzAcqf89/OKkTV2+K2C2p0bQInJJuzM+hnH9nIA7IHQFM8tPoEPITmyu73ZroUPITmaWn8CtoSsBJzjrALHAkHs2ZF5CSIGjaPfoeV5CzjAYSLMhGkGLyCXvzvgY6h5ZT/cNbxBqnK90Y3KWcb8ONZYXyr3OdQnt6KngHDDcsyF/+dp5fah6waNo9+j5UPVfN3SgzYZoBC0iAvQ8/HZWcC5MqLH0PPxOKfdIiir3bEZ+o2hPo+cpiRUDbjZEAVpE5HAK7F1btDp71zj1JGC5R9G5JkM8jp5vDPfNEf2+pAAtIrJ7VdnWkzKTexSd375nMo6XddcKpQAtIlLcL+cA/FKXnHKPoj2NngEICy/rrhVKAVpEpLhfzgH4pS55uUfRkM/oGaBx17LtlBcUoEVEivvlHIBf6pKXexQN+YyeAWq3LONeFU6nWYmI1G4JMZ2KdqBYTOeA/FIXz5J7hrC7rmVVaw/BOUBpBC0iAtD1CfD2JgsmBLo+Xrr9EZ86HWZY3DaEM2EK0CIiwaVxN7jx5cKDtAmBGyc75UVKkQK0iIhb3N1w1wfO9LUnMZ2d/Li7yrZf4rUu9buUSZ2yoH3QIiLZNe7mPA6nOOc5Zxx3jtZu3FX7nIPAtJ7T/N0Fn1GAFhHxpHZLBWTxK01xi4iIBCAFaBERkQCkAC0iIhKAFKBFREQCkAK0iIhIAFKAFhERCUA6zUpELnkX0tM5NncumadOe10npHIlqt16K6EREaXYM7mUKUCLyCUvfeFHHH7xb0WuZ8IqEnXnwFLokYimuEVEqHpNFwgNLVql0FCnnkgpUYAWkUtehYYNiezbt0h1Ivv2pULDhqXUIxEFaBERAGref5/3o+jQUKe8SClSgBYRoWijaI2epSwoQIuIuHg1itboWcqIArSIiIs3o2iNnqWsKECLiGRT4Chao2cpQwrQIiLZFDSK1uhZypICtIhILh5H0Ro9SxlTgBYRycXTKFqjZylrCtAiIh7kGEVr9Cx+oAAtIuJB9lG0Rs/iD7pZhohIPuo8/RQVW11OZN9+/u6KXIIUoEVE8hFatSpRA3W3KvEPTXGLiIgEIAVoERGRAKQALSIiEoB8FqCNMQ2MMW8ZYw4YYzKMMXuMMZOMMdWL2E6Uq94eVzsHXO02KO1li4iIBAqfHCRmjGkCrANqAwuAnUB74BGglzGmk7X2iBft1HC10wxYDrwHtAD+ANxgjOlord1dGssWEREJJL4aQU/DCZAPW2v7WWuftNb2AP4ONAee97Kd8TjB+e/W2kRXO/1wgm1t13JKa9kiIiIBo8QB2hjTGEgC9gCv5MoeDZwE7jLGVCmknSrAXa7yo3NlT3W1f51reT5dtoiISKDxxQi6h+t5ibU2M3uGtfY4sBaoDMQX0k5HoBKw1lUvezuZwBLXy+6lsGwREZGA4ot90M1dz9/kk/8tzii3GbCshO3gasfXy8YYsymfrKtSUlJo27ZtQdVFROQSlpKSAhDryzZ9EaAjXc/p+eS706uVQju+WnZBLpw+fTp98+bNe0rQRkFauJ53llL7Ura0PS8u2p4Xj9LelrHAL75ssCwu9Wlcz9YP7Xhdx1rrlyGye+Tur+WLb2l7Xly0PS8ewbgtfbEP2j1KjcwnPyJXOV+246tli4iIBBRfBOivXc/N8slv6nrObz9xSdrx1bJFREQCii8C9ArXc5IxJkd7xphwoBNwGlhfSDvrXeU6ueplbycE52Cv7Mvz5bJFREQCSokDtLX2O5xToGKB/8mV/SxQBXjHWnvSnWiMaWGMaZG9oLX2BDDTVX5MrnaGu9pfnP1KYsVZtoiISDAw1pb02C2Pl9tMATrgnLP8DZCQ/XKbxhgLYK01udrJfanPDUBLoC9w2NXOdyVZdqAJxgMXJH/anhcXbc+LRzBuS58EaABjTDQwFugF1ABSgfnAs9batFxlPQZoV14UzlXA+gH1gCPAx8CfrbX7S7psERGRYOCzAC0iIiK+o/tBi4iIBCAFaBERkQCkAC0iIhKAFKBFREQCkAK0iIhIAFKAFhERCUAK0HLRMsbsMcbsyfZ6sDHGGmMG+69XIlKWgvlzrwAdpIL5n05+ZYyJdW3HZH/3ReRSkvsHfCAqi/tBiwSKD3BunJLq746ISJkJ2s+9ArRcMqy16eje4CKXlGD+3GuKuwSMMe2NMXOMMT8aYzKMManGmCXGmAHZygw2xvzLGLPbGHPaGPOLMWatMebOfNpsbIz5hzFml6t8mjHmK2PMq66biWCMWQnMcFWZ4ZoidT9iS3m1A4pxDDfGbDfGnHFti6nGmEgPZT3uFnBPdRljIowxL7n+PmeMGZOtTDljzIPGmPWubXjKGPNf17I9fo4K+/9wtf+9q/igXNtxcLZ2Qowx9xtjvjDGnDDGnHT9/YCnZbvqrzTG1DXGvOFa/oVg3B3i2r6PGGN25N6++U1RGmNuN8asMMYcddVJMcaMMsaEeSjrfq9quj53qa5ttd0Y8wcP5bu56oxxbd9/uz6jOT57xpgGrn7udrV3xBjzoTHmtx7aDDfGPGOM2eb63zpujPnO9b8TNDd2KA5jTFVjzFljzNpc6ZVc284aY+7KlfegK32I6/VK1+sKxpg/G2O+dr3nya78HJ979zYEYoCYXJ+75FzLamGMSTbG/OBq85Ax5v+MMc09rEuyq43GxpiHjDFbjfMdvrK4749G0MVkjBkKTAcuAB8C3+LcUasd8CDwT1fR6cAO4FOcKZYaQG9gpjGmubX2mWxt1gO+ACKA/yFXlQcAAArbSURBVAD/AioCjYC7gKk4Nw9JBo7h3OVrAbAlW9eO+XxlA9sk4GGc9/YfwDmc96UDUAE462U7FXDuoBaFcwvTX3AFT2NMeWAhcB3wNfB/wBmcO6ZNcS0r95eIN/8fK4FqwCPAlzg3eHHLvk1nAncAPwBvABa4CZgGdAYGelifKJxpvRPAPCATOOTlexFIXgEeAA7gbN+zQB+gPVAeZ3tnMca8CQwB9uOs9zEgHngOSDTGXGutPZ9rGdWAta625+J85voDbxljMq21b3voV0fgKWAN8BZQ01UfY0wczv9QFLDY1Y+aODcAWmOMucla+x9XWQMsAhKAz3C273kgGugGrAY2FeH9CirW2hPGmA1AB2NMuLX2uCurE+D+QZWI8xlw6+F6XparuX8Bv8W5udJ8nDsgerIH53bEI1yvJ2XLy/rcGWN64Ww79+d/F9AAuBm4wRjT3Vq72UP7LwNdgH/jfI9fyKcfhbPW6lHEB3A5zhdDGtDKQ36DbH838ZBfAeef6xxQP1v6Qzhfvo94qFMFqJTt9WBX2cH+fj/8uB0SXO/BLiAqW3pFnC87C+wp7D3D+cBaYClQxcNyxrjypwCh2dJDgTddeX2L+f8R66qfnM863u7K3wxUzfX/sNGVd0euOtb1eAco5+/tVILt28W1Hl8D1bKlV8D5wZvf9p2X/bOSaxs+kivd/V69kWvbXo4TKHfkKt8tW537PPS5nOv/8QzQNVfeZcCPOD8mw1xpV7ja+sBDWyFAdX9vhzLYzmNd78EN2dImuN7/5cAPud6Tn4HvsqWtdNXfCtT00H5Bn/s9+fSpOnDUtazLc+W1wvnhuzlXerJrOT8CjXzx3miKu3gewPkgPmet3Z4702a7LabNdf9qV9pZnJFBOZxfh7md9lDnpLU2T/olzj0F+bzNdltRa+0ZnNFNUT1mrT2ZPcE1hTwcOAg8aq3N+jXs+vsxnA9l9lGs1/8fXhjien7SWnsiWxsngT+5Xt7rod5Z4I8272gxmAxyPT9vrc2aGXJ9fjxt30dwvtSHePisPIcz++RptuEUMDLXtt2BM6puaYwJ91Bni7X2NQ/pNwBNgCnW2lXZM6y1B4C/AnXJ+7n39JnPtNYe9bCMi417JJz9PUnEmTn4F9DAGNPMld4GZxYy9+gZ4Blr7c8+6tPdODMro13/C1lcn+nXgauNMZd7qPtXa+33HtKLTFPcxRPvev64sILGmIY4X6SJQEOgUq4i9bP9/SEwHnjFGHMdzvTYWpxf8bovaF5xrudVHvJW43xZe+sMzi/w/2/nTEOtqqI4/vtrhaiQGhrRaGQkClI0WVlm0USSRANF0YcGwiCsT5WNFA0QURpZQTZAEZXYCA1oGjaSSKEpldCkiKRWik2+Vh/WPt7z7jvn3ev1+t7xun5w2e/dvc85+9y191n7rGHXcyT+QPgOuN0tkj34Exib+7/p8dEEx+Dm6UUFdYtx89nRBXU/mFmZiW93IbuvJQV1n5GTr6TBwAT8jWdGiZz+prucMr4zsz8Kvv85lcOAzXV1X5T0eWIqD1UuhiHHmFSOxc2f3+Bm1cskHYq7rJYAX6aFyJ7Ap/gcOgNAHj9yDL6YWZjanAF8S828vZCelMmkFTI5TiiRY7ZgGIvLcJf0IxR0awxL5ZreGkk6HBfWcFxhvI9HE3bhps2rqPlZMLMfJR2Pm+POwX0dAD9LetjMZrXvFjqCLBCsh2/VzLokbdiBc60vWQTtl8oxwF29HD8093dT46NJ9gU2Fj2szWybpF9x33Y969pw7f5mR+Q7HBAwkt7lVERZ3Ea2ABhYUFf2+2bj5eIG1xwK2+9jCnAn7vd+KNVvlvQ8cGvectKJmNk/kpYAZ0oahSvHgcACM1spaS2uoOek0ihW0O0c85kcr23QbmjBd23rR5i4WyOb0Af22gpuxgV9tZlNNrMbzewOM7sbfzvugZmtNLNL03HHArfgcnpM0tVt6X3nkKVO7F9fIWkgtUnWDGUWiuwa881MvXxG545pdnw0w+/AiBSo1g1Je+HBR0Vvf51gccnuqxn5ZnJa1kBOha/WLdBovFzQoB/3bD+R2SYzu8nMDsYXgtcAq3DXypw29bfqLMQXWFNwJfw3bj0E+BA4PUXhTwJWFFmH2mxlzOQ4oYEciwII29aPUNCt8Vkqz23Q7ohUziuoO623A81sm5ktNbOH8EAh8CjQjMxfVrS631PIIiiLfstJtMdCtIoUCVykJEtodnxAYzkuw+fpqQV1p6bjiiJJO4FlqTyloO5EcvJNb5krgHGSRvRB38rIZD+plYPN7HszewYf01vwjIQ9gbwfegrwcYolyepG4LEdQyj2P7dCF+Xzbqfk2C5CQbfGHNz8dUdRkICkg9KfP6Rycl392RQE9qS8yh5vC9TeILbmvsvMe4c03evO47lUzsw/lCUNwqNAd5oUZDUbOACYJak+hgBJB9SNg2bHB3ikqFEux7mpfCD5WbNzDAYeTP8+0+Tt7G68kMqZyuW1S9oHj9Wo5xE8wnuupGH1lZKGpxSoXckbwGrgBknnFTWQNDGTpaTRksYVNBuOu7/2lMDQpdRSR8fRXQlnf2eBgUXm7VbYAIwsmtP4PhO/AXclt2M35HsTTG5TP0oJH3QLmNk3kqYDTwLLJL2BBxFlZunNeI7sE3ik8auS5uE+yfG4f/kV4NK6U1+OT+zFeKrGJjwidCpu8snn632KK+wZSTllfrrZ5jvndDxm9rGk2Xh62nJJr1HLg95E+7b2uxcPQLoemCppIS7LUbhJ8mRgJilYZAfGB+Z5oJ8DkyS9iAfCdAFvmtnXZvaSpAuAS4AVkl7HFfo0PD/+FTN7sU33WSnMbLGkp4Hr8Hufh8t3Km6CXIsH0GXt58o39pgOrJb0HvAT/vY1Grc4PIvLcVf1+V9JF+IurHckfYIHgW3Fc5uPAw7HF3xb8XE1X9JSYHm6p5H4GN6bmk+6ozGz/9JzL7MYLMjV/SRpNf4s7KI4KLQVFuDyeFfSR/gz9isze8vMNki6iLRNqKQFuIXmP3wxPRGfz4Pa1JdiWsnNis/2vLeJuPl6PZ7WshbfdOCiXJuT8BXfJvzBvAR/uE7GH7R359qegL99fYXn0P6JK+pngfEF1z8HV9RbqOVmHtbfv0sfy0C4r24lPsHW4ils+1KX50gL+ZB117kSn9Qbk7zXJHneBhzcyvhI7Y7AN0LYgD8AuvURt3RNx/Oet6bPUuAGYEDBdQ1Y1N+yaZN8BwA34a6GevluxtOd6o85H3g797uvw4M17wOOava3opbXeljuux7ztuTYUbiFY3mS1xZ8kfYacAUpPx3f+OJ+3N+6Lt3jL3gGwLn9/fv3sayzfSB+J5eTnuqeSnWfFxy3iOSCLjlv2bwfgj9vf8EtXj32I8CDeR9PsvsLj4tYhW+cMq3ReNnZj9KJgyAIdhskjcGtDS+b2WWN2gfB7kj4oIMgqCzy/cQH1H03mJq7Z37f9yoI+obwQQdBUGVm4Jt4LMJjCrJduA7CzcCv9l/XgmDXEgo6CIIq8wEeSHUWHuy1DTdtzwIetfDRBR1M+KCDIAiCoIKEDzoIgiAIKkgo6CAIgiCoIKGggyAIgqCChIIOgiAIggoSCjoIgiAIKkgo6CAIgiCoIKGggyAIgqCChIIOgiAIggoSCjoIgiAIKkgo6CAIgiCoIKGggyAIgqCChIIOgiAIggryP+msizkY/Qr+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f616569e610>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 148,
       "width": 244
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_categorical(feature_scores, 'feature', 'score', 'metric')\n",
    "plt.ylim(0, 0.066);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_scores.to_csv('datasets/feature_scores_ML.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_combi = [{'cast':jw, 'director':jw}, {'cast':jw, 'genres':tc}, {'cast':jw, 'director':jw, 'genres':tc}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['d-jw', 'c-jw']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combi_key = lambda f, s: '{}-{}'.format(f[0], ''.join([x[0] for x in s.split('-')]))\n",
    "[combi_key(f, s) for f, s in feat_combi[0].iteritems()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_sel_res_combi = {}\n",
    "folds = [1, 2, 3, 4, 5]\n",
    "for stype in tqdm_notebook(feat_combi, desc='combi', leave=False):\n",
    "    item_similarity = combine_similarity_data(meta_info[stype.keys()], similarity_type=stype)\n",
    "    similarities = {'userid': None, 'movieid': item_similarity}\n",
    "\n",
    "    data_model = SimilarityDataModel(similarities, sim_indices, ml_data, 'userid', 'movieid', 'rating', seed=0)\n",
    "    data_model.random_holdout = True\n",
    "    data_model.holdout_size = 1\n",
    "    data_model.verbose = False\n",
    "    \n",
    "    cb = ContentBased(data_model)\n",
    "\n",
    "    for fold in tqdm_notebook(folds, desc='fold', leave=False):\n",
    "        data_model.test_fold = fold\n",
    "        data_model.prepare()\n",
    "        cb.build()\n",
    "\n",
    "        key = '_'.join(combi_key(f, s) for f, s in stype.iteritems())\n",
    "        nusers = data_model.test.evalset[data_model.fields.userid].nunique()\n",
    "        feat_sel_res_combi.setdefault(key, []).append(cb.evaluate().true_positive / nusers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[c-jw_g-tc]</td>\n",
       "      <td>0.026338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[d-jw_c-jw]</td>\n",
       "      <td>0.033957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[d-jw_c-jw_g-tc]</td>\n",
       "      <td>0.031474</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            feature     score\n",
       "0       [c-jw_g-tc]  0.026338\n",
       "1       [d-jw_c-jw]  0.033957\n",
       "2  [d-jw_c-jw_g-tc]  0.031474"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_scores = pd.DataFrame(pd.Series(feat_sel_res_combi)\n",
    "                              .reset_index()\n",
    "                              .rename(columns={'index':'feature', 0:'score'})\n",
    "                              .assign(feature = lambda x: x.feature.str.split('--'))\n",
    "                              .assign(score = lambda x: x.score.apply(np.mean))\n",
    "                             )\n",
    "feature_scores"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "navigate_menu": false,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}