main_solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables = {\n",
    "    'orders': '/kaggle/input/test-recsys/kaggle_tab_1345/tab_1_orders.csv',\n",
    "    'products': list(sorted(glob.glob('/kaggle/input/test-recsys/sbermarket_tab_2*/*'))),\n",
    "    'categories': '/kaggle/input/test-recsys/kaggle_tab_1345/tab_3_categories.csv',\n",
    "    'users': '/kaggle/input/test-recsys/kaggle_tab_1345/tab_4_user_profiles.csv',\n",
    "    'product_properties': '/kaggle/input/test-recsys/kaggle_tab_1345/tab_5_product_properties.csv',\n",
    "    'cities': '/kaggle/input/test-recsys/tab_6_city.csv',\n",
    "    'sample_submission': '/kaggle/input/test-recsys/sample_submission.csv',\n",
    "}\n",
    "\n",
    "products_dtypes = {\n",
    "    'user_id': 'int32',\n",
    "    'order_id': 'int32',\n",
    "    'line_item_id': 'int32',\n",
    "    'price': 'float32',\n",
    "    'quantity': 'int32',\n",
    "    'discount': 'float32',\n",
    "    'product_id': 'int32',\n",
    "    'master_category_id': 'float32',\n",
    "    'parent_category_id': 'int32',\n",
    "}\n",
    "\n",
    "# tables['products']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((107068,), array([  51,   65,  766, 1132, 1578]))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subm = pd.read_csv(tables['sample_submission'])\n",
    "subm_ids = subm.Id.values\n",
    "subm_ids.shape, subm_ids[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(   user_id  order_id  line_item_id  product_id\n",
       " 0      525   5354800      28381452       37548\n",
       " 1      525   5354800      29242911        5636\n",
       " 2      525   5354800      29242919       22107\n",
       " 3      525   5354800      29243785        2530\n",
       " 4      525   5354800      29244246     3818486,\n",
       " (59596980, 4))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products = pd.concat(list(map(lambda x: pd.read_csv(x, usecols=['user_id', 'order_id', 'line_item_id', 'product_id'], dtype='int32'), tables['products'])))\n",
    "products.head(), products.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtered products only for test users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21677070, 4)\n"
     ]
    }
   ],
   "source": [
    "products_subm = products[products.user_id.isin(subm_ids)]\n",
    "print(products_subm.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(   user_id  order_id  line_item_id  product_id\n",
       " 0      525   5354800      28381452       37548\n",
       " 1      525   5354800      29242911        5636\n",
       " 2      525   5354800      29242919       22107\n",
       " 3      525   5354800      29243785        2530\n",
       " 4      525   5354800      29244246     3818486,\n",
       " (59586459, 4))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products = products[products.product_id.isin(products_subm.product_id.unique())]\n",
    "# products = products[products.product_id != 0]\n",
    "products.head(), products.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# products_subm = products"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode `line_item_id` inside each order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_id</th>\n",
       "      <th>line_item_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>line_item_pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>5324015</td>\n",
       "      <td>29478760</td>\n",
       "      <td>5501</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>5324015</td>\n",
       "      <td>29478763</td>\n",
       "      <td>19562</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>5324015</td>\n",
       "      <td>29478764</td>\n",
       "      <td>50190</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51</td>\n",
       "      <td>5324015</td>\n",
       "      <td>29478780</td>\n",
       "      <td>45435</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>5324015</td>\n",
       "      <td>29478782</td>\n",
       "      <td>3817487</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  order_id  line_item_id  product_id  line_item_pos\n",
       "0       51   5324015      29478760        5501              0\n",
       "1       51   5324015      29478763       19562              1\n",
       "2       51   5324015      29478764       50190              2\n",
       "3       51   5324015      29478780       45435              3\n",
       "4       51   5324015      29478782     3817487              4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products_subm = products_subm.sort_values(['user_id', 'order_id', 'line_item_id'], ascending=[True, True, True])\n",
    "products_subm.head()\n",
    "\n",
    "line_item_idx = products_subm.groupby(['user_id', 'order_id']).cumcount().reset_index()\n",
    "products_subm['line_item_pos'] = 0\n",
    "products_subm['line_item_pos'] = products_subm['line_item_pos'].astype('int32')\n",
    "products_subm['line_item_pos'] = line_item_idx.values[:, 1]\n",
    "products_subm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Take last n orders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_last_orders = 5\n",
    "\n",
    "# last_user_orders = products_subm.groupby(['user_id', 'order_id']).count().reset_index().sort_values(['user_id', 'order_id'], ascending=[True, False])\n",
    "# last_user_orders = last_user_orders.groupby('user_id').head(n_last_orders).reset_index(drop=True)[['user_id', 'order_id']]\n",
    "# last_user_orders.head()\n",
    "\n",
    "# products_subm = products_subm.merge(last_user_orders, how='inner', on=['user_id', 'order_id'])\n",
    "# products_subm.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mean line item position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>line_item_pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>40.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>159</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>397</td>\n",
       "      <td>50.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51</td>\n",
       "      <td>407</td>\n",
       "      <td>27.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>456</td>\n",
       "      <td>24.666666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  product_id  line_item_pos\n",
       "0       51           0      40.750000\n",
       "1       51         159       2.000000\n",
       "2       51         397      50.000000\n",
       "3       51         407      27.500000\n",
       "4       51         456      24.666666"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_product_line_pos_mean = products_subm.groupby(['user_id', 'product_id'])['line_item_pos'].mean().reset_index()\n",
    "user_product_line_pos_mean = user_product_line_pos_mean.astype({'user_id': 'int32', 'product_id': 'int32', 'line_item_pos': 'float32'})\n",
    "user_product_line_pos_mean.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Last order id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>last_order_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>7767199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>159</td>\n",
       "      <td>7767199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>397</td>\n",
       "      <td>6905330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51</td>\n",
       "      <td>407</td>\n",
       "      <td>9096837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>456</td>\n",
       "      <td>13911065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  product_id  last_order_id\n",
       "0       51           0        7767199\n",
       "1       51         159        7767199\n",
       "2       51         397        6905330\n",
       "3       51         407        9096837\n",
       "4       51         456       13911065"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_order_id = products_subm.groupby(['user_id', 'product_id'])['order_id'].max().reset_index()\n",
    "last_order_id = last_order_id.rename(columns={'order_id': 'last_order_id'})\n",
    "last_order_id = last_order_id.astype('int32')\n",
    "last_order_id.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User-product frequencies\n",
    "\n",
    "sorted by `cnt`, `last_order_id`, `mean_line_idx`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>56037</td>\n",
       "      <td>0</td>\n",
       "      <td>471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>222648</td>\n",
       "      <td>3818317</td>\n",
       "      <td>364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>56037</td>\n",
       "      <td>6375406</td>\n",
       "      <td>360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>222648</td>\n",
       "      <td>709</td>\n",
       "      <td>332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>56037</td>\n",
       "      <td>12565</td>\n",
       "      <td>330</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  product_id  cnt\n",
       "0    56037           0  471\n",
       "1   222648     3818317  364\n",
       "2    56037     6375406  360\n",
       "3   222648         709  332\n",
       "4    56037       12565  330"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_product_freqs = products_subm.groupby(['user_id', 'product_id'])['order_id'].count().sort_values(ascending=False).reset_index()\n",
    "user_product_freqs = user_product_freqs.rename(columns={'order_id':'cnt'})\n",
    "user_product_freqs = user_product_freqs.astype('int32')\n",
    "user_product_freqs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>cnt</th>\n",
       "      <th>last_order_id</th>\n",
       "      <th>line_item_pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10830</th>\n",
       "      <td>51</td>\n",
       "      <td>14863</td>\n",
       "      <td>16</td>\n",
       "      <td>13911065</td>\n",
       "      <td>20.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25694</th>\n",
       "      <td>51</td>\n",
       "      <td>63057</td>\n",
       "      <td>13</td>\n",
       "      <td>13911065</td>\n",
       "      <td>8.538462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25356</th>\n",
       "      <td>51</td>\n",
       "      <td>19562</td>\n",
       "      <td>13</td>\n",
       "      <td>13911065</td>\n",
       "      <td>24.153847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32099</th>\n",
       "      <td>51</td>\n",
       "      <td>3562687</td>\n",
       "      <td>12</td>\n",
       "      <td>13911065</td>\n",
       "      <td>27.416666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35209</th>\n",
       "      <td>51</td>\n",
       "      <td>709</td>\n",
       "      <td>11</td>\n",
       "      <td>13911065</td>\n",
       "      <td>7.727273</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  product_id  cnt  last_order_id  line_item_pos\n",
       "10830       51       14863   16       13911065      20.500000\n",
       "25694       51       63057   13       13911065       8.538462\n",
       "25356       51       19562   13       13911065      24.153847\n",
       "32099       51     3562687   12       13911065      27.416666\n",
       "35209       51         709   11       13911065       7.727273"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_product_freqs = user_product_freqs.merge(last_order_id, on=['user_id', 'product_id'])\n",
    "del last_order_id\n",
    "user_product_freqs = user_product_freqs.merge(user_product_line_pos_mean, on=['user_id', 'product_id'])\n",
    "del user_product_line_pos_mean\n",
    "\n",
    "user_product_freqs = user_product_freqs.sort_values(['user_id', 'cnt', 'last_order_id', 'line_item_pos'], ascending=[True, False, False, True])\n",
    "user_product_freqs = user_product_freqs.astype({'user_id': 'int32', 'product_id': 'int32', 'cnt': 'int32', 'last_order_id': 'int32', 'line_item_pos': 'float32'})\n",
    "user_product_freqs.head()\n",
    "\n",
    "# user_product_freqs = user_product_freqs.sort_values(['user_id', 'last_order_id', 'cnt', 'line_item_pos'], ascending=[True, False, False, True])\n",
    "# user_product_freqs = user_product_freqs.astype({'user_id': 'int32', 'product_id': 'int32', 'cnt': 'int32', 'last_order_id': 'int32', 'line_item_pos': 'float32'})\n",
    "# user_product_freqs.head()\n",
    "\n",
    "# user_product_freqs['score'] = user_product_freqs['cnt'] / (1 + np.log1p(user_product_freqs['line_item_pos']))\n",
    "# user_product_freqs = user_product_freqs.sort_values(['user_id', 'score', 'last_order_id'], ascending=[True, False, False,])\n",
    "# user_product_freqs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>cnt</th>\n",
       "      <th>last_order_id</th>\n",
       "      <th>line_item_pos</th>\n",
       "      <th>num_orders_total</th>\n",
       "      <th>ratio_orders</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>14863</td>\n",
       "      <td>16</td>\n",
       "      <td>13911065</td>\n",
       "      <td>20.500000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.941176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>63057</td>\n",
       "      <td>13</td>\n",
       "      <td>13911065</td>\n",
       "      <td>8.538462</td>\n",
       "      <td>17</td>\n",
       "      <td>0.764706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>19562</td>\n",
       "      <td>13</td>\n",
       "      <td>13911065</td>\n",
       "      <td>24.153847</td>\n",
       "      <td>17</td>\n",
       "      <td>0.764706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51</td>\n",
       "      <td>3562687</td>\n",
       "      <td>12</td>\n",
       "      <td>13911065</td>\n",
       "      <td>27.416666</td>\n",
       "      <td>17</td>\n",
       "      <td>0.705882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>709</td>\n",
       "      <td>11</td>\n",
       "      <td>13911065</td>\n",
       "      <td>7.727273</td>\n",
       "      <td>17</td>\n",
       "      <td>0.647059</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  product_id  cnt  last_order_id  line_item_pos  num_orders_total  \\\n",
       "0       51       14863   16       13911065      20.500000                17   \n",
       "1       51       63057   13       13911065       8.538462                17   \n",
       "2       51       19562   13       13911065      24.153847                17   \n",
       "3       51     3562687   12       13911065      27.416666                17   \n",
       "4       51         709   11       13911065       7.727273                17   \n",
       "\n",
       "   ratio_orders  \n",
       "0      0.941176  \n",
       "1      0.764706  \n",
       "2      0.764706  \n",
       "3      0.705882  \n",
       "4      0.647059  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_num_orders = products_subm.groupby(['user_id', 'order_id']).count().reset_index().groupby('user_id')['order_id'].count().reset_index()\n",
    "user_num_orders = user_num_orders.rename(columns={'order_id': 'num_orders_total'})\n",
    "\n",
    "user_product_freqs = user_product_freqs.merge(user_num_orders, on='user_id')\n",
    "user_product_freqs['ratio_orders'] = user_product_freqs['cnt'] / user_product_freqs['num_orders_total']\n",
    "user_product_freqs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>cnt</th>\n",
       "      <th>last_order_id</th>\n",
       "      <th>line_item_pos</th>\n",
       "      <th>num_orders_total</th>\n",
       "      <th>ratio_orders</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>14863</td>\n",
       "      <td>16</td>\n",
       "      <td>13911065</td>\n",
       "      <td>20.500000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.941176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>63057</td>\n",
       "      <td>13</td>\n",
       "      <td>13911065</td>\n",
       "      <td>8.538462</td>\n",
       "      <td>17</td>\n",
       "      <td>0.764706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>19562</td>\n",
       "      <td>13</td>\n",
       "      <td>13911065</td>\n",
       "      <td>24.153847</td>\n",
       "      <td>17</td>\n",
       "      <td>0.764706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51</td>\n",
       "      <td>3562687</td>\n",
       "      <td>12</td>\n",
       "      <td>13911065</td>\n",
       "      <td>27.416666</td>\n",
       "      <td>17</td>\n",
       "      <td>0.705882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>709</td>\n",
       "      <td>11</td>\n",
       "      <td>13911065</td>\n",
       "      <td>7.727273</td>\n",
       "      <td>17</td>\n",
       "      <td>0.647059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>51</td>\n",
       "      <td>5567</td>\n",
       "      <td>1</td>\n",
       "      <td>5324015</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>51</td>\n",
       "      <td>38597</td>\n",
       "      <td>1</td>\n",
       "      <td>5324015</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>51</td>\n",
       "      <td>5481691</td>\n",
       "      <td>1</td>\n",
       "      <td>5324015</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>51</td>\n",
       "      <td>21874</td>\n",
       "      <td>1</td>\n",
       "      <td>5324015</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>51</td>\n",
       "      <td>3562684</td>\n",
       "      <td>1</td>\n",
       "      <td>5324015</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>17</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>501 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  product_id  cnt  last_order_id  line_item_pos  num_orders_total  \\\n",
       "0         51       14863   16       13911065      20.500000                17   \n",
       "1         51       63057   13       13911065       8.538462                17   \n",
       "2         51       19562   13       13911065      24.153847                17   \n",
       "3         51     3562687   12       13911065      27.416666                17   \n",
       "4         51         709   11       13911065       7.727273                17   \n",
       "..       ...         ...  ...            ...            ...               ...   \n",
       "496       51        5567    1        5324015      60.000000                17   \n",
       "497       51       38597    1        5324015      61.000000                17   \n",
       "498       51     5481691    1        5324015      62.000000                17   \n",
       "499       51       21874    1        5324015      64.000000                17   \n",
       "500       51     3562684    1        5324015      65.000000                17   \n",
       "\n",
       "     ratio_orders  \n",
       "0        0.941176  \n",
       "1        0.764706  \n",
       "2        0.764706  \n",
       "3        0.705882  \n",
       "4        0.647059  \n",
       "..            ...  \n",
       "496      0.058824  \n",
       "497      0.058824  \n",
       "498      0.058824  \n",
       "499      0.058824  \n",
       "500      0.058824  \n",
       "\n",
       "[501 rows x 7 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_product_freqs[user_product_freqs.user_id == 51]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### User top products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>[14863, 63057, 19562, 3562687, 709, 9979, 3817...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65</td>\n",
       "      <td>[54728, 18450, 3817542, 709, 18439, 26062, 105...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>187</td>\n",
       "      <td>[88084, 88086]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>400</td>\n",
       "      <td>[94333, 3817507, 0, 813, 6565232, 61053, 42032...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>576</td>\n",
       "      <td>[177, 2600, 46469, 5046703, 102, 7103983, 1400...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                                         product_id\n",
       "0       51  [14863, 63057, 19562, 3562687, 709, 9979, 3817...\n",
       "1       65  [54728, 18450, 3817542, 709, 18439, 26062, 105...\n",
       "2      187                                     [88084, 88086]\n",
       "3      400  [94333, 3817507, 0, 813, 6565232, 61053, 42032...\n",
       "4      576  [177, 2600, 46469, 5046703, 102, 7103983, 1400..."
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_top_products = user_product_freqs.groupby('user_id')['product_id'].apply(list).reset_index()\n",
    "# user_top_products = user_product_freqs[user_product_freqs.ratio_orders >= 0.1].groupby('user_id')['product_id'].apply(list).reset_index()\n",
    "# user_top_products = user_product_freqs[user_product_freqs.product_id != 0].groupby('user_id')['product_id'].apply(list).reset_index()\n",
    "user_top_products.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>14863 63057 19562 3562687 709 9979 3817489 456...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65</td>\n",
       "      <td>54728 18450 3817542 709 18439 26062 105764 771...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>83</td>\n",
       "      <td>26589 74993 38831 4593463 63102 198543 38837 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>142</td>\n",
       "      <td>5051530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>187</td>\n",
       "      <td>88084 88086</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                                         product_id\n",
       "0       51  14863 63057 19562 3562687 709 9979 3817489 456...\n",
       "1       65  54728 18450 3817542 709 18439 26062 105764 771...\n",
       "2       83  26589 74993 38831 4593463 63102 198543 38837 5...\n",
       "3      142                                            5051530\n",
       "4      187                                        88084 88086"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# k_products = 50\n",
    "# user_top_products.product_id = user_top_products.product_id.map(lambda x: ' '.join(map(str, x[:k_products])))\n",
    "# user_top_products.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user_top_products.to_csv('/kaggle/working/user_top_products.csv', index=False)\n",
    "# user_top_products = pd.read_csv('/kaggle/working/user_top_products.csv')\n",
    "# user_top_products.product_id = a.product_id.apply(lambda x: x.split(' '))\n",
    "# user_top_products.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import coo_matrix, csr_matrix\n",
    "import os\n",
    "import implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids = sorted(products.user_id.unique())\n",
    "product_ids = sorted(products.product_id.unique())\n",
    "mappings = {\n",
    "    'user2idx': {user_id: i for i, user_id in enumerate(user_ids)},\n",
    "    'idx2user': {i: user_id for i, user_id in enumerate(user_ids)},\n",
    "    'product2idx': {product_id: i for i, product_id in enumerate(product_ids)},\n",
    "    'idx2product': {i: product_id for i, product_id in enumerate(product_ids)},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>num_orders</th>\n",
       "      <th>num_orders_total</th>\n",
       "      <th>ratio_orders</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>0.235294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>159</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>397</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0.117647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51</td>\n",
       "      <td>407</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>0.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>456</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>0.529412</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  product_id  num_orders  num_orders_total  ratio_orders\n",
       "0       51           0           4                17      0.235294\n",
       "1       51         159           1                17      0.058824\n",
       "2       51         397           2                17      0.117647\n",
       "3       51         407           6                17      0.352941\n",
       "4       51         456           9                17      0.529412"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_num_orders = products.groupby(['user_id', 'order_id']).count().reset_index().groupby('user_id')['order_id'].count().reset_index()\n",
    "user_num_orders = user_num_orders.rename(columns={'order_id': 'num_orders_total'})\n",
    "\n",
    "user_num_products = products.groupby(['user_id', 'product_id'])['order_id'].count().reset_index()\n",
    "user_num_products = user_num_products.rename(columns={'order_id': 'num_orders'})\n",
    "\n",
    "user_product_counts = user_num_products.merge(user_num_orders, on='user_id')\n",
    "user_product_counts['ratio_orders'] = user_product_counts['num_orders'] / user_product_counts['num_orders_total']\n",
    "user_product_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<91994x657480 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 46184547 stored elements in Compressed Sparse Column format>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_idxs = user_product_counts.user_id.map(mappings['user2idx']).values\n",
    "product_idxs = user_product_counts.product_id.map(mappings['product2idx']).values\n",
    "values = user_product_counts.ratio_orders.values\n",
    "\n",
    "user_item = csr_matrix((values, (user_idxs, product_idxs)))\n",
    "del user_idxs, product_idxs, values\n",
    "\n",
    "item_user = user_item.T\n",
    "\n",
    "data_conf = (item_user * 1).astype('double')\n",
    "data_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<91994x657480 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 46184547 stored elements in Compressed Sparse Column format>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# user_idxs = products.user_id.map(mappings['user2idx'])\n",
    "# product_idxs = products.product_id.map(mappings['product2idx'])\n",
    "# user_item = coo_matrix((np.ones_like(user_idxs), (user_idxs, product_idxs))).tocsr()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "179b7a1bf0cb4b48a1811061859dea97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=91994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# model = implicit.als.AlternatingLeastSquares(\n",
    "#     factors=64, regularization=0.1, iterations=10,\n",
    "#     calculate_training_loss=True, random_state=0\n",
    "# )\n",
    "\n",
    "# model = implicit.nearest_neighbours.ItemItemRecommender(K=10)\n",
    "\n",
    "model = implicit.nearest_neighbours.CosineRecommender(K=10)\n",
    "model.fit(data_conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(194, 1.6546121920487693),\n",
       " (33742, 1.0053961422450421),\n",
       " (3925, 0.9552985187757539),\n",
       " (32932, 0.859725834876824),\n",
       " (5298, 0.7740475975074037),\n",
       " (13203, 0.7647058823529421),\n",
       " (10589, 0.7646196089377536),\n",
       " (0, 0.7549764714008546),\n",
       " (180, 0.7545409117857674),\n",
       " (2625, 0.6439530968353565)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.recommend(0, data_conf, N=10, filter_already_liked_items=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions #1\n",
    "\n",
    "`Top Products + Model recommendations to fill K=50`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107068/107068 [02:16<00:00, 783.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "35054"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "k = 50\n",
    "ne_cnt = 0\n",
    "predictions = [None] * len(subm_ids)\n",
    "for i, user_id in enumerate(tqdm(subm_ids)):\n",
    "    user_tp = user_top_products[user_top_products.user_id == user_id].product_id.values\n",
    "    items = user_tp[0][:k] if len(user_tp) else []\n",
    "    if len(items) < k:\n",
    "        items_idx = list(map(lambda x: mappings['product2idx'][x], items))\n",
    "        user_idx = mappings['user2idx'][user_id]\n",
    "        items_a = model.recommend(user_idx, data_conf.T, k - len(items), \n",
    "                                  filter_items=items_idx, filter_already_liked_items=False)\n",
    "        items_a = list(map(lambda x: mappings['idx2product'][x[0]], items_a))\n",
    "        items += items_a\n",
    "        ne_cnt += 1   \n",
    "        \n",
    "    predictions[i] = (user_id, items)\n",
    "\n",
    "ne_cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, preds = zip(*predictions)\n",
    "preds = list(map(lambda x: ' '.join(map(str, x)), preds))\n",
    "test_submission = pd.DataFrame({'Id': subm_ids, 'Predicted': preds})\n",
    "# test_submission\n",
    "test_submission.to_csv('/kaggle/working/submission_top_sort3cdp_cosi2i_ratios.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions #2: \n",
    "\n",
    "`Top Products in Top-100 Model recommendations + Model recommendationsto fill K=50`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107068/107068 [03:49<00:00, 467.10it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "37454"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "k = 50\n",
    "k_rec = 100\n",
    "ne_cnt = 0\n",
    "predictions = [None] * len(subm_ids)\n",
    "for i, user_id in enumerate(tqdm(subm_ids)):\n",
    "    user_tp = user_top_products[user_top_products.user_id == user_id].product_id.values\n",
    "    items = user_tp[0][:k_rec] if len(user_tp) else []\n",
    "    \n",
    "    if len(items) > 0:\n",
    "        user_idx = mappings['user2idx'][user_id]\n",
    "        items_a = model.recommend(user_idx, data_conf.T, k_rec, filter_items=[0], filter_already_liked_items=False)      \n",
    "        items_a = list(map(lambda x: mappings['idx2product'][x[0]], items_a))\n",
    "        items_a_set = set(items_a)\n",
    "        items = [x for x in items if x in items_a_set][:k]\n",
    "    \n",
    "    if len(items) < k:\n",
    "        items_set = set(items)\n",
    "        items_a = [x for x in items_a if x not in items_set][:k - len(items)]\n",
    "        items += items_a\n",
    "        ne_cnt += 1   \n",
    "        \n",
    "    predictions[i] = (user_id, items)\n",
    "\n",
    "ne_cnt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}