Content-based Learning to Rank (inspired by Youtube RecSys).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%aimport utils, Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import os\n",
    "import time\n",
    "from math import ceil\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from utils import load_embeddings_and_ids, concatenate_featmats, User,\\\n",
    "        VisualSimilarityHandler, VisualSimilarityHandler_ContentAndStyle, get_decaying_learning_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use a single GPU because we want to be nice with other people :)\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Load pre-trained image embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "resnet50 = load_embeddings_and_ids('/mnt/workspace/Ugallery/ResNet50/', 'flatten_1.npy', 'ids')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "grammat_b1c1 = load_embeddings_and_ids('/mnt/workspace/Ugallery/gram_matrix/', 'block1_conv1.npy', 'ids')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Concatenate embeddings + z-score normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_list = [\n",
    "    resnet50,\n",
    "    grammat_b1c1,\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13297"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "artwork_ids_set = set()\n",
    "for embedding in embedding_list:\n",
    "    artwork_ids_set.update(embedding['index2id'])\n",
    "artwork_ids = list(artwork_ids_set)\n",
    "artwork_id2index = {_id:i for i,_id in enumerate(artwork_ids)}\n",
    "n_artworks = len(artwork_ids)\n",
    "n_artworks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "featmat_list = [tmp['featmat'] for tmp in embedding_list]\n",
    "id2index_list = [tmp['id2index'] for tmp in embedding_list]\n",
    "concat_featmat = concatenate_featmats(artwork_ids, featmat_list, id2index_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_featmat = StandardScaler().fit_transform(concat_featmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13297, 6144)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_featmat.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Load clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_clusters(json_path):\n",
    "    with open(json_path) as f:\n",
    "        artId2clustId = json.load(f)\n",
    "    cluster_ids = np.full((n_artworks,), -1, dtype=int)\n",
    "    for k, v in artId2clustId.items():\n",
    "        cluster_ids[artwork_id2index[int(k)]] = v\n",
    "    return cluster_ids, artId2clustId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_art_indexes_per_cluster(cluster_ids, n_clusters):\n",
    "    clusterId2artworkIndexes = [[] for _ in range(n_clusters)]\n",
    "    for i, cluster_id in enumerate(cluster_ids):\n",
    "        clusterId2artworkIndexes[cluster_id].append(i)\n",
    "    return clusterId2artworkIndexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('/mnt/workspace/Ugallery/Clustering/artworkId2clusterId(resnet50+alexnet+inceptionv3+vgg19+incepresv2).json') as f:\n",
    "\n",
    "content_cluster_ids, artId2clustId_content = load_clusters(\n",
    "    '/mnt/workspace/Ugallery/Clustering/artworkId2clusterId(resnet50).json')\n",
    "\n",
    "style_cluster_ids, artId2clustId_style = load_clusters(\n",
    "    '/mnt/workspace/Ugallery/Clustering/artworkId2clusterId(grammat_b1c1).json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 99 (13297,)\n",
      "0 99 (13297,)\n"
     ]
    }
   ],
   "source": [
    "print(content_cluster_ids.min(), content_cluster_ids.max(), content_cluster_ids.shape)\n",
    "print(style_cluster_ids.min(), style_cluster_ids.max(), style_cluster_ids.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_content_clusters = len(set(content_cluster_ids))\n",
    "n_content_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_style_clusters = len(set(style_cluster_ids))\n",
    "n_style_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "clustId2artIndexes_content = get_art_indexes_per_cluster(content_cluster_ids, n_content_clusters)\n",
    "clustId2artIndexes_style = get_art_indexes_per_cluster(style_cluster_ids, n_style_clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content_cluster_ids[0], style_cluster_ids[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Load PCA200 embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca200 = load_embeddings_and_ids(\n",
    "#     '/mnt/workspace/Ugallery/PCA200(resnet50+alexnet+inceptionv3+vgg19+incepresv2)/',\n",
    "    '/mnt/workspace/Ugallery/PCA200(resnet50)/',\n",
    "    'embeddings.npy',\n",
    "    'ids.npy',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca200_embeddings = pca200['featmat']\n",
    "pca200_index2id = pca200['index2id']\n",
    "pca200_id2index = pca200['id2index']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13297, 200)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca200_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.array_equal(artwork_ids, pca200_index2id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Load transactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sales_df = pd.read_csv('./data/valid_sales.csv')\n",
    "artworks_df = pd.read_csv('./data/valid_artworks.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "artist_ids = np.full((n_artworks,), -1, dtype=int)\n",
    "for _artworkId, _artistId in zip(artworks_df.id, artworks_df.artist_id):\n",
    "    i = artwork_id2index[_artworkId]\n",
    "    artist_ids[i] = _artistId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "artistId2artworkIndexes = dict()\n",
    "for i, _artistId in enumerate(artist_ids):\n",
    "    if _artistId == -1:\n",
    "        continue\n",
    "    try:\n",
    "        artistId2artworkIndexes[_artistId].append(i)\n",
    "    except KeyError:\n",
    "        artistId2artworkIndexes[_artistId] = [i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collect transactions per user (making sure we hide the last nonfirst purchase basket per user)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### create list of users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2919"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ids = sales_df.customer_id.unique()\n",
    "user_id2index = { _id:i for i,_id in enumerate(user_ids) }\n",
    "users = [User(uid) for uid in user_ids]\n",
    "n_users = len(user_ids)\n",
    "n_users"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### collect and sanity check transactions per user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_sales_df = sales_df.sort_values('order_date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clear structures to prevent possible duplicate elements\n",
    "for user in users:\n",
    "    user.clear()\n",
    "\n",
    "# collect transactions per user sorted by timestamp\n",
    "for uid, aid, t in zip(sorted_sales_df.customer_id,\n",
    "                       sorted_sales_df.artwork_id,\n",
    "                       sorted_sales_df.order_date):\n",
    "    users[user_id2index[uid]].append_transaction(\n",
    "        aid, t, artwork_id2index, artist_ids, content_cluster_ids, style_cluster_ids)\n",
    "    assert users[user_id2index[uid]]._uid == uid\n",
    "    \n",
    "# bin transctions with same timestamps into purchase baskets\n",
    "for user in users:\n",
    "    user.build_purchase_baskets()\n",
    "    user.sanity_check_purchase_baskets()\n",
    "    user.remove_last_nonfirst_purchase_basket(\n",
    "        artwork_id2index, artist_ids, content_cluster_ids, style_cluster_ids)\n",
    "    user.sanity_check_purchase_baskets()\n",
    "    user.refresh_nonpurchased_cluster_ids(n_content_clusters, n_style_clusters)\n",
    "    user.refresh_cluster_ids()\n",
    "    user.refresh_artist_ids()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "_MOD = 402653189\n",
    "_BASE = 92821\n",
    "def hash_triple(profile, pi, ni):\n",
    "    h = 0\n",
    "    for x in profile:\n",
    "        h = ((h * _BASE) % _MOD + x) % _MOD\n",
    "    h = ((h * _BASE) % _MOD + pi) % _MOD\n",
    "    h = ((h * _BASE) % _MOD + ni) % _MOD\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanity_check_instance(instance,\n",
    "                          pos_in_profile=True,\n",
    "                          \n",
    "                          pos_sharing_content_cluster=None,\n",
    "                          pos_sharing_style_cluster=None,\n",
    "                          pos_sharing_any_cluster=None,\n",
    "                          pos_sharing_both_clusters=None,\n",
    "                          pos_sharing_artist=None,\n",
    "                          \n",
    "                          neg_sharing_artist=None,                          \n",
    "                          neg_sharing_content_cluster=None,\n",
    "                          neg_sharing_style_cluster=None,\n",
    "                          neg_sharing_any_cluster=None,\n",
    "                          \n",
    "                          profile_set=None,\n",
    "                          content_clusters_set=None,\n",
    "                          style_clusters_set=None,\n",
    "                          artists_set=None,                          \n",
    "                         ):\n",
    "    profile, pi, ni, ui = instance\n",
    "    try:\n",
    "        assert 0 <= pi < n_artworks\n",
    "        assert 0 <= ni < n_artworks\n",
    "        assert pi != ni        \n",
    "        assert not vissimhandler.same(pi,ni)\n",
    "        if ui == -1: return\n",
    "        \n",
    "        assert 0 <= ui < n_users\n",
    "        user = users[ui]\n",
    "        \n",
    "        assert all(i in user.artwork_idxs_set for i in profile)\n",
    "        \n",
    "        user_profile = user.artwork_idxs_set if profile_set is None else profile_set\n",
    "        \n",
    "        # neg not in profile            \n",
    "        assert ni not in user_profile\n",
    "        \n",
    "        # pos in profile\n",
    "        if pos_in_profile is not None:\n",
    "            assert (pi in user_profile) == pos_in_profile\n",
    "            \n",
    "        # pos sharing content cluster\n",
    "        if pos_sharing_content_cluster is not None:\n",
    "            assert (content_cluster_ids[pi] in content_clusters_set) == pos_sharing_content_cluster\n",
    "            \n",
    "        # pos sharing style cluster\n",
    "        if pos_sharing_style_cluster is not None:\n",
    "            assert (style_cluster_ids[pi] in style_clusters_set) == pos_sharing_style_cluster\n",
    "            \n",
    "        # pos sharing any cluster\n",
    "        if pos_sharing_any_cluster is not None:\n",
    "            assert ((content_cluster_ids[pi] in content_clusters_set) or\n",
    "                    (style_cluster_ids[pi] in style_clusters_set)) == pos_sharing_any_cluster\n",
    "        \n",
    "        # pos sharing both clusters\n",
    "        if pos_sharing_both_clusters is not None:\n",
    "            assert ((content_cluster_ids[pi] in content_clusters_set) and\n",
    "                    (style_cluster_ids[pi] in style_clusters_set)) == pos_sharing_both_clusters\n",
    "            \n",
    "        # pos sharing artist\n",
    "        if pos_sharing_artist is not None:\n",
    "            assert (artist_ids[pi] in artists_set) == pos_sharing_artist\n",
    "        \n",
    "        \n",
    "        # neg sharing artist\n",
    "        if neg_sharing_artist is not None:\n",
    "            assert (artist_ids[ni] in artists_set) == neg_sharing_artist\n",
    "            \n",
    "        # neg sharing content cluster\n",
    "        if neg_sharing_content_cluster is not None:\n",
    "            assert (content_cluster_ids[ni] in content_clusters_set) == neg_sharing_content_cluster\n",
    "            \n",
    "        # neg sharing style cluster\n",
    "        if neg_sharing_style_cluster is not None:\n",
    "            assert (style_cluster_ids[ni] in style_clusters_set) == neg_sharing_style_cluster\n",
    "            \n",
    "        # neg sharing any cluster\n",
    "        if neg_sharing_any_cluster is not None:\n",
    "            assert ((content_cluster_ids[ni] in content_clusters_set) or\n",
    "                    (style_cluster_ids[ni] in style_clusters_set)) == neg_sharing_any_cluster\n",
    "\n",
    "    except AssertionError:\n",
    "        print('profile = ', profile)\n",
    "        print('pi = ', pi)\n",
    "        print('ni = ', ni)\n",
    "        print('ui = ', ui)\n",
    "        raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_instance(container, instance, **kwargs):\n",
    "    global _hash_collisions\n",
    "    profile, pi, ni, ui = instance\n",
    "    \n",
    "    h = hash_triple(profile, pi, ni)\n",
    "    if h in used_hashes:\n",
    "        _hash_collisions += 1\n",
    "        return False\n",
    "    \n",
    "    if vissimhandler.same(pi, ni):\n",
    "        return False\n",
    "    \n",
    "    sanity_check_instance(instance, **kwargs)\n",
    "    container.append(instance)\n",
    "    used_hashes.add(h)\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_triple(t):\n",
    "    profile, pi, ni, ui = t\n",
    "    print ('profile = ', [artwork_ids[i] for i in profile])\n",
    "    print ('pi = ', artwork_ids[pi])\n",
    "    print ('ni = ', artwork_ids[ni])\n",
    "    print ('ui = ', user_ids[ui] if ui != -1 else -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_num_samples(sampler_func):\n",
    "    def wrapper(instances_container, n_samples):        \n",
    "        while True:\n",
    "            len_before = len(instances_container)\n",
    "            sampler_func(instances_container, n_samples)\n",
    "            actual_samples = len(instances_container) - len_before\n",
    "            delta = n_samples - actual_samples\n",
    "            print('  target samples: %d' % n_samples)\n",
    "            print('  actual samples: %d' % actual_samples)\n",
    "            print('  delta: %d' % (delta))\n",
    "            if delta <= 0: break\n",
    "            print('  ** delta > 0 -> sampling more instances again ...')\n",
    "            n_samples = delta\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vissimhandler = VisualSimilarityHandler_ContentAndStyle(\n",
    "#     content_cluster_ids, style_cluster_ids, pca200_embeddings)\n",
    "vissimhandler = VisualSimilarityHandler(content_cluster_ids, pca200_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "vissimhandler.count = 0\n",
    "used_hashes = set()\n",
    "_hash_collisions = 0\n",
    "train_instances = []\n",
    "test_instances = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000000 250000\n",
      "0 0\n"
     ]
    }
   ],
   "source": [
    "N_STRATEGIES_FAKE = 2\n",
    "N_STRATEGIES_REAL = 4\n",
    "FAKE_COEF = 1.\n",
    "TOTAL_SAMPLES__TRAIN = 10000000\n",
    "TOTAL_SAMPLES__TEST =  TOTAL_SAMPLES__TRAIN * 0.05\n",
    "\n",
    "N_SAMPLES_PER_FAKE_STRATEGY__TRAIN = ceil(TOTAL_SAMPLES__TRAIN * FAKE_COEF / N_STRATEGIES_FAKE)\n",
    "N_SAMPLES_PER_FAKE_STRATEGY__TEST = ceil(TOTAL_SAMPLES__TEST * FAKE_COEF / N_STRATEGIES_FAKE)\n",
    "N_SAMPLES_PER_REAL_STRATEGY__TRAIN = ceil(TOTAL_SAMPLES__TRAIN * (1. - FAKE_COEF) / N_STRATEGIES_REAL)\n",
    "N_SAMPLES_PER_REAL_STRATEGY__TEST = ceil(TOTAL_SAMPLES__TEST * (1. - FAKE_COEF) / N_STRATEGIES_REAL)\n",
    "\n",
    "print(N_SAMPLES_PER_FAKE_STRATEGY__TRAIN, N_SAMPLES_PER_FAKE_STRATEGY__TEST)\n",
    "print(N_SAMPLES_PER_REAL_STRATEGY__TRAIN, N_SAMPLES_PER_REAL_STRATEGY__TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "FINE_GRAINED_THRESHOLD = 0.7\n",
    "VISUAL_CONFIDENCE_THRESHOLD = 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Original BPR strategy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1) given profile, recommend profile (real users)\n",
    "Given a user's profile, all items in the profile should be ranked higher than items outside the profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index(pi):\n",
    "    if random.random() <= FINE_GRAINED_THRESHOLD:\n",
    "        if random.randint(0, 1) == 0:\n",
    "            clustId2artIndexes = clustId2artIndexes_content\n",
    "            cluster_ids = content_cluster_ids\n",
    "        else:\n",
    "            clustId2artIndexes = clustId2artIndexes_style\n",
    "            cluster_ids = style_cluster_ids\n",
    "        ni = random.choice(clustId2artIndexes[cluster_ids[pi]])\n",
    "    else:\n",
    "        if random.randint(0, 1) == 0:\n",
    "            clustId2artIndexes = clustId2artIndexes_content\n",
    "            n_clusters = n_content_clusters\n",
    "        else:\n",
    "            clustId2artIndexes = clustId2artIndexes_style\n",
    "            n_clusters = n_style_clusters\n",
    "        c = random.randint(0, n_clusters-1)\n",
    "        ni = random.choice(clustId2artIndexes[c])\n",
    "    return ni"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index__outsideprofile(profile_set, pi):\n",
    "    while True:\n",
    "        ni = sample_artwork_index(pi)\n",
    "        if ni not in profile_set:\n",
    "            return ni"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "@print_num_samples\n",
    "def generate_samples__rank_profile_above_nonprofile(instances_container, n_samples):\n",
    "    n_samples_per_user = ceil(n_samples / n_users)    \n",
    "    for ui, user in enumerate(users):\n",
    "        profile = user.artwork_idxs\n",
    "        profile_set = user.artwork_idxs_set\n",
    "        for _ in range(n_samples_per_user):\n",
    "            for __ in range(5):\n",
    "                pi = random.choice(profile)\n",
    "                ni = sample_artwork_index__outsideprofile(profile_set, pi)\n",
    "                if append_instance(instances_container, (profile, pi, ni, ui),\n",
    "                                   pos_in_profile=True,\n",
    "                                   profile_set=profile_set):\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# print('=======================================\\nsampling train instances ...')\n",
    "# generate_samples__rank_profile_above_nonprofile(\n",
    "#     train_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TRAIN)\n",
    "\n",
    "# print('=======================================\\nsampling test instances ...')\n",
    "# generate_samples__rank_profile_above_nonprofile(\n",
    "#     test_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TEST)\n",
    "\n",
    "# print(len(train_instances), len(test_instances))\n",
    "# print('hash_collisions = ', _hash_collisions)\n",
    "# print('visual_collisions = ', vissimhandler.count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2) Given profile, recommend profile (fake 1-item profiles)\n",
    "Given a fake profile of a single item, such item should be ranked higher than any other item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index__nonidentical(pi):        \n",
    "    while True:\n",
    "        ni = sample_artwork_index(pi)\n",
    "        if ni != pi:\n",
    "            return ni"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "@print_num_samples\n",
    "def generate_samples__rank_single_item_above_anything_else(instances_container, n_samples):\n",
    "    n_samples_per_item = ceil(n_samples / n_artworks)\n",
    "    for pi in range(n_artworks):\n",
    "        profile = (pi,)\n",
    "        n = n_samples_per_item\n",
    "        while n > 0:\n",
    "            ni = sample_artwork_index__nonidentical(pi)\n",
    "            if append_instance(instances_container, (profile, pi, ni, -1)):\n",
    "                n -= 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=======================================\n",
      "sampling train instances ...\n",
      "  target samples: 5000000\n",
      "  actual samples: 5012969\n",
      "  delta: -12969\n",
      "=======================================\n",
      "sampling test instances ...\n",
      "  target samples: 250000\n",
      "  actual samples: 252643\n",
      "  delta: -2643\n",
      "5012969 252643\n",
      "hash_collisions =  2317206\n",
      "visual_collisions =  13127\n"
     ]
    }
   ],
   "source": [
    "print('=======================================\\nsampling train instances ...')\n",
    "generate_samples__rank_single_item_above_anything_else(\n",
    "    train_instances, n_samples=N_SAMPLES_PER_FAKE_STRATEGY__TRAIN)\n",
    "\n",
    "print('=======================================\\nsampling test instances ...')\n",
    "generate_samples__rank_single_item_above_anything_else(\n",
    "    test_instances, n_samples=N_SAMPLES_PER_FAKE_STRATEGY__TEST)\n",
    "\n",
    "print(len(train_instances), len(test_instances))\n",
    "print('hash_collisions = ', _hash_collisions)\n",
    "print('visual_collisions = ', vissimhandler.count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Domain-specific strategies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 3) Recommend visually similar items from favorite artists (real users)\n",
    "Given a user, any item outside the user's profile that shares artist and visual cluster with items in the user's profile should be ranked higher than any item from an artist and visual cluster not present in the user's profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index__outsideprofile__sharing_artist_content_cluster(\n",
    "        profile_set, artists_list, content_clusters_set):\n",
    "    for _ in range(4): # try at most 4 times\n",
    "        # sharing artist\n",
    "        a = random.choice(artists_list)\n",
    "        i = random.choice(artistId2artworkIndexes[a])\n",
    "        # sharing cluster\n",
    "        if content_cluster_ids[i] not in content_clusters_set:\n",
    "            continue\n",
    "        # oustide profile\n",
    "        if i in profile_set: continue\n",
    "        # done\n",
    "        return i\n",
    "    return None # failed to find"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index__notsharing_artist_content_cluster(\n",
    "        artists_set, unused_content_clusters_list):\n",
    "    while True:\n",
    "        # not sharing content cluster\n",
    "        c = random.choice(unused_content_clusters_list)\n",
    "        i = random.choice(clustId2artIndexes_content[c])        \n",
    "        # not sharing artist\n",
    "        if artist_ids[i] in artists_set:\n",
    "            continue\n",
    "        return i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "@print_num_samples\n",
    "def generate_samples__rank_sharing_artist_content_cluster_above_notsharing_artist_content_cluster(\n",
    "        instances_container, n_samples):\n",
    "    \n",
    "    n_samples_per_user = ceil(n_samples / n_users)    \n",
    "    for ui, user in enumerate(users):\n",
    "        profile = user.artwork_idxs\n",
    "        profile_set = user.artwork_idxs_set\n",
    "        artists_list = user.artist_ids\n",
    "        artists_set = user.artist_ids_set\n",
    "        content_clusters_set = user.content_cluster_ids_set\n",
    "        unused_content_clusters_list = user.nonp_content_cluster_ids\n",
    "        for _ in range(n_samples_per_user):\n",
    "            for __ in range(5):\n",
    "                pi = sample_artwork_index__outsideprofile__sharing_artist_content_cluster(\n",
    "                    profile_set, artists_list, content_clusters_set)\n",
    "                if pi is None: continue\n",
    "                ni = sample_artwork_index__notsharing_artist_content_cluster(\n",
    "                    artists_set, unused_content_clusters_list)\n",
    "                if append_instance(instances_container, (profile, pi, ni, ui),\n",
    "                                   pos_in_profile=False,\n",
    "                                   pos_sharing_artist=True,\n",
    "                                   pos_sharing_content_cluster=True,\n",
    "                                   neg_sharing_artist=False,\n",
    "                                   neg_sharing_content_cluster=False,\n",
    "                                   profile_set = profile_set,\n",
    "                                   content_clusters_set = content_clusters_set,\n",
    "                                   artists_set = artists_set,\n",
    "                                  ):\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# print('=======================================\\nsampling train instances ...')\n",
    "# generate_samples__rank_sharing_artist_content_cluster_above_notsharing_artist_content_cluster(\n",
    "#     train_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TRAIN)\n",
    "\n",
    "# print('=======================================\\nsampling test instances ...')\n",
    "# generate_samples__rank_sharing_artist_content_cluster_above_notsharing_artist_content_cluster(\n",
    "#     test_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TEST)\n",
    "\n",
    "# print(len(train_instances), len(test_instances))\n",
    "# print('hash_collisions = ', _hash_collisions)\n",
    "# print('visual_collisions = ', vissimhandler.count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 4) Recommend visual similar items from favorite artists (fake 1-item profiles)\n",
    "Given a fake profile of a single item, other items sharing same artist should be ranked higher than items from different artists as long as the PCA200 embedding agrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index__nonidentical_sharing_artist(i):\n",
    "    a = artist_ids[i]\n",
    "    assert a != -1\n",
    "    candidate_idxs = artistId2artworkIndexes[a]\n",
    "    assert len(candidate_idxs) >= 2\n",
    "    while True:\n",
    "        pi = random.choice(candidate_idxs) # sharing artist\n",
    "        if pi != i: # non-identical\n",
    "            return pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_artwork_index__notsharing_artist__visually_acceptable(i, pi):\n",
    "    for _ in range(4): # try at most 4 times\n",
    "        ni = sample_artwork_index(i)\n",
    "        if artist_ids[ni] == artist_ids[i]: # not sharing artist\n",
    "            continue\n",
    "        if vissimhandler.validate_triple(i, pi, ni, margin=VISUAL_CONFIDENCE_THRESHOLD): # visually acceptable\n",
    "            return ni\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "@print_num_samples\n",
    "def generate_samples__rank_sharing_artist_above_notsharing_artist__visuallyacceptable__single_item(\n",
    "        instances_container, n_samples):\n",
    "    \n",
    "    n_valid_items = sum(1 for i in range(n_artworks) if artist_ids[i] != -1 and\\\n",
    "                        len(artistId2artworkIndexes[artist_ids[i]]) >= 2)\n",
    "    n_samples_per_item = ceil(n_samples / n_valid_items)\n",
    "    \n",
    "    \n",
    "    print('(debug) n_valid_items = ', n_valid_items)\n",
    "    print('(debug) n_samples_per_item = ', n_samples_per_item)\n",
    "    count = 0\n",
    "    for i in range(n_artworks):\n",
    "        a = artist_ids[i]\n",
    "        if a == -1 or len(artistId2artworkIndexes[a]) < 2:\n",
    "            continue\n",
    "        count += 1\n",
    "        if (count == n_valid_items or count % 100 == 0):\n",
    "            print('(debug) %d/%d' % (count, n_valid_items), flush=True, end='\\r')\n",
    "        profile = (i,)\n",
    "        for _ in range(n_samples_per_item):\n",
    "            for __ in range(5):\n",
    "                pi = sample_artwork_index__nonidentical_sharing_artist(i)\n",
    "                ni = sample_artwork_index__notsharing_artist__visually_acceptable(i, pi)\n",
    "                if ni is None:\n",
    "                    continue\n",
    "                if append_instance(instances_container, (profile, pi, ni, -1)):\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=======================================\n",
      "sampling train instances ...\n",
      "(debug) n_valid_items =  7587\n",
      "(debug) n_samples_per_item =  660\n",
      "  target samples: 5000000\n",
      "  actual samples: 4939496\n",
      "  delta: 60504\n",
      "  ** delta > 0 -> sampling more instances again ...\n",
      "(debug) n_valid_items =  7587\n",
      "(debug) n_samples_per_item =  8\n",
      "  target samples: 60504\n",
      "  actual samples: 59582\n",
      "  delta: 922\n",
      "  ** delta > 0 -> sampling more instances again ...\n",
      "(debug) n_valid_items =  7587\n",
      "(debug) n_samples_per_item =  1\n",
      "  target samples: 922\n",
      "  actual samples: 7447\n",
      "  delta: -6525\n",
      "=======================================\n",
      "sampling test instances ...\n",
      "(debug) n_valid_items =  7587\n",
      "(debug) n_samples_per_item =  33\n",
      "  target samples: 250000\n",
      "  actual samples: 245653\n",
      "  delta: 4347\n",
      "  ** delta > 0 -> sampling more instances again ...\n",
      "(debug) n_valid_items =  7587\n",
      "(debug) n_samples_per_item =  1\n",
      "  target samples: 4347\n",
      "  actual samples: 7457\n",
      "  delta: -3110\n",
      "10019494 505753\n",
      "hash_collisions =  2834378\n",
      "visual_collisions =  13127\n"
     ]
    }
   ],
   "source": [
    "print('=======================================\\nsampling train instances ...')\n",
    "generate_samples__rank_sharing_artist_above_notsharing_artist__visuallyacceptable__single_item(\n",
    "    train_instances, n_samples=N_SAMPLES_PER_FAKE_STRATEGY__TRAIN)\n",
    "\n",
    "print('=======================================\\nsampling test instances ...')\n",
    "generate_samples__rank_sharing_artist_above_notsharing_artist__visuallyacceptable__single_item(\n",
    "    test_instances, n_samples=N_SAMPLES_PER_FAKE_STRATEGY__TEST)\n",
    "\n",
    "print(len(train_instances), len(test_instances))\n",
    "print('hash_collisions = ', _hash_collisions)\n",
    "print('visual_collisions = ', vissimhandler.count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 5) Predict next purchase basket\n",
    "Given all previous purchases, rank each  item of the next purchase basket higher than any item from a never purchased artist and cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "@print_num_samples\n",
    "def generate_samples__given_past_rank_next(instances_container, n_samples):\n",
    "    \n",
    "    n_valid_users = sum(1 for user in users if len(user.baskets) >= 2) # at last 2 purchase baskets\n",
    "    n_samples_per_user = ceil(n_samples / n_valid_users)\n",
    "    \n",
    "    for ui, user in enumerate(users):\n",
    "        n = len(user.baskets)\n",
    "        if n <= 1:\n",
    "            continue\n",
    "        past_items = []        \n",
    "        n_samples_per_basket = ceil(n_samples_per_user / (n-1))\n",
    "        for bi in range(n-1):\n",
    "            cur_b = user.baskets[bi]\n",
    "            for j in range(cur_b[0], cur_b[0] + cur_b[1]):\n",
    "                past_items.append(user.artwork_idxs[j])\n",
    "            next_b  = user.baskets[bi+1]\n",
    "            profile = past_items.copy()\n",
    "            for _ in range(n_samples_per_basket):\n",
    "                for __ in range(5):\n",
    "                    pi = user.artwork_idxs[random.randint(next_b[0], next_b[0] + next_b[1] - 1)]\n",
    "                    ni = sample_artwork_index__notsharing_artist_content_cluster(\n",
    "                        user.artist_ids_set, user.nonp_content_cluster_ids)\n",
    "                    if append_instance(instances_container, (profile, pi, ni, ui),\n",
    "                                       neg_sharing_artist=False,\n",
    "                                       neg_sharing_content_cluster=False,\n",
    "                                       artists_set=user.artist_ids_set,\n",
    "                                       content_clusters_set=user.content_cluster_ids_set,\n",
    "                                      ):\n",
    "                        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=======================================\n",
      "sampling train instances ...\n",
      "  target samples: 250000\n",
      "  actual samples: 250450\n",
      "  delta: -450\n",
      "=======================================\n",
      "sampling test instances ...\n",
      "  target samples: 12500\n",
      "  actual samples: 13136\n",
      "  delta: -636\n",
      "9762710 499482\n",
      "hash_collisions =  2050355\n",
      "visual_collisions =  9325\n"
     ]
    }
   ],
   "source": [
    "print('=======================================\\nsampling train instances ...')\n",
    "generate_samples__given_past_rank_next(\n",
    "    train_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TRAIN)\n",
    "\n",
    "print('=======================================\\nsampling test instances ...')\n",
    "generate_samples__given_past_rank_next(\n",
    "    test_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TEST)\n",
    "\n",
    "print(len(train_instances), len(test_instances))\n",
    "print('hash_collisions = ', _hash_collisions)\n",
    "print('visual_collisions = ', vissimhandler.count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 6) Predict hidden item in the k-th purchase basket given first k\n",
    "Given the first k purchase baskets of a user, hide one item in the k-th purchase basket, use the rest as profile and rank the hidden item higher than any item from a never purchased artist and cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "@print_num_samples\n",
    "def generate_samples__hide_and_predict_one_from_last__first_k_purchase_baskets(instances_container, n_samples):\n",
    "    \n",
    "    n_valid_baskets_list = [sum(1 for b in user.baskets if b[1] >= 2) for user in users]\n",
    "    n_valid_users = sum(1 for x in n_valid_baskets_list if x > 0)\n",
    "    n_samples_per_user = ceil(n_samples / n_valid_users)\n",
    "    \n",
    "    for ui, (user, n_valid_baskets) in enumerate(zip(users, n_valid_baskets_list)):\n",
    "        if n_valid_baskets == 0:\n",
    "            continue\n",
    "        n_samples_per_basket = ceil(n_samples_per_user / n_valid_baskets)\n",
    "        u_artwork_idxs = user.artwork_idxs\n",
    "        purchased = []\n",
    "        for b in user.baskets:            \n",
    "            bs = b[0]\n",
    "            be = b[0] + b[1]\n",
    "            purchased.extend(u_artwork_idxs[j] for j in range(bs, be))\n",
    "            assert len(purchased) == be\n",
    "            if b[1] < 2:\n",
    "                continue            \n",
    "            n_samples_per_item = ceil(n_samples_per_basket / b[1])            \n",
    "            for i in range(bs, be):\n",
    "                profile = [purchased[j] for j in range(be) if j != i]\n",
    "                assert len(profile) == be - 1\n",
    "                assert len(profile) > 0\n",
    "                pi = purchased[i]\n",
    "                for _ in range(n_samples_per_item):\n",
    "                    for __ in range(5):\n",
    "                        ni = sample_artwork_index__notsharing_artist_content_cluster(\n",
    "                            user.artist_ids_set, user.nonp_content_cluster_ids)\n",
    "                        if append_instance(instances_container, (profile, pi, ni, ui),\n",
    "                                           neg_sharing_artist=False,\n",
    "                                           neg_sharing_content_cluster=False,\n",
    "                                           artists_set=user.artist_ids_set,\n",
    "                                           content_clusters_set=user.content_cluster_ids_set,\n",
    "                                          ):\n",
    "                            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=======================================\n",
      "sampling train instances ...\n",
      "  target samples: 250000\n",
      "  actual samples: 250914\n",
      "  delta: -914\n",
      "=======================================\n",
      "sampling test instances ...\n",
      "  target samples: 12500\n",
      "  actual samples: 13705\n",
      "  delta: -1205\n",
      "10013624 513187\n",
      "hash_collisions =  2060634\n",
      "visual_collisions =  9325\n"
     ]
    }
   ],
   "source": [
    "print('=======================================\\nsampling train instances ...')\n",
    "generate_samples__hide_and_predict_one_from_last__first_k_purchase_baskets(\n",
    "    train_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TRAIN)\n",
    "\n",
    "print('=======================================\\nsampling test instances ...')\n",
    "generate_samples__hide_and_predict_one_from_last__first_k_purchase_baskets(\n",
    "    test_instances, n_samples=N_SAMPLES_PER_REAL_STRATEGY__TEST)\n",
    "\n",
    "print(len(train_instances), len(test_instances))\n",
    "print('hash_collisions = ', _hash_collisions)\n",
    "print('visual_collisions = ', vissimhandler.count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### sort train and test instances by profile size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(train_instances)\n",
    "train_instances.sort(key=lambda x: len(x[0]))\n",
    "test_instances.sort(key=lambda x: len(x[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_minibatches(tuples, max_users_items_per_batch):\n",
    "    ui_count = 0\n",
    "    offset = 0\n",
    "    \n",
    "    batch_ranges = []\n",
    "    for i, t in enumerate(tuples):\n",
    "        ui_count += len(t[0]) + 3\n",
    "        if ui_count > max_users_items_per_batch:\n",
    "            batch_ranges.append((offset, i))\n",
    "            ui_count = len(t[0]) + 3\n",
    "            offset = i\n",
    "            assert ui_count <= max_users_items_per_batch\n",
    "    assert offset < len(tuples)\n",
    "    batch_ranges.append((offset, len(tuples)))\n",
    "            \n",
    "    n_tuples = len(tuples)\n",
    "    n_batches = len(batch_ranges)\n",
    "    print('n_tuples = ', n_tuples)\n",
    "    print('n_batches = ', n_batches)\n",
    "    \n",
    "    assert batch_ranges[0][0] == 0\n",
    "    assert all(batch_ranges[i][1] == batch_ranges[i+1][0] for i in range(n_batches-1))\n",
    "    assert batch_ranges[-1][1] == n_tuples\n",
    "    assert sum(b[1] - b[0] for b in batch_ranges) == n_tuples\n",
    "    \n",
    "    profile_indexes_batches = [None] * n_batches\n",
    "    profile_size_batches = [None] * n_batches\n",
    "    positive_index_batches = [None] * n_batches\n",
    "    negative_index_batches = [None] * n_batches\n",
    "    \n",
    "    for i, (jmin, jmax) in enumerate(batch_ranges):\n",
    "        actual_batch_size = jmax - jmin\n",
    "        profile_maxlen = max(len(tuples[j][0]) for j in range(jmin, jmax))\n",
    "        profile_indexes_batch = np.full((actual_batch_size, profile_maxlen), 0, dtype=int)\n",
    "        profile_size_batch = np.empty((actual_batch_size,))\n",
    "        positive_index_batch = np.empty((actual_batch_size,), dtype=int)\n",
    "        negative_index_batch = np.empty((actual_batch_size,), dtype=int)\n",
    "        \n",
    "        for j in range(actual_batch_size):\n",
    "            # profile indexes\n",
    "            for k,v in enumerate(tuples[jmin+j][0]):\n",
    "                profile_indexes_batch[j][k] = v\n",
    "            # profile size\n",
    "            profile_size_batch[j] = len(tuples[jmin+j][0])        \n",
    "            # positive index\n",
    "            positive_index_batch[j] = tuples[jmin+j][1]\n",
    "            # negative index\n",
    "            negative_index_batch[j] = tuples[jmin+j][2]\n",
    "            \n",
    "        profile_indexes_batches[i] = profile_indexes_batch\n",
    "        profile_size_batches[i] = profile_size_batch\n",
    "        positive_index_batches[i] = positive_index_batch\n",
    "        negative_index_batches[i] = negative_index_batch\n",
    "        \n",
    "    return dict(\n",
    "        profile_indexes_batches = profile_indexes_batches,\n",
    "        profile_size_batches    = profile_size_batches,\n",
    "        positive_index_batches  = positive_index_batches,\n",
    "        negative_index_batches  = negative_index_batches,\n",
    "        n_batches               = n_batches,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanity_check_minibatches(minibatches):\n",
    "    profile_indexes_batches = minibatches['profile_indexes_batches']\n",
    "    profile_size_batches = minibatches['profile_size_batches']\n",
    "    positive_index_batches = minibatches['positive_index_batches']\n",
    "    negative_index_batches = minibatches['negative_index_batches']\n",
    "    n_batches = minibatches['n_batches']\n",
    "    assert n_batches == len(profile_indexes_batches)\n",
    "    assert n_batches == len(profile_size_batches)\n",
    "    assert n_batches == len(positive_index_batches)\n",
    "    assert n_batches == len(negative_index_batches)\n",
    "    assert n_batches > 0\n",
    "    \n",
    "    for profile_indexes, profile_size, positive_index, negative_index in zip(\n",
    "        profile_indexes_batches,\n",
    "        profile_size_batches,\n",
    "        positive_index_batches,\n",
    "        negative_index_batches\n",
    "    ):\n",
    "        n = profile_size.shape[0]\n",
    "        assert n == profile_indexes.shape[0]\n",
    "        assert n == positive_index.shape[0]\n",
    "        assert n == negative_index.shape[0]\n",
    "        \n",
    "        for i in range(n):\n",
    "            assert positive_index[i] != negative_index[i]\n",
    "            psz = int(profile_size[i])\n",
    "            m = profile_indexes[i].shape[0]\n",
    "            assert psz <= m\n",
    "            for j in range(psz, m):\n",
    "                assert profile_indexes[i][j] == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from Networks import ContentBasedLearn2RankNetwork_Train, TrainLogger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# # DEBUGGING NETWORK\n",
    "# with tf.Graph().as_default():\n",
    "#     network = ContentBasedLearn2RankNetwork_Train(\n",
    "#         pretrained_embedding_dim=concat_featmat.shape[1],\n",
    "#         user_layer_units=[300, 300, 200],\n",
    "#         item_layer_units=[200, 200],\n",
    "#         weight_decay=0.0001,\n",
    "#         profile_pooling_mode='AVG+MAX',\n",
    "#     )\n",
    "#     with tf.Session() as sess:\n",
    "#         sess.run(tf.global_variables_initializer())\n",
    "#         debug_ = sess.run(\n",
    "#             [network._profile_masks__avgpool,\n",
    "#              network._profile_masks__maxpool,\n",
    "#              network._masked_profile_item_embeddings__avgpool,\n",
    "#              network._masked_profile_item_embeddings__maxpool,\n",
    "#              network._profile_items_avgpool,\n",
    "#              network._profile_items_maxpool,\n",
    "#             ], feed_dict={\n",
    "#                 network._pretrained_embeddings: concat_featmat,\n",
    "#                 network._profile_item_indexes: [[0,1,2,9,9,9], [1,2,3,4,5,6]],\n",
    "#                 network._profile_sizes: [3,6],\n",
    "#             }\n",
    "#         )\n",
    "#         for x in tf.global_variables():\n",
    "#             print(x)        \n",
    "# #         for op in sess.graph.get_operations():\n",
    "# #             print(op.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def train_network(train_minibatches, test_minibatches,\n",
    "                  n_train_instances, n_test_instances, batch_size,\n",
    "                  pretrained_embeddings,\n",
    "                  user_layer_units,\n",
    "                  item_layer_units,\n",
    "                  profile_pooling_mode,\n",
    "                  model_path,\n",
    "                  max_seconds_training=3600,\n",
    "                  min_seconds_to_check_improvement=60,\n",
    "                  early_stopping_checks=4,\n",
    "                  weight_decay=0.001,\n",
    "                  learning_rates=[1e-3]):\n",
    "    \n",
    "    n_train_batches = train_minibatches['n_batches']\n",
    "    n_test_batches = test_minibatches['n_batches']\n",
    "    \n",
    "    print('learning_rates = ', learning_rates)\n",
    "    \n",
    "    with tf.Graph().as_default():\n",
    "        network = ContentBasedLearn2RankNetwork_Train(\n",
    "            pretrained_embedding_dim=pretrained_embeddings.shape[1],\n",
    "            user_layer_units=user_layer_units,\n",
    "            item_layer_units=item_layer_units,\n",
    "            weight_decay=weight_decay,\n",
    "            profile_pooling_mode=profile_pooling_mode,\n",
    "        )\n",
    "        \n",
    "        print('Variables to be trained:')\n",
    "        for x in tf.global_variables():\n",
    "            print('\\t', x)            \n",
    "        \n",
    "        gpu_options = tf.GPUOptions(\n",
    "            per_process_gpu_memory_fraction=0.99,\n",
    "            allow_growth=True\n",
    "        )\n",
    "        config = tf.ConfigProto(gpu_options=gpu_options)\n",
    "        with tf.Session(config=config) as sess:\n",
    "            try:\n",
    "                saver = tf.train.Saver()            \n",
    "                saver.restore(sess, tf.train.latest_checkpoint(model_path))\n",
    "                print('model successfully restored from checkpoint!')\n",
    "            except ValueError:\n",
    "                print('no checkpoint found: initializing variables with random values')\n",
    "                os.makedirs(model_path, exist_ok=True)\n",
    "                sess.run(tf.global_variables_initializer())            \n",
    "            trainlogger = TrainLogger(model_path + 'train_logs.csv')\n",
    "\n",
    "            # ========= BEFORE TRAINING ============\n",
    "            \n",
    "            initial_test_acc = 0.\n",
    "            for profile_indexes, profile_size, positive_index, negative_index in zip(\n",
    "                test_minibatches['profile_indexes_batches'],\n",
    "                test_minibatches['profile_size_batches'],\n",
    "                test_minibatches['positive_index_batches'],\n",
    "                test_minibatches['negative_index_batches']\n",
    "            ):\n",
    "                minibatch_test_acc = network.get_test_accuracy(\n",
    "                    sess, pretrained_embeddings, profile_indexes, profile_size, positive_index, negative_index)\n",
    "                initial_test_acc += minibatch_test_acc\n",
    "            initial_test_acc /= n_test_instances\n",
    "\n",
    "            print(\"Before training: test_accuracy = %f\" % initial_test_acc)\n",
    "            \n",
    "            best_test_acc = initial_test_acc\n",
    "            seconds_training = 0\n",
    "            elapsed_seconds_from_last_check = 0\n",
    "            checks_with_no_improvement = 0\n",
    "            last_improvement_loss = None\n",
    "            \n",
    "            # ========= TRAINING ============\n",
    "            \n",
    "            print ('Starting training ...')\n",
    "            n_lr = len(learning_rates)\n",
    "            lr_i = 0\n",
    "            train_loss_ema = 0. # exponential moving average\n",
    "            \n",
    "            while seconds_training < max_seconds_training:\n",
    "                \n",
    "                for train_i, (profile_indexes, profile_size, positive_index, negative_index) in enumerate(zip(\n",
    "                    train_minibatches['profile_indexes_batches'],\n",
    "                    train_minibatches['profile_size_batches'],\n",
    "                    train_minibatches['positive_index_batches'],\n",
    "                    train_minibatches['negative_index_batches']\n",
    "                )):\n",
    "                    # optimize and get traing loss\n",
    "                    start_t = time.time()\n",
    "                    _, minibatch_train_loss = network.optimize_and_get_train_loss(\n",
    "                        sess, learning_rates[lr_i], pretrained_embeddings, profile_indexes,\n",
    "                        profile_size, positive_index, negative_index)\n",
    "                    delta_t = time.time() - start_t\n",
    "                    \n",
    "                    # update train loss exponential moving average\n",
    "                    train_loss_ema = 0.999 * train_loss_ema + 0.001 * minibatch_train_loss\n",
    "                    \n",
    "                    # update time tracking variables\n",
    "                    seconds_training += delta_t\n",
    "                    elapsed_seconds_from_last_check += delta_t\n",
    "                    \n",
    "                    # check for improvements using test set if it's time to do so\n",
    "                    if elapsed_seconds_from_last_check >= min_seconds_to_check_improvement:\n",
    "                        \n",
    "                        # --- testing\n",
    "                        test_acc = 0.\n",
    "                        for _profile_indexes, _profile_size, _positive_index, _negative_index in zip(\n",
    "                            test_minibatches['profile_indexes_batches'],\n",
    "                            test_minibatches['profile_size_batches'],\n",
    "                            test_minibatches['positive_index_batches'],\n",
    "                            test_minibatches['negative_index_batches']\n",
    "                        ):\n",
    "                            minibatch_test_acc = network.get_test_accuracy(\n",
    "                                sess, pretrained_embeddings, _profile_indexes,\n",
    "                                _profile_size, _positive_index, _negative_index)                            \n",
    "                            test_acc += minibatch_test_acc\n",
    "                        test_acc /= n_test_instances\n",
    "                    \n",
    "                        print((\"train_i=%d, train_loss = %.12f, test_accuracy = %.7f,\"\n",
    "                               \" check_secs = %.2f, total_secs = %.2f\") % (\n",
    "                                train_i, train_loss_ema, test_acc, elapsed_seconds_from_last_check, seconds_training))                        \n",
    "                        \n",
    "                        # check for improvements\n",
    "                        if (test_acc > best_test_acc) or (\n",
    "                            test_acc == best_test_acc and (\n",
    "                                last_improvement_loss is not None and\\\n",
    "                                last_improvement_loss > train_loss_ema\n",
    "                            )\n",
    "                        ):  \n",
    "                            last_improvement_loss = train_loss_ema\n",
    "                            best_test_acc = test_acc\n",
    "                            checks_with_no_improvement = 0\n",
    "                            saver = tf.train.Saver()\n",
    "                            save_path = saver.save(sess, model_path)                    \n",
    "                            print(\"   ** improvement detected: model saved to path \", save_path)\n",
    "                            model_updated = True\n",
    "                        else:\n",
    "                            checks_with_no_improvement += 1                            \n",
    "                            model_updated = False\n",
    "\n",
    "                        # --- logging ---                        \n",
    "                        trainlogger.log_update(\n",
    "                            train_loss_ema, test_acc, n_train_instances, n_test_instances,\n",
    "                            elapsed_seconds_from_last_check, batch_size, learning_rates[lr_i], 't' if model_updated else 'f')\n",
    "                        \n",
    "                        # --- check for early stopping\n",
    "                        if checks_with_no_improvement >= early_stopping_checks:\n",
    "                            if lr_i + 1 < len(learning_rates):\n",
    "                                lr_i += 1\n",
    "                                checks_with_no_improvement = 0\n",
    "                                print(\"   *** %d checks with no improvements -> using a smaller learning_rate = %.8f\" % (\n",
    "                                    early_stopping_checks, learning_rates[lr_i]))\n",
    "                            else:\n",
    "                                print(\"   *** %d checks with no improvements -> early stopping :(\" % early_stopping_checks)\n",
    "                                return\n",
    "                        \n",
    "                        # --- reset check variables\n",
    "                        elapsed_seconds_from_last_check = 0\n",
    "            print('====== TIMEOUT ======')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_tuples =  10019494\n",
      "n_batches =  802\n"
     ]
    }
   ],
   "source": [
    "train_minibatches = generate_minibatches(train_instances, max_users_items_per_batch=5000*10)\n",
    "sanity_check_minibatches(train_minibatches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_tuples =  505753\n",
      "n_batches =  41\n"
     ]
    }
   ],
   "source": [
    "test_minibatches = generate_minibatches(test_instances, max_users_items_per_batch=5000*10)\n",
    "sanity_check_minibatches(test_minibatches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0001,\n",
       " 6e-05,\n",
       " 3.6e-05,\n",
       " 2.16e-05,\n",
       " 1.296e-05,\n",
       " 7.776e-06,\n",
       " 4.6656e-06,\n",
       " 2.79936e-06,\n",
       " 1.679616e-06,\n",
       " 1.0077696e-06]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learning_rates = get_decaying_learning_rates(1e-4, 1e-6, 0.6)\n",
    "learning_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7, 0.1)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FINE_GRAINED_THRESHOLD, VISUAL_CONFIDENCE_THRESHOLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls /mnt/workspace/pamessina_models/ugallery/youtube_like/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MODEL_PATH = ('/mnt/workspace/pamessina_models/ugallery/youtube_like/v9(10M-300K,avg+max,'\n",
    "#              '+prof(rl&fk)-nprof,+nproffavc(rl&fk)-nfavc_loc,+nxt+hide1+hideacc-nfavc_glob)/')\n",
    "# MODEL_PATH = ('/mnt/workspace/pamessina_models/ugallery/youtube_like/v16c(10M-400K,avg+max,rsnt50+alxnt+incepv3,'\n",
    "#               'u(300,300,200)i(200,200),+p(rl&fk)-np,+npfavc(rl&fk)-nfavc,+nxt+hdaccklst-nfavc,fg.6,vcf.1)/')\n",
    "# MODEL_PATH = ('/mnt/workspace/pamessina_models/ugallery/youtube_like/v26(10M-500K,avg+max,rsnt50+grmmtb1c1,'\n",
    "#               'u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.6,vcf.1,wd.0001)/')\n",
    "MODEL_PATH = ('/mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,'\n",
    "              'u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/')\n",
    "MODEL_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12494"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_train_batch_size = ceil(np.mean([b.shape[0] for b in train_minibatches['profile_indexes_batches']]))\n",
    "avg_train_batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning_rates =  [0.0001, 6e-05, 3.6e-05, 2.16e-05, 1.296e-05, 7.776e-06, 4.6656e-06, 2.79936e-06, 1.679616e-06, 1.0077696e-06]\n",
      "Variables to be trained:\n",
      "\t <tf.Variable 'trainable_item_embedding/fc1/kernel:0' shape=(6144, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc1/bias:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc2/kernel:0' shape=(200, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc2/bias:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_1/kernel:0' shape=(400, 300) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_1/bias:0' shape=(300,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_2/kernel:0' shape=(300, 300) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_2/bias:0' shape=(300,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_vector/kernel:0' shape=(300, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_vector/bias:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'beta1_power:0' shape=() dtype=float32_ref>\n",
      "\t <tf.Variable 'beta2_power:0' shape=() dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc1/kernel/Adam:0' shape=(6144, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc1/kernel/Adam_1:0' shape=(6144, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc1/bias/Adam:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc1/bias/Adam_1:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc2/kernel/Adam:0' shape=(200, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc2/kernel/Adam_1:0' shape=(200, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc2/bias/Adam:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'trainable_item_embedding/fc2/bias/Adam_1:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_1/kernel/Adam:0' shape=(400, 300) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_1/kernel/Adam_1:0' shape=(400, 300) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_1/bias/Adam:0' shape=(300,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_1/bias/Adam_1:0' shape=(300,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_2/kernel/Adam:0' shape=(300, 300) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_2/kernel/Adam_1:0' shape=(300, 300) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_2/bias/Adam:0' shape=(300,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_hidden_2/bias/Adam_1:0' shape=(300,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_vector/kernel/Adam:0' shape=(300, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_vector/kernel/Adam_1:0' shape=(300, 200) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_vector/bias/Adam:0' shape=(200,) dtype=float32_ref>\n",
      "\t <tf.Variable 'user_vector/bias/Adam_1:0' shape=(200,) dtype=float32_ref>\n",
      "no checkpoint found: initializing variables with random values\n",
      "Before training: test_accuracy = 0.474139\n",
      "Starting training ...\n",
      "train_i=302, train_loss = 0.130368719302, test_accuracy = 0.9892457, check_secs = 90.29, total_secs = 90.29\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=605, train_loss = 0.130667459599, test_accuracy = 0.9942205, check_secs = 90.08, total_secs = 180.37\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=106, train_loss = 0.123789076292, test_accuracy = 0.9959368, check_secs = 90.11, total_secs = 270.48\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=409, train_loss = 0.115452769839, test_accuracy = 0.9968502, check_secs = 90.05, total_secs = 360.53\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=712, train_loss = 0.107796353720, test_accuracy = 0.9974118, check_secs = 90.11, total_secs = 450.64\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=214, train_loss = 0.101098874602, test_accuracy = 0.9978191, check_secs = 90.20, total_secs = 540.83\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=517, train_loss = 0.095576651797, test_accuracy = 0.9979338, check_secs = 90.19, total_secs = 631.02\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=17, train_loss = 0.091039804614, test_accuracy = 0.9982185, check_secs = 90.01, total_secs = 721.03\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=320, train_loss = 0.087221291262, test_accuracy = 0.9983609, check_secs = 90.24, total_secs = 811.27\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=622, train_loss = 0.084269859254, test_accuracy = 0.9985546, check_secs = 90.16, total_secs = 901.43\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=122, train_loss = 0.081757530295, test_accuracy = 0.9986555, check_secs = 90.08, total_secs = 991.51\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=424, train_loss = 0.079770495662, test_accuracy = 0.9986317, check_secs = 90.22, total_secs = 1081.72\n",
      "train_i=726, train_loss = 0.078198477188, test_accuracy = 0.9987029, check_secs = 90.27, total_secs = 1172.00\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=226, train_loss = 0.076774375339, test_accuracy = 0.9987899, check_secs = 90.25, total_secs = 1262.25\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=528, train_loss = 0.075581085827, test_accuracy = 0.9988196, check_secs = 90.17, total_secs = 1352.42\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=24, train_loss = 0.074368385111, test_accuracy = 0.9987622, check_secs = 90.30, total_secs = 1442.71\n",
      "train_i=322, train_loss = 0.073339052645, test_accuracy = 0.9988038, check_secs = 90.28, total_secs = 1533.00\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00006000\n",
      "train_i=619, train_loss = 0.071667501900, test_accuracy = 0.9993020, check_secs = 90.01, total_secs = 1623.01\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_i=114, train_loss = 0.070105076726, test_accuracy = 0.9993594, check_secs = 90.04, total_secs = 1713.05\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=412, train_loss = 0.068724140850, test_accuracy = 0.9993930, check_secs = 90.11, total_secs = 1803.16\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=711, train_loss = 0.067543312405, test_accuracy = 0.9993772, check_secs = 90.25, total_secs = 1893.41\n",
      "train_i=206, train_loss = 0.066512797893, test_accuracy = 0.9993475, check_secs = 90.16, total_secs = 1983.56\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00003600\n",
      "train_i=504, train_loss = 0.065469533414, test_accuracy = 0.9995294, check_secs = 90.09, total_secs = 2073.66\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=1, train_loss = 0.064497181640, test_accuracy = 0.9995690, check_secs = 90.14, total_secs = 2163.80\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=299, train_loss = 0.063616030452, test_accuracy = 0.9995274, check_secs = 90.20, total_secs = 2254.00\n",
      "train_i=597, train_loss = 0.062812976847, test_accuracy = 0.9995077, check_secs = 90.25, total_secs = 2344.25\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00002160\n",
      "train_i=93, train_loss = 0.062034801113, test_accuracy = 0.9995868, check_secs = 90.03, total_secs = 2434.28\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=391, train_loss = 0.061336099289, test_accuracy = 0.9996125, check_secs = 90.11, total_secs = 2524.39\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=690, train_loss = 0.060663873826, test_accuracy = 0.9996085, check_secs = 90.30, total_secs = 2614.69\n",
      "train_i=187, train_loss = 0.060009013103, test_accuracy = 0.9996362, check_secs = 90.27, total_secs = 2704.96\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=485, train_loss = 0.059354026044, test_accuracy = 0.9996105, check_secs = 90.01, total_secs = 2794.96\n",
      "train_i=784, train_loss = 0.058702467393, test_accuracy = 0.9995690, check_secs = 90.16, total_secs = 2885.13\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00001296\n",
      "train_i=280, train_loss = 0.058041406137, test_accuracy = 0.9996303, check_secs = 90.08, total_secs = 2975.21\n",
      "train_i=577, train_loss = 0.057409444959, test_accuracy = 0.9996184, check_secs = 90.28, total_secs = 3065.49\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00000778\n",
      "train_i=70, train_loss = 0.056825623411, test_accuracy = 0.9996382, check_secs = 90.03, total_secs = 3155.52\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=367, train_loss = 0.056287374444, test_accuracy = 0.9996619, check_secs = 90.13, total_secs = 3245.64\n",
      "   ** improvement detected: model saved to path  /mnt/workspace/pamessina_models/ugallery/youtube_like/v30(10M-500K(fk=1.),avg+max,rsnt50+grmmtb1c1,u(300,300,200)i(200,200),+p(fk)-np,+npfavc(fk)-nfavc,fg.7,vcf.1,wd.0001)/\n",
      "train_i=664, train_loss = 0.055775570036, test_accuracy = 0.9996560, check_secs = 90.05, total_secs = 3335.70\n",
      "train_i=158, train_loss = 0.055271611541, test_accuracy = 0.9996599, check_secs = 90.17, total_secs = 3425.86\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00000467\n",
      "train_i=455, train_loss = 0.054783362144, test_accuracy = 0.9996579, check_secs = 90.14, total_secs = 3516.01\n",
      "train_i=752, train_loss = 0.054324016110, test_accuracy = 0.9996540, check_secs = 90.03, total_secs = 3606.04\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00000280\n",
      "train_i=247, train_loss = 0.053894651171, test_accuracy = 0.9996520, check_secs = 90.06, total_secs = 3696.09\n",
      "train_i=544, train_loss = 0.053499858985, test_accuracy = 0.9996560, check_secs = 90.00, total_secs = 3786.10\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00000168\n",
      "train_i=39, train_loss = 0.053138452611, test_accuracy = 0.9996520, check_secs = 90.11, total_secs = 3876.21\n",
      "train_i=336, train_loss = 0.052809902842, test_accuracy = 0.9996560, check_secs = 90.12, total_secs = 3966.33\n",
      "   *** 2 checks with no improvements -> using a smaller learning_rate = 0.00000101\n",
      "train_i=633, train_loss = 0.052513453007, test_accuracy = 0.9996560, check_secs = 90.07, total_secs = 4056.40\n",
      "train_i=129, train_loss = 0.052246068346, test_accuracy = 0.9996540, check_secs = 90.13, total_secs = 4146.53\n",
      "   *** 2 checks with no improvements -> early stopping :(\n"
     ]
    }
   ],
   "source": [
    "train_network(\n",
    "    train_minibatches, test_minibatches,\n",
    "    len(train_instances), len(test_instances),\n",
    "    batch_size=avg_train_batch_size,\n",
    "    pretrained_embeddings=concat_featmat,\n",
    "    user_layer_units=[300,300,200],\n",
    "    item_layer_units=[200,200],\n",
    "    profile_pooling_mode='AVG+MAX',\n",
    "    model_path = MODEL_PATH,\n",
    "    max_seconds_training=3600 * 5,\n",
    "    min_seconds_to_check_improvement=90,\n",
    "    early_stopping_checks=2,\n",
    "    weight_decay=.0001,\n",
    "    learning_rates=learning_rates,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": ".venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}