choice-aware-topicmodel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "bHGihPrKMAJa",
    "outputId": "b35381df-a180-4808-ca0c-995ea6c0afdb"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.cuda.is_available()\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "#device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nejGa1C50IrM"
   },
   "outputs": [],
   "source": [
    "\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "pd.options.mode.chained_assignment = None\n",
    "from sklearn.metrics import confusion_matrix, recall_score, precision_score, accuracy_score, f1_score,roc_auc_score\n",
    "from ucm_model import UniversalChoiceModel as UCM\n",
    "from ucm_model import *\n",
    "import datetime\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from torch.optim import Adam, AdamW, SparseAdam, Adamax, Adagrad\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from IPython.display import clear_output\n",
    "import math\n",
    "from sklearn.metrics import log_loss\n",
    "torch.manual_seed(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "IxbVpBuAYrCO",
    "outputId": "9008b76a-3499-4833-f2c2-4fe369a0f511"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8Wg8-7D2YrCT"
   },
   "outputs": [],
   "source": [
    "#data frame with the filtered topic labels and features that characterize each data entry:\n",
    "nt = 50\n",
    "file = open(\"gsdmm_topics.pkl\",'rb')\n",
    "df = pickle.load(file)\n",
    "file.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(font_scale=0.7)\n",
    "df['text_tokens'].value_counts().plot(kind='bar', figsize=(9, 6), rot=0)\n",
    "plt.xlabel(\"Topic labels\", labelpad=14)\n",
    "plt.ylabel(\"Number of topics\", labelpad=14)\n",
    "plt.title(\"Histogram showing frequency of topics in the dataset\", y=1.02);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tweet count for all the users considered\n",
    "df['engaging_user_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XbFWA-NBYrCc"
   },
   "outputs": [],
   "source": [
    "#Convert date to yyyy-mm-dd hh:mm:ss format\n",
    "\n",
    "df['tweet_timestamp'] = df[\"tweet_timestamp\"].apply(lambda x: datetime.datetime.fromtimestamp(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "h0qCQt8hYrCp"
   },
   "outputs": [],
   "source": [
    "t1 = df[(df[\"tweet_timestamp\"] >= '2020-02-05 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-06 06:00:00')]\n",
    "t2 = df[(df[\"tweet_timestamp\"] >= '2020-02-06 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-06 18:00:00')]\n",
    "t3 = df[(df[\"tweet_timestamp\"] >= '2020-02-06 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-07 06:00:00')]\n",
    "t4 = df[(df[\"tweet_timestamp\"] >= '2020-02-07 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-07 18:00:00')]\n",
    "t5 = df[(df[\"tweet_timestamp\"] >= '2020-02-07 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-08 06:00:00')]\n",
    "t6 = df[(df[\"tweet_timestamp\"] >= '2020-02-08 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-08 18:00:00')]\n",
    "t7 = df[(df[\"tweet_timestamp\"] >= '2020-02-08 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-09 06:00:00')]\n",
    "t8 = df[(df[\"tweet_timestamp\"] >= '2020-02-09 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-09 18:00:00')]\n",
    "t9 = df[(df[\"tweet_timestamp\"] >= '2020-02-09 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-10 06:00:00')]\n",
    "t10 = df[(df[\"tweet_timestamp\"] >= '2020-02-10 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-10 18:00:00')]\n",
    "t11 = df[(df[\"tweet_timestamp\"] >= '2020-02-10 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-11 06:00:00')]\n",
    "t12 = df[(df[\"tweet_timestamp\"] >= '2020-02-11 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-11 18:00:00')]\n",
    "t13 = df[(df[\"tweet_timestamp\"] >= '2020-02-11 18:00:00') & (df[\"tweet_timestamp\"] < '2020-02-12 06:00:00')]\n",
    "t14 = df[(df[\"tweet_timestamp\"] >= '2020-02-12 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-12 18:00:00')]\n",
    "\n",
    "# Each time period is 12 hrs\n",
    "# Engagement frequency of user is from time periods t1 to t11 during training\n",
    "# Recent history of user is considered from time periods t8 to t11 training \n",
    "# Training will be carrid out on time period t12\n",
    "# Validation carried out on time period t13\n",
    "# Testing will be carried out on time period t14\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Engagement frequency for all available timeperiod\n",
    "eng_frequency = df[df[\"tweet_timestamp\"] < '2020-02-12 18:00:00']\n",
    "eng_frequency['retweet'] = np.where(pd.notnull(eng_frequency['retweet_timestamp']), 1, 0)\n",
    "columns = ['tweet_timestamp', 'tweet_id', 'reply_timestamp', \"retweet_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"]\n",
    "eng_frequency.drop(columns, axis=1,inplace=True)\n",
    "engagement_history = eng_frequency.groupby(['engaging_user_id', 'text_tokens'])[['retweet']].agg('sum')\n",
    "input_engagement_history = engagement_history.pivot_table(index='engaging_user_id', columns='text_tokens', values='retweet')\n",
    "input_engagement_history.fillna(0,inplace = True)\n",
    "history_frequency = pd.DataFrame(input_engagement_history.to_records())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Engagement frequency for testing time\n",
    "eng_frequency_test = df[df[\"tweet_timestamp\"] < '2020-02-12 6:00:00']\n",
    "eng_frequency_test['retweet'] = np.where(pd.notnull(eng_frequency_test['retweet_timestamp']), 1, 0)\n",
    "columns = ['tweet_timestamp', 'tweet_id', 'reply_timestamp', \"retweet_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"]\n",
    "eng_frequency_test.drop(columns, axis=1,inplace=True)\n",
    "engagement_history_test = eng_frequency_test.groupby(['engaging_user_id', 'text_tokens'])[['retweet']].agg('sum')\n",
    "input_engagement_history_test = engagement_history_test.pivot_table(index='engaging_user_id', columns='text_tokens', values='retweet')\n",
    "input_engagement_history_test.fillna(0,inplace = True)\n",
    "history_frequency_test = pd.DataFrame(input_engagement_history_test.to_records())\n",
    "left_out_rows_xt = history_frequency[~history_frequency['engaging_user_id'].isin(history_frequency_test['engaging_user_id'])]\n",
    "for col in left_out_rows_xt.columns:\n",
    "    if col != 'engaging_user_id':\n",
    "        left_out_rows_xt[col].values[:] = 0\n",
    "final_history_xt = history_frequency_test.append(left_out_rows_xt)\n",
    "final_history_xt = final_history_xt.sort_values('engaging_user_id')\n",
    "history_frequency_test = final_history_xt.reset_index(drop=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Engagement frequency for training time\n",
    "eng_frequency_train = df[df[\"tweet_timestamp\"] < '2020-02-11 6:00:00']\n",
    "eng_frequency_train['retweet'] = np.where(pd.notnull(eng_frequency_train['retweet_timestamp']), 1, 0)\n",
    "columns = ['tweet_timestamp', 'tweet_id', 'reply_timestamp', \"retweet_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"]\n",
    "eng_frequency_train.drop(columns, axis=1,inplace=True)\n",
    "engagement_history_train = eng_frequency_train.groupby(['engaging_user_id', 'text_tokens'])[['retweet']].agg('sum')\n",
    "input_engagement_history_train = engagement_history_train.pivot_table(index='engaging_user_id', columns='text_tokens', values='retweet')\n",
    "input_engagement_history_train.fillna(0,inplace = True)\n",
    "history_frequency_train = pd.DataFrame(input_engagement_history_train.to_records())\n",
    "left_out_rows_xt = history_frequency[~history_frequency['engaging_user_id'].isin(history_frequency_train['engaging_user_id'])]\n",
    "for col in left_out_rows_xt.columns:\n",
    "    if col != 'engaging_user_id':\n",
    "        left_out_rows_xt[col].values[:] = 0\n",
    "final_history_xt = history_frequency_train.append(left_out_rows_xt)\n",
    "final_history_xt = final_history_xt.sort_values('engaging_user_id')\n",
    "history_frequency_train = final_history_xt.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y-UI0k4SYrCr"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "columns = ['reply_timestamp','tweet_timestamp', 'tweet_id',\"retweet_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def final_history(z):\n",
    "    xt = z\n",
    "    columns = ['reply_timestamp','tweet_timestamp', 'tweet_id',\"retweet_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"]\n",
    "    xt['retweet'] = np.where(pd.notnull(xt['retweet_timestamp']), 1, 0)\n",
    "    xt.drop(columns, axis=1)\n",
    "    x_t = xt.groupby(['engaging_user_id', 'text_tokens'])[['retweet']].agg('sum')\n",
    "    xt_history = x_t.pivot_table(index='engaging_user_id', columns='text_tokens', values='retweet')\n",
    "    xt_history.fillna(0,inplace = True)\n",
    "    history_xt = pd.DataFrame(xt_history.to_records())\n",
    "    for col in history_xt.columns:\n",
    "      if col != 'engaging_user_id':\n",
    "        history_xt.loc[history_xt[col] > 1, col] = 1\n",
    "\n",
    "    left_out_rows_xt = history_frequency[~history_frequency['engaging_user_id'].isin(history_xt['engaging_user_id'])]\n",
    "    for col in left_out_rows_xt.columns:\n",
    "        if col != 'engaging_user_id':\n",
    "            left_out_rows_xt[col].values[:] = 0\n",
    "    final_history_xt = history_xt.append(left_out_rows_xt)\n",
    "    final_history_xt = final_history_xt.sort_values('engaging_user_id')\n",
    "    final_history_xt.reset_index(drop=True)\n",
    "    final_history_xt = final_history_xt.fillna(0)\n",
    "    return final_history_xt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataframes with engagement histories  \n",
    "from functools import reduce\n",
    "recent_history = reduce(lambda x,y: pd.merge(x,y, on='engaging_user_id', how='outer'), [final_history(t8), final_history(t9), final_history(t10), final_history(t11)])\n",
    "recent_history_test = reduce(lambda x,y: pd.merge(x,y, on='engaging_user_id', how='outer'), [final_history(t10), final_history(t11), final_history(t12), final_history(t13)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PelCIh_vH8fv"
   },
   "outputs": [],
   "source": [
    "history_frequency = history_frequency.sort_values('engaging_user_id')\n",
    "history_frequency_train = history_frequency_train.sort_values('engaging_user_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweet_recommend(time):\n",
    "    #time = df[(df[\"tweet_timestamp\"] >= '2020-02-12 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-12 18:00:00')]\n",
    "    left = time[time['retweet_timestamp'] >= 0]\n",
    "    time['retweet_timestamp'].fillna(0,inplace=True)\n",
    "\n",
    "    left['retweet_timestamp'] = left[\"retweet_timestamp\"].apply(lambda x: datetime.datetime.fromtimestamp(x))\n",
    "    time = pd.concat([time,left]).drop_duplicates(keep=False)\n",
    "    time['retweet_timestamp'] = time['tweet_timestamp']\n",
    "    final = pd.concat([time,left])\n",
    "\n",
    "    final = final.sort_values(['engaging_user_id', 'retweet_timestamp'], ascending=[True, False])\n",
    "    final = final.reset_index(drop=True)\n",
    "\n",
    "    columns = ['tweet_id',\"reply_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"]\n",
    "    final.drop(columns, axis=1,inplace=True)\n",
    "    \n",
    "    final = active_recommend(final)\n",
    "    column = ['tweet_timestamp','retweet_timestamp']\n",
    "    final.drop(column, axis=1,inplace=True)\n",
    "    print (final['recommend'].value_counts())\n",
    "    final_1 = final.groupby(['engaging_user_id', 'text_tokens'])[['recommend']].agg('sum')\n",
    "    final_2 = final_1.pivot_table(index='engaging_user_id', columns='text_tokens', values='recommend')\n",
    "    final_2.fillna(0,inplace = True)\n",
    "    final_3 = pd.DataFrame(final_2.to_records())\n",
    "\n",
    "    for col in final_3.columns:\n",
    "      if col != 'engaging_user_id':\n",
    "        final_3.loc[final_3[col] > 1, col] = 1\n",
    "\n",
    "\n",
    "    left_out_rows_f = history_frequency[~history_frequency['engaging_user_id'].isin(final_3['engaging_user_id'])]\n",
    "    for col in left_out_rows_f.columns:\n",
    "        if col != 'engaging_user_id':\n",
    "            left_out_rows_f[col].values[:] = 0\n",
    "    final_4 = final_3.append(left_out_rows_f)\n",
    "    recommend_test = final_4.sort_values('engaging_user_id')\n",
    "    recommend_test.reset_index(drop=True, inplace=True)\n",
    "    return(recommend_test)\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3hYN4tkv7vCT"
   },
   "outputs": [],
   "source": [
    "# We consider the positive examples for topic recommendations from all the tweets posted during the active states and vice versa. \n",
    "# An active state is defined as a period between when a tweet is published and engaged\n",
    "\n",
    "\n",
    "#Function action_recommend for determining the active states\n",
    "def active_recommend(x):\n",
    "    g = 3\n",
    "    for index, row in x.iterrows():\n",
    "      p = 0  \n",
    "      if g != 10 :\n",
    "        a = row['engaging_user_id']\n",
    "        b = row['tweet_timestamp']\n",
    "        c = row['retweet_timestamp']\n",
    "        p = 0\n",
    "        if b != c :\n",
    "          g = 10\n",
    "      if (row['tweet_timestamp'] != row['retweet_timestamp']):\n",
    "          p = 1\n",
    "          d = row['engaging_user_id']\n",
    "          e = row['tweet_timestamp']\n",
    "          f = row['retweet_timestamp']\n",
    "      elif row['tweet_timestamp'] == row['retweet_timestamp']:\n",
    "          if row['engaging_user_id'] == a:\n",
    "            p = 0\n",
    "          elif row['engaging_user_id'] == d:\n",
    "            if (row['tweet_timestamp'] >= e) & (row['tweet_timestamp'] <= f):\n",
    "              p = 1\n",
    "            else:\n",
    "              p = 0\n",
    "      x.loc[index,'recommend'] = p\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recommend_test = tweet_recommend(df[(df[\"tweet_timestamp\"] >= '2020-02-12 06:00:00') & (df[\"tweet_timestamp\"] < '2020-02-12 18:00:00')])\n",
    "recommend = tweet_recommend(df[(df[\"tweet_timestamp\"] >= '2020-02-11 6:00:00') & (df[\"tweet_timestamp\"] < '2020-02-11 18:00:00')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_gq6BFzyYrDB"
   },
   "outputs": [],
   "source": [
    "X_recent_2 = recent_history.drop('engaging_user_id', axis=1).values\n",
    "X_recent_1 = X_recent_2.reshape(-1,nt,4)\n",
    "\n",
    "X_recent_2_test = recent_history_test.drop('engaging_user_id', axis=1).values\n",
    "X_recent_1_test = X_recent_2_test.reshape(-1,nt,4)\n",
    "\n",
    "\n",
    "\n",
    "X_freq_1 = history_frequency_test.drop('engaging_user_id', axis=1).values\n",
    "X_freq_1_train = history_frequency_train.drop('engaging_user_id', axis=1).values\n",
    "\n",
    "\n",
    "X_recommend_1 = recommend.drop('engaging_user_id', axis=1).values\n",
    "X_recommend_t = recommend_test.drop('engaging_user_id', axis=1).values\n",
    "\n",
    "y_1 = final_history(t12).drop('engaging_user_id', axis=1).values\n",
    "\n",
    "y_t = final_history(t14).drop('engaging_user_id', axis=1).values\n",
    "\n",
    "#INPUTS\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_recent = torch.FloatTensor(X_recent_1)              #Recent Engagement History tensor input for training\n",
    "X_freq = torch.FloatTensor(X_freq_1_train)            #Engagement Frequency tensor input for training\n",
    "X_recommend = torch.FloatTensor(X_recommend_1)        #Topic Recommendation tensor input for training\n",
    "y = torch.FloatTensor(y_1)                            #Engagement output tensor input for training\n",
    "\n",
    "\n",
    "X_recent_test = torch.FloatTensor(X_recent_1_test)    #Recent Engagement History tensor input for testing\n",
    "X_freq_test = torch.FloatTensor(X_freq_1)             #Engagement Frequency tensor input for testing\n",
    "X_recommend_test = torch.FloatTensor(X_recommend_t)   #Topic Recommendation tensor input for testing\n",
    "Y_test = torch.FloatTensor(y_t)                       #Engagement output tensor input for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_freq = F.normalize(X_freq, p=2, dim=1)              #Normalized Engagement Frequency tensor input for training\n",
    "X_freq_test = F.normalize(X_freq_test, p=2, dim=1)    #Normalized Engagement Frequency tensor input for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = 20\n",
    "L = 10\n",
    "class DNN_f(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        #frequency input\n",
    "        self.fc1 = nn.Linear(in_features=nt, out_features=L)\n",
    "\n",
    "        #history input 32 * 50 * 4\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = filters, kernel_size = (1,4) ,stride = 1)\n",
    "        self.leaky = nn.ReLU()\n",
    "        self.conv2 = nn.Conv2d(in_channels = filters, out_channels = filters, kernel_size = (nt-L+1,1) ,stride = 1)\n",
    "        self.conv3 = nn.ConvTranspose2d(in_channels = filters, out_channels = filters, kernel_size =(nt-L+1,1), stride =1)\n",
    "\n",
    "\n",
    "        #recommend input\n",
    "        self.fc3 = nn.Linear(in_features=nt, out_features=L)\n",
    "        \n",
    "        #final linear\n",
    "        self.conv4 = nn.Conv2d(in_channels = 1, out_channels=1, kernel_size=(1, 5+2*(filters)), stride = 1) \n",
    "        \n",
    "        self.fc4 = nn.Linear(in_features=5+2*(filters), out_features=1)\n",
    "        \n",
    "\n",
    " \n",
    "    def forward(self, x, y, z, a):\n",
    "        x1 = self.fc1(x)      ### EH\n",
    "        x2 = F.linear(x1, self.fc1.weight.t())  #### EH BAR\n",
    "\n",
    "        \n",
    "        y_ = y.view(-1,1,nt,4)\n",
    "        y1 = self.conv1(y_)\n",
    "        y1 = self.leaky(y1)\n",
    "        y2 = y1.view(-1,nt,filters)   ####   ET       \n",
    "        y1 = self.conv2(y1)\n",
    "        y1 = self.conv3(y1)\n",
    "        #y1 = self.leaky(y1)\n",
    "        y3 = y1.view(-1,nt,filters)   ####   ETBAR\n",
    "        \n",
    "        z1 = self.fc3(z)    ####  DT\n",
    "        z2 = F.linear(z1, self.fc3.weight.t())    #### DTBAR\n",
    "\n",
    "        \n",
    "        w = torch.stack((x, x2,z, z2), dim=2)\n",
    "        v = torch.cat((y2,y3),dim =2)\n",
    "        u = torch.cat((w,v),dim=2)\n",
    "        a = a.view(-1,nt,1)\n",
    "        r = torch.cat((a,u),dim = 2)\n",
    "        \n",
    "        #print(r.shape)\n",
    "        r = r.view(-1,1,nt,5+2*(filters))\n",
    "        #t = self.conv4(r)\n",
    "        #s = t.view(-1,nt)\n",
    "        t = self.fc4(r)\n",
    "        s = t.view(-1,nt)\n",
    "        #s = torch.diagonal(t, dim1=1, dim2=2)\n",
    "        #print(s.shape)\n",
    "\n",
    "        return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.ones([32, 50], dtype=torch.float64).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6M5PLdzGYrDO"
   },
   "outputs": [],
   "source": [
    "model = DNN_f()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# custom function\n",
    "def sigmoid(x):\n",
    "  return 1 / (1 + math.exp(-x))\n",
    "\n",
    "# define vectorized sigmoid\n",
    "sigmoid_v = np.vectorize(sigmoid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IYXUuJAjYrDQ"
   },
   "outputs": [],
   "source": [
    "optimizer = Adam(model.parameters(), lr=5e-3)\n",
    "loss_func = nn.BCEWithLogitsLoss(reduction = 'mean').cuda()\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "train_loss = 0\n",
    "losses = []\n",
    "steps = []\n",
    "step = 0\n",
    "count = 0\n",
    "EPOCHS = 50\n",
    "trained = []\n",
    "tested = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_indicator = torch.ones([107017, nt])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZKAPwzquMI6f"
   },
   "outputs": [],
   "source": [
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "train_dataset = TensorDataset(X_freq, X_recent, X_recommend, y, X_indicator)\n",
    "train_sampler = RandomSampler(train_dataset)\n",
    "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))\n",
    "\n",
    "\n",
    "for epoch_num in range(EPOCHS):\n",
    "    model.train()\n",
    "    for step_num, batch_data in enumerate(train_dataloader):\n",
    "        freq, recent, recommend, labels, indicator = tuple(t.to(device)for t in batch_data)\n",
    "        probas = model(freq, recent, recommend, indicator)\n",
    "        batch_loss = loss_func(probas, labels)\n",
    "        train_loss += batch_loss.item()\n",
    "\n",
    "        model.zero_grad()\n",
    "        batch_loss.backward()\n",
    "\n",
    "\n",
    "        #clip_grad_norm_(parameters=model.parameters(), max_norm=1.0)\n",
    "        optimizer.step()\n",
    "\n",
    "        clear_output(wait=True)\n",
    "        print('Epoch: ', epoch_num + 1)\n",
    "        print(\"{0}/{1} loss: {2} \".format(step_num, len(y)/BATCH_SIZE, train_loss / (count + 1)))\n",
    "        print (\"--- %s seconds ---\" % (time.time() - start_time))\n",
    "        losses.append(batch_loss.item())\n",
    "        steps.append(step)\n",
    "        step += 1\n",
    "        count += 1\n",
    "#     lg_train = model(X_freq.to(device), X_recent.to(device), X_recommend.to(device), X_indicator.to(device)).to('cpu')\n",
    "#     numpy_lg_train = lg_train.detach().numpy()\n",
    "#     prob_tr = sigmoid_v(numpy_lg_train)\n",
    "#     trained.append(log_loss(y.numpy(),prob_tr))\n",
    "    lg_test = model(X_freq_test.to(device), X_recent_test.to(device), X_recommend_test.to(device), X_indicator.to(device)).to('cpu')\n",
    "    numpy_lg_test = lg_test.detach().numpy()\n",
    "    prob_tt = sigmoid_v(numpy_lg_test)\n",
    "    tested.append(log_loss(Y_test.numpy(),prob_tt))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UeAm0QIHYrDV"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#torch.save({'state_dict': model.state_dict()}, 'checkpoint.pth.tar')\n",
    "\n",
    "#model = model()\n",
    "#checkpoint = torch.load('checkpoint.pth.tar')\n",
    "#model.load_state_dict(checkpoint['state_dict'])\n",
    "\n",
    "torch.save(model, \"model_dnn_july19_10l_20f.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = torch.load(\"model_dnn_march30_12743.pkl\")\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "\n",
    "with torch.no_grad():\n",
    "\n",
    "    logits_train = model(X_freq.to(device), X_recent.to(device), X_recommend.to(device), X_indicator.to(device)).to('cpu')\n",
    "    logits_test = model(X_freq_test.to(device), X_recent_test.to(device), X_recommend_test.to(device), X_indicator.to(device)).to('cpu')\n",
    "    numpy_logits_train = logits_train.detach().numpy()\n",
    "    numpy_logits_test = logits_test.detach().numpy()\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Converting logits to probabilities\n",
    "numpy_probas_test = sigmoid_v(numpy_logits_test)\n",
    "numpy_probas_train = sigmoid_v(numpy_logits_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "log_loss(Y_test.numpy(),numpy_probas_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Performance Metrics          \n",
    "def binary_classification_performance(y_test, y_pred):\n",
    "    tp, fp, fn, tn = confusion_matrix(y_test, y_pred).ravel()\n",
    "    accuracy = round(accuracy_score(y_pred = y_pred, y_true = y_test),2)\n",
    "    precision = round(precision_score(y_pred = y_pred, y_true = y_test),2)\n",
    "    recall = round(recall_score(y_pred = y_pred, y_true = y_test),2)\n",
    "    f1_score = round(2*precision*recall/(precision + recall),2)\n",
    "    specificity = round(tn/(tn+fp),2)\n",
    "    npv = round(tn/(tn+fn),2)\n",
    "\n",
    "\n",
    "    result = pd.DataFrame({'Accuracy' : [accuracy],\n",
    "                         'Precision (or PPV)' : [precision],\n",
    "                         'Recall (senitivity or TPR)' : [recall],\n",
    "                         'f1 score' : [f1_score],\n",
    "                         'Specificty (or TNR)': [specificity],\n",
    "                         'NPV' : [npv],\n",
    "                         'True Positive' : [tp],\n",
    "                         'True Negative' : [tn],\n",
    "                         'False Positive':[fp],\n",
    "                         'False Negative':[fn]})\n",
    "    return result\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh = []\n",
    "roc = []\n",
    "for i in range(nt):    \n",
    "    import sklearn.metrics as metrics\n",
    "    fpr, tpr, threshold = metrics.roc_curve(Y_test[:,i].numpy(), numpy_probas_test[:,i])\n",
    "    roc_auc = metrics.auc(fpr, tpr)\n",
    "    optimal_idx = np.argmax(tpr - fpr)\n",
    "    optimal_threshold = threshold[optimal_idx]\n",
    "    roc.append(roc_auc)\n",
    "    thresh.append(optimal_threshold)\n",
    "    \n",
    "    gmeans = np.sqrt(tpr * (1-fpr))\n",
    "    ix = np.argmax(gmeans)\n",
    "    print('Best Threshold=%f, G-Mean=%.3f' % (threshold[ix], gmeans[ix]))\n",
    "\n",
    "    # method I: plt\n",
    "    import matplotlib.pyplot as plt\n",
    "    plt.title('Receiver Operating Characteristic')\n",
    "    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "    plt.scatter(fpr[ix], tpr[ix], marker='o', color='black', label='Best')\n",
    "    plt.legend(loc = 'lower right')\n",
    "    plt.plot([0, 1], [0, 1],'r--')\n",
    "    plt.xlim([0, 1])\n",
    "    plt.ylim([0, 1])\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(roc)/nt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_labels(pos_probs, threshold):\n",
    "\treturn (pos_probs >= threshold).astype('int')\n",
    "\n",
    "for i in range(nt):\n",
    "    numpy_probas_test[:,i] = to_labels(numpy_probas_test[:,i],thresh[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = []\n",
    "for i in range(nt):\n",
    "    scores.append(binary_classification_performance(Y_test[:,i].numpy(),numpy_probas_test[:,i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = pd.concat(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import multilabel_confusion_matrix\n",
    "multilabel_confusion_matrix(Y_test.numpy(), numpy_probas_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNL model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "before_norm_df = history_frequency_train.loc[:, history_frequency_train.columns != 'engaging_user_id']\n",
    "before_norm_df.loc['Total',:]= before_norm_df.sum(axis=0)\n",
    "before_norm_df.loc[0:107016,:] = before_norm_df.loc['Total'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "before_norm_df.drop(['Total'],inplace=True)\n",
    "before_norm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "after_norm_df = before_norm_df/1890150\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recom = recommend_test.drop('engaging_user_id', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = np.multiply(after_norm_df, recom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3['50'] = df3.sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3['50'] = df3['50']+1156967/1890150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new = df3.loc[:,\"0\":\"49\"].div(df3[\"50\"], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob_mnl = df_new.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "log_loss(Y_test.numpy(),prob_mnl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh = []\n",
    "roc = []\n",
    "for i in range(nt):    \n",
    "    import sklearn.metrics as metrics\n",
    "    fpr, tpr, threshold = metrics.roc_curve(Y_test[:,i].numpy(), prob_mnl[:,i])\n",
    "    roc_auc = metrics.auc(fpr, tpr)\n",
    "    optimal_idx = np.argmax(tpr - fpr)\n",
    "    optimal_threshold = threshold[optimal_idx]\n",
    "    roc.append(roc_auc)\n",
    "    thresh.append(optimal_threshold)\n",
    "    \n",
    "    gmeans = np.sqrt(tpr * (1-fpr))\n",
    "    ix = np.argmax(gmeans)\n",
    "    print('Best Threshold=%f, G-Mean=%.3f' % (threshold[ix], gmeans[ix]))\n",
    "\n",
    "    # method I: plt\n",
    "    import matplotlib.pyplot as plt\n",
    "    plt.title('Receiver Operating Characteristic')\n",
    "    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "    plt.scatter(fpr[ix], tpr[ix], marker='o', color='black', label='Best')\n",
    "    plt.legend(loc = 'lower right')\n",
    "    plt.plot([0, 1], [0, 1],'r--')\n",
    "    plt.xlim([0, 1])\n",
    "    plt.ylim([0, 1])\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UCM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = read_data(\"data.txt\")\n",
    "# choices_to_add = read_data_choices_to_add(\"freqent_itemset_size2.txt\")\n",
    "# ucm_tafeng_H_twitter = initialize_model_with_H(data, choices_to_add)\n",
    "\n",
    "# a = log_likelihood(ucm_tafeng_H_twitter, data)\n",
    "\n",
    "# b = get_subset_counts(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Binary Logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Converting engagement frequencies from tensor to numpy array\n",
    "\n",
    "X_freq_1 = X_freq.numpy()\n",
    "X_freq_1_test = X_freq_test.numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty_train_p = []\n",
    "empty_test_p = []\n",
    "for i in range(nt):\n",
    "    recent_lm_train = X_recent_2[:,[i,i+1*nt,i+2*nt,i+3*nt]]     #Engagement history by topic as input for training\n",
    "    freq_lm_train = X_freq_1[:,i]                            #Engagement frequncy by topic as input for training\n",
    "    recommend_lm_train = X_recommend_1[:,i]                  #Recommendations by topic as input for training\n",
    "    \n",
    "    recent_lm_test = X_recent_2_test[:,[i,i+1*nt,i+2*nt,i+3*nt]] #Engagement history by topic as input for testing\n",
    "    freq_lm_test = X_freq_1_test[:,i]                        #Engagement frequncy by topic as input for testing\n",
    "    recommend_lm_test = X_recommend_t[:,i]                   #Recommendations by topic as input for testing\n",
    "\n",
    "\n",
    "    y_lm_train = y_1[:,i]                                    #Engagement labels\n",
    "\n",
    "    X = np.column_stack([recent_lm_train,freq_lm_train,recommend_lm_train])  #Concatenated input features for training\n",
    "\n",
    "    #LR = LogisticRegression(class_weight=\"balanced\")\n",
    "    LR = LogisticRegression()\n",
    "\n",
    "    LR.fit(X,y_lm_train)\n",
    "    \n",
    "    X_test = np.column_stack([recent_lm_test,freq_lm_test,recommend_lm_test]) #Concatenated input features for testing\n",
    "    \n",
    "    empty_train_p.append((LR.predict_proba(X)[:,1]))\n",
    "       \n",
    "    empty_test_p.append((LR.predict_proba(X_test)[:,1]))\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_lr_train_p = np.transpose(np.array(empty_train_p))  #Predictions for training data\n",
    "prediction_lr_test_p = np.transpose(np.array(empty_test_p))    #Predictions for test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "log_loss(y_t,prediction_lr_test_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "thresh_lr = []\n",
    "roc_lr = []\n",
    "for i in range(nt):    \n",
    "    import sklearn.metrics as metrics\n",
    "    fpr, tpr, threshold = metrics.roc_curve(Y_test[:,i].numpy(), prediction_lr_test_p[:,i])\n",
    "    roc_auc = metrics.auc(fpr, tpr)\n",
    "    optimal_idx = np.argmax(tpr - fpr)\n",
    "    optimal_threshold = threshold[optimal_idx]\n",
    "    thresh_lr.append(optimal_threshold)\n",
    "    roc_lr.append(roc_auc)\n",
    "    \n",
    "    gmeans = np.sqrt(tpr * (1-fpr))\n",
    "    ix = np.argmax(gmeans)\n",
    "    print('Best Threshold=%f, G-Mean=%.3f' % (threshold[ix], gmeans[ix]))\n",
    "\n",
    "    # method I: plt\n",
    "    import matplotlib.pyplot as plt\n",
    "    plt.title('Receiver Operating Characteristic')\n",
    "    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "    plt.scatter(fpr[ix], tpr[ix], marker='o', color='black', label='Best')\n",
    "    plt.legend(loc = 'lower right')\n",
    "    plt.plot([0, 1], [0, 1],'r--')\n",
    "    plt.xlim([0, 1])\n",
    "    plt.ylim([0, 1])\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(roc_lr)/nt  #Average AUC-ROC score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_lr_test_p.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_lr_test_p = np.empty([107017, nt])\n",
    "for i in range(nt):\n",
    "    label_lr_test_p[:,i] = to_labels(prediction_lr_test_p[:,i],thresh_lr[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import multilabel_confusion_matrix\n",
    "multilabel_confusion_matrix(Y_test.numpy(), label_lr_test_p)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_lr = []\n",
    "for i in range(nt):\n",
    "    scores_lr.append(binary_classification_performance(Y_test[:,i].numpy(),label_lr_test_p[:,i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_lr = pd.concat(scores_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_lr.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Light GBM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty_train_1p = []\n",
    "empty_test_1p = []\n",
    "\n",
    "\n",
    "for i in range(nt):\n",
    "    recent_lm_train = X_recent_2[:,[i,i+nt,i+2*nt,i+3*nt]]\n",
    "    freq_lm_train = X_freq_1[:,i]\n",
    "    recommend_lm_train = X_recommend_1[:,i]\n",
    "    \n",
    "    recent_lm_test = X_recent_2_test[:,[i,i+nt,i+2*nt,i+3*nt]]\n",
    "    freq_lm_test = X_freq_1_test[:,i]\n",
    "    recommend_lm_test = X_recommend_t[:,i]\n",
    "\n",
    "\n",
    "    y_lm_train = y_1[:,i]\n",
    "    \n",
    "\n",
    "    X = np.column_stack([recent_lm_train,freq_lm_train,recommend_lm_train])\n",
    "\n",
    "    LGBM = LGBMClassifier()\n",
    "\n",
    "    LGBM.fit(X,y_lm_train)\n",
    "    \n",
    "    X_test = np.column_stack([recent_lm_test,freq_lm_test,recommend_lm_test])\n",
    "    \n",
    "    empty_train_1p.append((LGBM.predict_proba(X)[:,1]))\n",
    "\n",
    "    empty_test_1p.append((LGBM.predict_proba(X_test)[:,1]))\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_lgbm_train_p = np.transpose(np.array(empty_train_1p))\n",
    "prediction_lgbm_test_p = np.transpose(np.array(empty_test_1p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "log_loss(Y_test.numpy(),prediction_lgbm_test_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh_lgbm = []\n",
    "roc_lgbm = []\n",
    "for i in range(nt):    \n",
    "    import sklearn.metrics as metrics\n",
    "    # calculate the fpr and tpr for all thresholds of the classification\n",
    "    #probs = model.predict_proba(X_test)\n",
    "    #preds = probs[:,1]\n",
    "    fpr, tpr, threshold = metrics.roc_curve(Y_test[:,i].numpy(), prediction_lgbm_test_p[:,i])\n",
    "    roc_auc = metrics.auc(fpr, tpr)\n",
    "    optimal_idx = np.argmax(tpr - fpr)\n",
    "    optimal_threshold = threshold[optimal_idx]\n",
    "    print(optimal_threshold)\n",
    "    thresh_lgbm.append(optimal_threshold)\n",
    "    roc_lgbm.append(roc_auc)\n",
    "    \n",
    "    gmeans = np.sqrt(tpr * (1-fpr))\n",
    "    ix = np.argmax(gmeans)\n",
    "    print('Best Threshold=%f, G-Mean=%.3f' % (threshold[ix], gmeans[ix]))\n",
    "\n",
    "    # method I: plt\n",
    "    import matplotlib.pyplot as plt\n",
    "    plt.title('Receiver Operating Characteristic')\n",
    "    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "    plt.scatter(fpr[ix], tpr[ix], marker='o', color='black', label='Best')\n",
    "    plt.legend(loc = 'lower right')\n",
    "    plt.plot([0, 1], [0, 1],'r--')\n",
    "    plt.xlim([0, 1])\n",
    "    plt.ylim([0, 1])\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(roc_lgbm)/nt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_lgbm_test_p = np.empty([107017, nt])\n",
    "for i in range(nt):\n",
    "    label_lgbm_test_p[:,i] = to_labels(prediction_lgbm_test_p[:,i],thresh_lgbm[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multilabel_confusion_matrix(Y_test.numpy(), label_lgbm_test_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_lgbm = []\n",
    "for i in range(nt):\n",
    "    scores_lgbm.append(binary_classification_performance(Y_test[:,i].numpy(),label_lgbm_test_p[:,i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_lgbm = pd.concat(scores_lgbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_lgbm.mean(axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exhaustive search for Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = pd.DataFrame(X_recommend_t)\n",
    "x.sum(axis=1).value_counts()\n",
    "x['Total'] = x.sum(axis=1)\n",
    "y=x[x['Total'] >= 6] #Considering only the users who are shown atleast six tweets\n",
    "#del y['Total']\n",
    "\n",
    "infinity = pd.DataFrame(X_freq_1_test)\n",
    "histo = pd.DataFrame(X_recent_2_test)\n",
    "\n",
    "inf_new=infinity.loc[y.index,:]\n",
    "hist_new=histo.loc[y.index,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Greedy Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "def greedy(freq, hist, y, nt):\n",
    "    empty =[]\n",
    "    for user in range(y.shape[0]):\n",
    "        s=set()\n",
    "        for i,x in enumerate(y.iloc[user]):\n",
    "            if x==1.0:\n",
    "                s.add(i)\n",
    "        output=[[0.0]*nt]\n",
    "        removed=set()\n",
    "        l_=[]\n",
    "\n",
    "        for j in range(5):\n",
    "            ma=-float('inf')\n",
    "            for i in s:\n",
    "                l_.append(i)\n",
    "                output[0][i]=1.0\n",
    "                inf_new_ = torch.FloatTensor(freq.iloc[user].values).reshape(1,nt)\n",
    "                hist_new_ = torch.FloatTensor(hist.iloc[user].values.reshape(-1,nt,4))\n",
    "                y_ = torch.FloatTensor(np.array(output[0])).reshape(1,nt)\n",
    "                indicator = torch.ones([nt]).reshape(1,nt)\n",
    "                \n",
    "                logits_opt = model(inf_new_.to(device), hist_new_.to(device), y_.to(device), indicator.to(device))\n",
    "                numpy_logits_opt = logits_opt.to('cpu').detach().numpy()\n",
    "                numpy_probas_opt = sigmoid_v(numpy_logits_opt)\n",
    "                sum_=sum(numpy_probas_opt[0][k] for k in l_)\n",
    "                if sum_>ma:\n",
    "                    result_l=l_[:]\n",
    "                    numpy_logits_opt_max=numpy_logits_opt\n",
    "                    numpy_probas_opt_max=numpy_probas_opt\n",
    "                    ma=sum_\n",
    "                    index=i\n",
    "                l_.pop()\n",
    "                output[0][i]=0.0\n",
    "\n",
    "            output[0][index]=1.0\n",
    "            l_.append(index)\n",
    "\n",
    "            s.remove(index)\n",
    "        empty.append([y.index[user],ma,result_l])\n",
    "    n=pd.DataFrame(empty,columns=['USER','MAX_SUM','LIST_OF_INDEX'])\n",
    "    return (n['MAX_SUM'].mean())\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy(inf_new, hist_new, y, nt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uplift score obtained using combination of topics a given model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#prob : output probabilities for a given model\n",
    "#y : input recommendations considering overall timeline\n",
    "#freq: engagement frequency\n",
    "#hist: engagement history\n",
    "\n",
    "def uplift_score(prob, freq, hist, y, nt):     \n",
    "    mdl = pd.DataFrame(prob)\n",
    "    mdl_y = mdl.loc[y.index]\n",
    "    p = mdl_y*y.iloc[:,0:nt].values\n",
    "    s=pd.DataFrame(np.zeros((p.shape[0],p.shape[1]))).set_index(p.index)\n",
    "    for ind in p.index:\n",
    "        s.loc[ind,p.loc[ind].nlargest(5).index.values]=1\n",
    "    s_=s.reset_index()\n",
    "    s_.index = s_['index']\n",
    "    del s_['index']\n",
    "    inf_new_T = torch.FloatTensor(freq.values)\n",
    "    hist_new_T = torch.FloatTensor(hist.values.reshape(-1,nt,4))\n",
    "    rec_T = torch.FloatTensor(s_.values)\n",
    "    indicator_T = torch.ones([len(rec_T), nt])\n",
    "    logits_opt = model(inf_new_T.to(device), hist_new_T.to(device), rec_T.to(device), indicator_T.to(device))\n",
    "    numpy_logits_opt = logits_opt.to('cpu').detach().numpy()\n",
    "    numpy_probas_opt = sigmoid_v(numpy_logits_opt)\n",
    "    sp = pd.DataFrame(numpy_probas_opt, index = y.index)\n",
    "    sc = sp*s_.values\n",
    "    return(sc.sum(axis = 1).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uplift_score(prediction_lr_test_p,inf_new, hist_new, y, nt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uplift_score(prediction_lgbm_test_p,inf_new, hist_new, y, nt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Product_choice_model_bucketing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}