Initialization_rado.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialization of libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %matplotlib inline\n",
    "import os\n",
    "import sys\n",
    "# import matplotlib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "# import matplotlib.pyplot as plt\n",
    "# import networkx as nx\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "from sktensor import dtensor, cp_als\n",
    "\n",
    "# matplotlib.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Manufacturing company emails"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Looking for the dataset\n",
    "if not os.path.isdir('radoslaw_email'):\n",
    "    print 'The dataset has not be found'\n",
    "else:\n",
    "    # Loading the dataset\n",
    "    tab = pd.read_csv('radoslaw_email/out.radoslaw_email_email', sep=' ', skiprows=2, header=None, index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Display\n",
    "# tab.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Keeping desired columns and naming them\n",
    "tab.drop(2, axis=1, inplace=True)\n",
    "tab.columns = ['sender', 'receiver', 'weight', 'datetime']\n",
    "\n",
    "# Date Time format\n",
    "tab.datetime = pd.to_datetime(tab.datetime,unit='s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Display\n",
    "# tab.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quick Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print '%d mails exchanged between %d employees from %s to %s' %(tab.shape[0],\n",
    "#                                                                 len(pd.unique(pd.concat([tab.sender,tab.receiver]))),\n",
    "#                                                                 str(min(tab.datetime.dt.date)),\n",
    "#                                                                 str(max(tab.datetime.dt.date))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tab.datetime.groupby(tab.datetime.dt.week).count().plot(\n",
    "#     title='Temporal Distribution per week',\n",
    "#     kind='bar',\n",
    "#     figsize=(16, 10),\n",
    "#     color='#348ABD',\n",
    "#     alpha=0.6,\n",
    "#     lw='1',\n",
    "#     edgecolor='#348ABD',\n",
    "#     grid=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of active days / sender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sent_mails = tab.datetime.dt.date.groupby(tab.sender).nunique()\n",
    "\n",
    "# sent_mails.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of receivers / sender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sent_mails = tab.receiver.groupby(tab.sender).nunique()\n",
    "\n",
    "# sent_mails.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sent_mails.argmax()\n",
    "# max(sent_mails) == len(pd.unique(tab.ix[tab.sender == 38, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top senders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# top_senders = tab.sender.groupby(tab.sender).count()\n",
    "# top_senders.sort(ascending=False)\n",
    "# top_senders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top receivers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# top_receivers = tab.receiver.groupby(tab.receiver).count()\n",
    "# top_receivers.sort(ascending=False)\n",
    "# top_receivers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating a graph (Dipou's Birthday example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dipou_brthd = datetime.strptime('2010-01-19', '%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# row_list = tab.datetime.dt.date == dipou_brthd.date()\n",
    "# edge_list = zip(tab.sender[row_list], tab.receiver[row_list])\n",
    "\n",
    "# G=nx.Graph()\n",
    "# G.add_edges_from(edge_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print '%d mails exchanged between %d employees the %s' %(nx.number_of_edges(G), \n",
    "#                                                          nx.number_of_nodes(G), \n",
    "#                                                          str(dipou_brthd.date()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# nx.draw(G)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Converting datetimes into integers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dates = tab.datetime.dt.date[:]\n",
    "tab.datetime = dates.apply(lambda dates: dates.toordinal())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Listing all the unique values of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tab.iloc[:, [0, 1, 3]]\n",
    "\n",
    "i_list = pd.unique(X.iloc[:, 0])\n",
    "j_list = pd.unique(X.iloc[:, 1])\n",
    "k_list = pd.unique(X.iloc[:, 2])\n",
    "\n",
    "i_list.sort()\n",
    "j_list.sort()\n",
    "k_list.sort()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating and filling the tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "T = np.zeros((len(i_list), len(j_list), len(k_list)))\n",
    "T = dtensor(T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Going through the dataframe\n",
    "for i, j, k in zip(X.sender, X.receiver, X.datetime):\n",
    "    # Finding the corresponding index in the tensor\n",
    "    i_ind = int(np.where(i_list == i)[0])\n",
    "    j_ind = int(np.where(j_list == j)[0])\n",
    "    k_ind = int(np.where(k_list == k)[0])\n",
    "    \n",
    "    # Modifying the tensor value for the tuple (i_ind, j_ind, k_ind)\n",
    "    T[i_ind, j_ind, k_ind] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logarithmic transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nonz = T.nonzero()\n",
    "for ind in range(len(nonz[0])):\n",
    "    i_ind = nonz[0][ind]\n",
    "    j_ind = nonz[1][ind] \n",
    "    k_ind = nonz[2][ind]\n",
    "    \n",
    "    T[i_ind, j_ind, k_ind] = 1 + np.log(T[i_ind, j_ind, k_ind]) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Useful Code for finding dataframe lines corresponding to given values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sender</th>\n",
       "      <th>receiver</th>\n",
       "      <th>datetime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8   </th>\n",
       "      <td> 1</td>\n",
       "      <td> 10</td>\n",
       "      <td> 733774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593 </th>\n",
       "      <td> 1</td>\n",
       "      <td> 10</td>\n",
       "      <td> 733776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1030</th>\n",
       "      <td> 1</td>\n",
       "      <td> 10</td>\n",
       "      <td> 733777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040</th>\n",
       "      <td> 1</td>\n",
       "      <td> 10</td>\n",
       "      <td> 733777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1764</th>\n",
       "      <td> 1</td>\n",
       "      <td> 10</td>\n",
       "      <td> 733779</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      sender  receiver  datetime\n",
       "8          1        10    733774\n",
       "593        1        10    733776\n",
       "1030       1        10    733777\n",
       "1040       1        10    733777\n",
       "1764       1        10    733779"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = 1  # sender \"1\"\n",
    "j = 10  # receiver \"10\"\n",
    "# k = \n",
    "\n",
    "X.ix[(X.sender == i)&(X.receiver == j)].head()  # using \"head()\" for only showing the 5 first results\n",
    "# X.ix[(X.sender == i)&(X.receiver == j)&(X.datetime == k)]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}