2019/Texts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Used to read the Parquet data\n",
    "import pandas as pandas\n",
    "from os import walk\n",
    "import pyarrow.parquet as parquet\n",
    "\n",
    "# Used to train document embeddings\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "\n",
    "# Used to train the baseline model\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import numpy as numpy\n",
    "\n",
    "# Where the downloaded data are\n",
    "input_path = './'\n",
    "# Where to store results\n",
    "output_path = './'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dmitriybugaichenko/anaconda/envs/python37/lib/python3.7/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
      "  labels, = index.labels\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>objectId</th>\n",
       "      <th>preprocessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>517288</td>\n",
       "      <td>[квартирник, нтв, маргулис, групп, пилот]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9501964</td>\n",
       "      <td>[родител, очен, трогательн, песн, артур, халат]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23007371</td>\n",
       "      <td>[сух, суперджет, откажет, западн, комплект, те...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38353886</td>\n",
       "      <td>[сгорел, сара, гор, хат]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21192138</td>\n",
       "      <td>[живодёр, отруб, лап, собак]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>26415073</td>\n",
       "      <td>[ажурн, маков, кулич, так, нежн, воздушн, мяки...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>36734526</td>\n",
       "      <td>[друг, уснул, бар, теб, нужн, тащ, дом, очен, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8699823</td>\n",
       "      <td>[никифор, ден, дат, год, март, понедельник, др...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12236843</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>38393782</td>\n",
       "      <td>[днр, лнр, новост, войск, берут, кольц, донбас...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   objectId                                       preprocessed\n",
       "0    517288          [квартирник, нтв, маргулис, групп, пилот]\n",
       "1   9501964    [родител, очен, трогательн, песн, артур, халат]\n",
       "2  23007371  [сух, суперджет, откажет, западн, комплект, те...\n",
       "3  38353886                           [сгорел, сара, гор, хат]\n",
       "4  21192138                       [живодёр, отруб, лап, собак]\n",
       "5  26415073  [ажурн, маков, кулич, так, нежн, воздушн, мяки...\n",
       "6  36734526  [друг, уснул, бар, теб, нужн, тащ, дом, очен, ...\n",
       "7   8699823  [никифор, ден, дат, год, март, понедельник, др...\n",
       "8  12236843                                                 []\n",
       "9  38393782  [днр, лнр, новост, войск, берут, кольц, донбас..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the test texts\n",
    "test_texts = parquet.read_table(input_path + '/texts/textsTest/', \\\n",
    "                                columns = ['objectId','preprocessed']).to_pandas()\n",
    "test_texts.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build document embeddings for text documents\n",
    "doc2vec = Doc2Vec(\\\n",
    "                [TaggedDocument(lines,'tag') for lines in test_texts.preprocessed], \\\n",
    "                vector_size=5, window=2, min_count=1, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>objectId</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22429313</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14676953</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11562101</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20892119</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16063005</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14576490</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11811947</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1936012</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>20867189</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28967185</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   objectId  label\n",
       "0  22429313    0.0\n",
       "1  14676953    0.0\n",
       "2  11562101    0.0\n",
       "3  20892119    0.0\n",
       "4  16063005    0.0\n",
       "5  14576490    0.0\n",
       "6  11811947    0.0\n",
       "7   1936012    1.0\n",
       "8  20867189    0.0\n",
       "9  28967185    0.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read a single day to train model on as Pandas dataframe\n",
    "data = parquet.read_table(input_path + '/textsTrain/date=2018-02-07', \\\n",
    "                          columns = ['instanceId_objectId','feedback']).to_pandas()\n",
    "data.rename(columns = {'instanceId_objectId':'objectId'}, inplace = True)\n",
    "data['label'] = data['feedback'].apply(lambda x: 1.0 if(\"Liked\" in x) else 0.0).values\n",
    "data = data[['objectId','label']]\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with part-00000-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00001-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00002-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00003-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00004-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00005-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00006-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00007-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00008-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00009-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00010-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00011-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00012-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00013-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00014-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00015-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00016-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00017-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00018-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00019-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00020-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00021-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00022-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00023-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00024-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00025-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n",
      "Done with part-00026-1b50c8f5-87db-4a53-9677-17f1113c3f8d-c000.gz.parquet\n"
     ]
    }
   ],
   "source": [
    "parts = []\n",
    "\n",
    "# Get unique object ids\n",
    "ids = data.groupby('objectId').count()\n",
    "\n",
    "# In order to save memory iterate part by part\n",
    "for (dirpath, dirnames, filenames) in walk(input_path + '/texts/textsTrain/'):\n",
    "    for name in filenames:\n",
    "        if name.startswith('part'):\n",
    "            # Read single part\n",
    "            texts = parquet.read_table(input_path + '/texts/textsTrain/' + name, \\\n",
    "                                       columns = ['objectId','preprocessed']).to_pandas()            \n",
    "            # Filter documents we need\n",
    "            joined = ids.join(texts.set_index('objectId'), how='inner', on = 'objectId')\n",
    "            # Evaluate embeddings\n",
    "            joined['embedding'] = joined.preprocessed.apply(doc2vec.infer_vector)\n",
    "            # Memorize\n",
    "            parts.append(joined[['embedding']])\n",
    "            print('Done with ' + name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>objectId</th>\n",
       "      <th>label</th>\n",
       "      <th>embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22429313</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[1.7023556, -0.61280745, -2.3916838, 3.7633474...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14676953</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[1.6586152, -0.36108485, -1.9833397, 2.0739386...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11562101</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.15386434, -0.33638588, -0.5353734, 0.122175...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20892119</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.9824828, -0.5868671, -0.98727924, 1.5039121...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16063005</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.99467623, -0.548113, -3.524225, 3.7854316, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14576490</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.021854596, -0.40157402, -0.023478763, 0.198...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11811947</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.6359992, -0.75050163, -1.2051268, 0.7758712...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1936012</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[-0.11006393, -0.64717454, -0.58839494, 0.2012...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>20867189</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[1.5456027, 0.36930153, -2.1795776, 2.1914325,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28967185</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.3342202, 0.13672762, -0.16420954, 0.2752888...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   objectId  label                                          embedding\n",
       "0  22429313    0.0  [1.7023556, -0.61280745, -2.3916838, 3.7633474...\n",
       "1  14676953    0.0  [1.6586152, -0.36108485, -1.9833397, 2.0739386...\n",
       "2  11562101    0.0  [0.15386434, -0.33638588, -0.5353734, 0.122175...\n",
       "3  20892119    0.0  [0.9824828, -0.5868671, -0.98727924, 1.5039121...\n",
       "4  16063005    0.0  [0.99467623, -0.548113, -3.524225, 3.7854316, ...\n",
       "5  14576490    0.0  [0.021854596, -0.40157402, -0.023478763, 0.198...\n",
       "6  11811947    0.0  [0.6359992, -0.75050163, -1.2051268, 0.7758712...\n",
       "7   1936012    1.0  [-0.11006393, -0.64717454, -0.58839494, 0.2012...\n",
       "8  20867189    0.0  [1.5456027, 0.36930153, -2.1795776, 2.1914325,...\n",
       "9  28967185    0.0  [0.3342202, 0.13672762, -0.16420954, 0.2752888..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine all the parts\n",
    "train = data.join(pandas.concat(parts), on = 'objectId')\n",
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct the label (liked objects)\n",
    "y = train['label'].values\n",
    "\n",
    "# Extract the most interesting features\n",
    "X = numpy.stack(train['embedding'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.26446775, -0.09334847,  0.06236364, -0.00363809,  0.18652154]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model and check the weights\n",
    "model = LogisticRegression(random_state=0, solver='lbfgs').fit(X, y)\n",
    "model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>objectId</th>\n",
       "      <th>preprocessed</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>517288</td>\n",
       "      <td>[квартирник, нтв, маргулис, групп, пилот]</td>\n",
       "      <td>-0.193637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9501964</td>\n",
       "      <td>[родител, очен, трогательн, песн, артур, халат]</td>\n",
       "      <td>-0.181544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23007371</td>\n",
       "      <td>[сух, суперджет, откажет, западн, комплект, те...</td>\n",
       "      <td>-0.170146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38353886</td>\n",
       "      <td>[сгорел, сара, гор, хат]</td>\n",
       "      <td>-0.170694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21192138</td>\n",
       "      <td>[живодёр, отруб, лап, собак]</td>\n",
       "      <td>-0.168873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>26415073</td>\n",
       "      <td>[ажурн, маков, кулич, так, нежн, воздушн, мяки...</td>\n",
       "      <td>-0.118487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>36734526</td>\n",
       "      <td>[друг, уснул, бар, теб, нужн, тащ, дом, очен, ...</td>\n",
       "      <td>-0.172113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8699823</td>\n",
       "      <td>[никифор, ден, дат, год, март, понедельник, др...</td>\n",
       "      <td>-0.181164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12236843</td>\n",
       "      <td>[]</td>\n",
       "      <td>-0.179489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>38393782</td>\n",
       "      <td>[днр, лнр, новост, войск, берут, кольц, донбас...</td>\n",
       "      <td>-0.181132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   objectId                                       preprocessed    weight\n",
       "0    517288          [квартирник, нтв, маргулис, групп, пилот] -0.193637\n",
       "1   9501964    [родител, очен, трогательн, песн, артур, халат] -0.181544\n",
       "2  23007371  [сух, суперджет, откажет, западн, комплект, те... -0.170146\n",
       "3  38353886                           [сгорел, сара, гор, хат] -0.170694\n",
       "4  21192138                       [живодёр, отруб, лап, собак] -0.168873\n",
       "5  26415073  [ажурн, маков, кулич, так, нежн, воздушн, мяки... -0.118487\n",
       "6  36734526  [друг, уснул, бар, теб, нужн, тащ, дом, очен, ... -0.172113\n",
       "7   8699823  [никифор, ден, дат, год, март, понедельник, др... -0.181164\n",
       "8  12236843                                                 [] -0.179489\n",
       "9  38393782  [днр, лнр, новост, войск, берут, кольц, донбас... -0.181132"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Weight the test documents\n",
    "test_texts['weight'] = -model.predict_proba(numpy.stack(\\\n",
    "    test_texts.preprocessed.apply(lambda x : doc2vec.infer_vector(x))))[:, 1]\n",
    "test_texts.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dmitriybugaichenko/anaconda/envs/python37/lib/python3.7/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
      "  labels, = index.labels\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instanceId_userId</th>\n",
       "      <th>objectId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1006</td>\n",
       "      <td>34577503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1006</td>\n",
       "      <td>37520199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1618</td>\n",
       "      <td>546086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1618</td>\n",
       "      <td>546086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1618</td>\n",
       "      <td>546086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1618</td>\n",
       "      <td>35981492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1618</td>\n",
       "      <td>26764305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1810</td>\n",
       "      <td>958605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1810</td>\n",
       "      <td>20479574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1810</td>\n",
       "      <td>36254478</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   instanceId_userId  objectId\n",
       "0               1006  34577503\n",
       "1               1006  37520199\n",
       "2               1618    546086\n",
       "3               1618    546086\n",
       "4               1618    546086\n",
       "5               1618  35981492\n",
       "6               1618  26764305\n",
       "7               1810    958605\n",
       "8               1810  20479574\n",
       "9               1810  36254478"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the test data\n",
    "test = parquet.read_table(input_path + '/textsTest', \\\n",
    "    columns = ['instanceId_userId','instanceId_objectId']).to_pandas()\n",
    "test.rename(columns = {'instanceId_objectId':'objectId'}, inplace = True)\n",
    "test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instanceId_userId</th>\n",
       "      <th>objectId</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">316</th>\n",
       "      <th>17997084</th>\n",
       "      <td>-0.128333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37758420</th>\n",
       "      <td>-0.172750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">631</th>\n",
       "      <th>15478935</th>\n",
       "      <td>-0.156306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30513650</th>\n",
       "      <td>-0.154663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38118098</th>\n",
       "      <td>-0.163259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">742</th>\n",
       "      <th>10672856</th>\n",
       "      <td>-0.137634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24302446</th>\n",
       "      <td>-0.158201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28816291</th>\n",
       "      <td>-0.173662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34685448</th>\n",
       "      <td>-0.151551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <th>11640701</th>\n",
       "      <td>-0.162864</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              weight\n",
       "instanceId_userId objectId          \n",
       "316               17997084 -0.128333\n",
       "                  37758420 -0.172750\n",
       "631               15478935 -0.156306\n",
       "                  30513650 -0.154663\n",
       "                  38118098 -0.163259\n",
       "742               10672856 -0.137634\n",
       "                  24302446 -0.158201\n",
       "                  28816291 -0.173662\n",
       "                  34685448 -0.151551\n",
       "868               11640701 -0.162864"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Join test documents and elliminate possible duplicates\n",
    "scores = test.join(\\\n",
    "    test_texts[['objectId','weight']].set_index('objectId'), \\\n",
    "    how = 'inner', \\\n",
    "    on = 'objectId') \\\n",
    "    .groupby(['instanceId_userId','objectId']).min()\n",
    "scores.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instanceId_userId</th>\n",
       "      <th>objectId</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>316</td>\n",
       "      <td>37758420</td>\n",
       "      <td>-0.172750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>316</td>\n",
       "      <td>17997084</td>\n",
       "      <td>-0.128333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>631</td>\n",
       "      <td>38118098</td>\n",
       "      <td>-0.163259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>631</td>\n",
       "      <td>15478935</td>\n",
       "      <td>-0.156306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>631</td>\n",
       "      <td>30513650</td>\n",
       "      <td>-0.154663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>742</td>\n",
       "      <td>28816291</td>\n",
       "      <td>-0.173662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>742</td>\n",
       "      <td>24302446</td>\n",
       "      <td>-0.158201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>742</td>\n",
       "      <td>34685448</td>\n",
       "      <td>-0.151551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>742</td>\n",
       "      <td>10672856</td>\n",
       "      <td>-0.137634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>868</td>\n",
       "      <td>29193052</td>\n",
       "      <td>-0.197144</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   instanceId_userId  objectId    weight\n",
       "0                316  37758420 -0.172750\n",
       "1                316  17997084 -0.128333\n",
       "2                631  38118098 -0.163259\n",
       "3                631  15478935 -0.156306\n",
       "4                631  30513650 -0.154663\n",
       "5                742  28816291 -0.173662\n",
       "6                742  24302446 -0.158201\n",
       "7                742  34685448 -0.151551\n",
       "8                742  10672856 -0.137634\n",
       "9                868  29193052 -0.197144"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  Sort for each user\n",
    "result = scores.sort_values(by=['instanceId_userId', 'weight']).reset_index()\n",
    "result.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "instanceId_userId\n",
       "316                                  [37758420, 17997084]\n",
       "631                        [38118098, 15478935, 30513650]\n",
       "742              [28816291, 24302446, 34685448, 10672856]\n",
       "868     [29193052, 22115500, 29650308, 30882080, 35655...\n",
       "979                                   [37950972, 7996257]\n",
       "1006                                 [37520199, 34577503]\n",
       "1276                       [36856262, 22812401, 31000576]\n",
       "1444                                 [36806487, 20963755]\n",
       "1483                                 [34991228, 38036543]\n",
       "1618                         [546086, 26764305, 35981492]\n",
       "Name: objectId, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Collect predictions for each user\n",
    "submit = result.groupby(\"instanceId_userId\")['objectId'].apply(list)\n",
    "submit.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Persist the first submit\n",
    "submit.to_csv(output_path + \"/textSubmit.csv.gz\", header = False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}