predicting_news_popularity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inline plotting instead of popping out\n",
    "%matplotlib inline\n",
    "\n",
    "import os, itertools, csv, timeit, json, re\n",
    "from datetime import datetime\n",
    "from IPython.display import Image\n",
    "from IPython.display import display\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import matplotlib.pyplot as plt\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# nltk\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tag import pos_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sklearn\n",
    "from sklearn.model_selection import train_test_split, KFold, GridSearchCV, cross_val_score\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
    "from sklearn.compose import ColumnTransformer, make_column_transformer\n",
    "from sklearn.preprocessing import StandardScaler, PolynomialFeatures, OneHotEncoder, LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB, ComplementNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, VotingClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error, roc_curve, auc, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirname = './data'\n",
    "f_train = 'preprocessed_train_2.csv'\n",
    "f_test = 'preprocessed_test_2.csv'\n",
    "f_sub = 'sample_submission.csv'\n",
    "p_train = os.path.join(dirname, f_train)\n",
    "p_test = os.path.join(dirname, f_test)\n",
    "p_sub = os.path.join(dirname, f_sub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pop</th>\n",
       "      <th>content</th>\n",
       "      <th>article</th>\n",
       "      <th>title</th>\n",
       "      <th>topics</th>\n",
       "      <th>data_channel</th>\n",
       "      <th>see_also</th>\n",
       "      <th>p_year</th>\n",
       "      <th>p_month</th>\n",
       "      <th>...</th>\n",
       "      <th>title_pos_tag_7</th>\n",
       "      <th>title_pos_tag_8</th>\n",
       "      <th>title_pos_tag_9</th>\n",
       "      <th>title_pos_tag_10</th>\n",
       "      <th>title_pos_tag_11</th>\n",
       "      <th>title_pos_tag_12</th>\n",
       "      <th>title_pos_tag_13</th>\n",
       "      <th>title_pos_tag_14</th>\n",
       "      <th>title_pos_tag_15</th>\n",
       "      <th>num_topics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt; &lt;span c...</td>\n",
       "      <td>There may be killer asteroids headed for Earth...</td>\n",
       "      <td>NASA's Grand Challenge: Stop Asteroids From De...</td>\n",
       "      <td>['Asteroid', 'Asteroids', 'challenge', 'Earth'...</td>\n",
       "      <td>world</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>IN</td>\n",
       "      <td>VBG</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Google took a stand of sorts against patent-la...</td>\n",
       "      <td>Google's New Open Source Patent Pledge: We Won...</td>\n",
       "      <td>['Apps and Software', 'Google', 'open source',...</td>\n",
       "      <td>tech</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NN</td>\n",
       "      <td>PRP</td>\n",
       "      <td>MD</td>\n",
       "      <td>RB</td>\n",
       "      <td>VB</td>\n",
       "      <td>IN</td>\n",
       "      <td>VBN</td>\n",
       "      <td>RB</td>\n",
       "      <td>NONE</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>You've spend countless hours training to be an...</td>\n",
       "      <td>Ballin': 2014 NFL Draft Picks Get to Choose Th...</td>\n",
       "      <td>['Entertainment', 'NFL', 'NFL Draft', 'Sports'...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>TO</td>\n",
       "      <td>VB</td>\n",
       "      <td>PRP$</td>\n",
       "      <td>JJ</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Tired of the same old sports fails and news f...</td>\n",
       "      <td>Cameraperson Fails Deliver Slapstick Laughs</td>\n",
       "      <td>['Sports', 'Video', 'Videos', 'Watercooler']</td>\n",
       "      <td>watercooler</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>At 6-foot-5 and 298 pounds, All-Pro NFL star J...</td>\n",
       "      <td>NFL Star Helps Young Fan Prove Friendship With...</td>\n",
       "      <td>['Entertainment', 'instagram', 'instagram vide...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NN</td>\n",
       "      <td>IN</td>\n",
       "      <td>JJ</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  pop                                            content  \\\n",
       "0   0   -1  <html><head><div class=\"article-info\"> <span c...   \n",
       "1   1    1  <html><head><div class=\"article-info\"><span cl...   \n",
       "2   2    1  <html><head><div class=\"article-info\"><span cl...   \n",
       "3   3   -1  <html><head><div class=\"article-info\"><span cl...   \n",
       "4   4   -1  <html><head><div class=\"article-info\"><span cl...   \n",
       "\n",
       "                                             article  \\\n",
       "0  There may be killer asteroids headed for Earth...   \n",
       "1  Google took a stand of sorts against patent-la...   \n",
       "2  You've spend countless hours training to be an...   \n",
       "3   Tired of the same old sports fails and news f...   \n",
       "4  At 6-foot-5 and 298 pounds, All-Pro NFL star J...   \n",
       "\n",
       "                                               title  \\\n",
       "0  NASA's Grand Challenge: Stop Asteroids From De...   \n",
       "1  Google's New Open Source Patent Pledge: We Won...   \n",
       "2  Ballin': 2014 NFL Draft Picks Get to Choose Th...   \n",
       "3        Cameraperson Fails Deliver Slapstick Laughs   \n",
       "4  NFL Star Helps Young Fan Prove Friendship With...   \n",
       "\n",
       "                                              topics   data_channel  see_also  \\\n",
       "0  ['Asteroid', 'Asteroids', 'challenge', 'Earth'...          world         1   \n",
       "1  ['Apps and Software', 'Google', 'open source',...           tech         1   \n",
       "2  ['Entertainment', 'NFL', 'NFL Draft', 'Sports'...  entertainment         1   \n",
       "3       ['Sports', 'Video', 'Videos', 'Watercooler']    watercooler         1   \n",
       "4  ['Entertainment', 'instagram', 'instagram vide...  entertainment         1   \n",
       "\n",
       "   p_year  p_month  ...  title_pos_tag_7  title_pos_tag_8  title_pos_tag_9  \\\n",
       "0  2013.0      6.0  ...               IN              VBG               NN   \n",
       "1  2013.0      3.0  ...               NN              PRP               MD   \n",
       "2  2014.0      5.0  ...               TO               VB             PRP$   \n",
       "3  2013.0     10.0  ...             NONE             NONE             NONE   \n",
       "4  2014.0      4.0  ...               NN               IN               JJ   \n",
       "\n",
       "   title_pos_tag_10  title_pos_tag_11 title_pos_tag_12  title_pos_tag_13  \\\n",
       "0              NONE              NONE             NONE              NONE   \n",
       "1                RB                VB               IN               VBN   \n",
       "2                JJ                NN               NN              NONE   \n",
       "3              NONE              NONE             NONE              NONE   \n",
       "4                NN              NONE             NONE              NONE   \n",
       "\n",
       "   title_pos_tag_14  title_pos_tag_15 num_topics  \n",
       "0              NONE              NONE         73  \n",
       "1                RB              NONE        122  \n",
       "2              NONE              NONE         61  \n",
       "3              NONE              NONE         44  \n",
       "4              NONE              NONE         66  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(p_train)\n",
    "df_train.rename(\n",
    "    columns={'Id': 'id', 'Popularity': 'pop', 'Page content': 'content', 'author scalar': 'author_scalar'}, \n",
    "    inplace=True\n",
    ")\n",
    "pop_group = df_train.groupby(['pop'])\n",
    "if 'Unnamed: 0' in df_train.columns:\n",
    "    df_train.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>article</th>\n",
       "      <th>title</th>\n",
       "      <th>topics</th>\n",
       "      <th>data_channel</th>\n",
       "      <th>see_also</th>\n",
       "      <th>p_year</th>\n",
       "      <th>p_month</th>\n",
       "      <th>p_day</th>\n",
       "      <th>...</th>\n",
       "      <th>title_pos_tag_7</th>\n",
       "      <th>title_pos_tag_8</th>\n",
       "      <th>title_pos_tag_9</th>\n",
       "      <th>title_pos_tag_10</th>\n",
       "      <th>title_pos_tag_11</th>\n",
       "      <th>title_pos_tag_12</th>\n",
       "      <th>title_pos_tag_13</th>\n",
       "      <th>title_pos_tag_14</th>\n",
       "      <th>title_pos_tag_15</th>\n",
       "      <th>num_topics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27643</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Note to humanity: One Direction fandom ain't ...</td>\n",
       "      <td>Soccer Star Gets Twitter Death Threats After T...</td>\n",
       "      <td>['Entertainment', 'Music', 'One Direction', 's...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>IN</td>\n",
       "      <td>VBG</td>\n",
       "      <td>CD</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27644</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Shortly after announcing a hardware upgrade fo...</td>\n",
       "      <td>Google Glass Gets an Accessory Store</td>\n",
       "      <td>['Gadgets', 'glass', 'Google', 'Google Glass',...</td>\n",
       "      <td>tech</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27645</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Well, that was quick. Just hours after going o...</td>\n",
       "      <td>OUYA Gaming Console Already Sold Out on Amazon</td>\n",
       "      <td>['amazon', 'amazon kindle', 'Business', 'Gaming']</td>\n",
       "      <td>business</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>...</td>\n",
       "      <td>IN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27646</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Between Two Ferns: Oscar Buzz Edition Part 1 ...</td>\n",
       "      <td>'Between Two Ferns' Mocks Oscar Nominees</td>\n",
       "      <td>['Between Two Ferns', 'Movies', 'The Oscars', ...</td>\n",
       "      <td>film</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NNS</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27647</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Ever since The Hurt Locker it seems like we h...</td>\n",
       "      <td>'American Sniper' Trailer: Looks Like Eastwood...</td>\n",
       "      <td>['American Sniper', 'Awards', 'Bradley Cooper'...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NN</td>\n",
       "      <td>MD</td>\n",
       "      <td>VB</td>\n",
       "      <td>RB</td>\n",
       "      <td>IN</td>\n",
       "      <td>DT</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                            content  \\\n",
       "0  27643  <html><head><div class=\"article-info\"><span cl...   \n",
       "1  27644  <html><head><div class=\"article-info\"><span cl...   \n",
       "2  27645  <html><head><div class=\"article-info\"><span cl...   \n",
       "3  27646  <html><head><div class=\"article-info\"><span cl...   \n",
       "4  27647  <html><head><div class=\"article-info\"><span cl...   \n",
       "\n",
       "                                             article  \\\n",
       "0   Note to humanity: One Direction fandom ain't ...   \n",
       "1  Shortly after announcing a hardware upgrade fo...   \n",
       "2  Well, that was quick. Just hours after going o...   \n",
       "3   Between Two Ferns: Oscar Buzz Edition Part 1 ...   \n",
       "4   Ever since The Hurt Locker it seems like we h...   \n",
       "\n",
       "                                               title  \\\n",
       "0  Soccer Star Gets Twitter Death Threats After T...   \n",
       "1               Google Glass Gets an Accessory Store   \n",
       "2     OUYA Gaming Console Already Sold Out on Amazon   \n",
       "3           'Between Two Ferns' Mocks Oscar Nominees   \n",
       "4  'American Sniper' Trailer: Looks Like Eastwood...   \n",
       "\n",
       "                                              topics   data_channel  see_also  \\\n",
       "0  ['Entertainment', 'Music', 'One Direction', 's...  entertainment         1   \n",
       "1  ['Gadgets', 'glass', 'Google', 'Google Glass',...           tech         1   \n",
       "2  ['amazon', 'amazon kindle', 'Business', 'Gaming']       business         1   \n",
       "3  ['Between Two Ferns', 'Movies', 'The Oscars', ...           film         1   \n",
       "4  ['American Sniper', 'Awards', 'Bradley Cooper'...  entertainment         1   \n",
       "\n",
       "   p_year  p_month  p_day  ...  title_pos_tag_7  title_pos_tag_8  \\\n",
       "0  2013.0      9.0    9.0  ...               IN              VBG   \n",
       "1  2013.0     10.0   31.0  ...             NONE             NONE   \n",
       "2  2013.0      6.0   25.0  ...               IN               NN   \n",
       "3  2013.0      2.0   13.0  ...              NNS             NONE   \n",
       "4  2014.0     10.0    3.0  ...               NN               MD   \n",
       "\n",
       "   title_pos_tag_9  title_pos_tag_10 title_pos_tag_11  title_pos_tag_12  \\\n",
       "0               CD                NN               NN              NONE   \n",
       "1             NONE              NONE             NONE              NONE   \n",
       "2             NONE              NONE             NONE              NONE   \n",
       "3             NONE              NONE             NONE              NONE   \n",
       "4               VB                RB               IN                DT   \n",
       "\n",
       "   title_pos_tag_13  title_pos_tag_14 title_pos_tag_15  num_topics  \n",
       "0              NONE              NONE             NONE          63  \n",
       "1              NONE              NONE             NONE          70  \n",
       "2              NONE              NONE             NONE          49  \n",
       "3              NONE              NONE             NONE          94  \n",
       "4                NN              NONE             NONE         104  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv(p_test)\n",
    "df_test.rename(\n",
    "    columns={'Id': 'id', 'Page content': 'content'}, \n",
    "    inplace=True\n",
    ")\n",
    "if 'Unnamed: 0' in df_test.columns:\n",
    "    df_test.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text_to_clean, no_punc=True):\n",
    "    res = text_to_clean.lower()\n",
    "    res = re.sub(r\"i'm\", \"i am\", res)\n",
    "    res = re.sub(r\"he's\", \"he is\", res)\n",
    "    res = re.sub(r\"she's\", \"she is\", res)\n",
    "    res = re.sub(r\"it's\", \"it is\", res)\n",
    "    res = re.sub(r\"that's\", \"that is\", res)\n",
    "    res = re.sub(r\"what's\", \"what is\", res)\n",
    "    res = re.sub(r\"where's\", \"where is\", res)\n",
    "    res = re.sub(r\"how's\", \"how is\", res)\n",
    "#     res = re.sub(r\"\\'s\", \" is\", res)\n",
    "    res = re.sub(r\"\\'ll\", \" will\", res)\n",
    "    res = re.sub(r\"\\'ve\", \" have\", res)\n",
    "    res = re.sub(r\"\\'re\", \" are\", res)\n",
    "    res = re.sub(r\"\\'d\", \" would\", res)\n",
    "    res = re.sub(r\"\\'re\", \" are\", res)\n",
    "    res = re.sub(r\"won't\", \"will not\", res)\n",
    "    res = re.sub(r\"can't\", \"cannot\", res)\n",
    "    res = re.sub(r\"n't\", \" not\", res)\n",
    "    res = re.sub(r\"n'\", \"ng\", res)\n",
    "    res = re.sub(r\"'bout\", \"about\", res)\n",
    "    res = re.sub(r\"'til\", \"until\", res)\n",
    "    \n",
    "    if no_punc:\n",
    "        res = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|.!?,]\", \"\", res)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = stopwords.words('english')\n",
    "\n",
    "def preprocessor(text):\n",
    "    porter = PorterStemmer()\n",
    "    res = clean_text(text)\n",
    "    \n",
    "    tokens = [porter.stem(w) for w in re.split('\\s+', res.strip()) \\\n",
    "        if w not in stop_words and re.match('[a-zA-Z]+', w)]\n",
    "    \n",
    "    return ' '.join(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample popular & unpopular news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/laura-vitto/\"><img alt=\"2016%2f09%2f21%2f3f%2flauravittoheadshotslowres1.b1191\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/OAcgWfEl5UfjasprN9z1m4SEIXo=/90x90/2016%2F09%2F21%2F3f%2FLauraVittoheadshotslowres1.b1191.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/laura-vitto/\">Laura Vitto</a></span><time datetime=\"Tue, 28 Jan 2014 20:15:47 +0000\">2014-01-28 20:15:47 UTC</time></span></div></head><body><h1 class=\"title\">Vin Diesel's 7-Minute Dance Video Is His Greatest Work Yet</h1><figure class=\"article-image\"></figure><article data-channel=\"entertainment\"><section class=\"article-content\"> <p>Vin Diesel is an actor, producer and, apparently, a huge Beyoncé fan.</p> <p>The <em>Fast and the Furious</em> star took to his <a href=\"http://mashable.com/category/facebook/\" target=\"_blank\">Facebook</a> page on Tuesday to share his latest (and possibly greatest) piece of work. The self-recorded clip shows the muscle-bound star showing off his best moves to Katy Perry's \"Dark Horse\" and Beyoncé's \"Drunk in Love\".</p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2014/01/23/beyonce-parody-morning-face/\">Beyoncé Parody Shows Just How Flawless You Look in the Morning</a></p></div> <div id=\"fb-root\"></div> <div class=\"fb-post\" data-href=\"https://www.facebook.com/photo.php?v=10152278510183313\" data-width=\"466\"><div class=\"fb-xfbml-parse-ignore\"><p><a href=\"https://www.facebook.com/photo.php?v=10152278510183313\" target=\"_blank\">Post</a> by <a href=\"https://www.facebook.com/VinDiesel\" target=\"_blank\">Vin Diesel</a>.</p></div></div> <p>Diesel's 63 million Facebook followers are loving the clip, which has amassed nearly 54,000 comments and 500,000 likes in the seven hours since its posting. Others took to Twitter to share their comments.</p> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>Still laughing about how Vin Diesel pantomimed riding a SURFBORDT.</p> <p>— pam.mie.pie (@ohnopdidnt) <a href=\"https://twitter.com/ohnopdidnt/statuses/428238259747950592\" target=\"_blank\">January 28, 2014</a></p> </blockquote> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>the most disturbing thing about vin diesel's video is the fact that it's obvious he's holding back.</p> <p>— C.R. (@ChristinaRpr) <a href=\"https://twitter.com/ChristinaRpr/statuses/428237792921931776\" target=\"_blank\">January 28, 2014</a></p> </blockquote> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>Every time I get a moment, I can't help but to press play. Vin Diesel, dancing to Drunk In Love, complete with SERFBORT ad-libbing. hahahaha</p> <p>— Brooke Thomas (@BrookeOnAir) <a href=\"https://twitter.com/BrookeOnAir/statuses/428147847691632640\" target=\"_blank\">January 28, 2014</a></p> </blockquote> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>Speaking of mirror stage, I'll be retweeting any GIFs of Vin Diesel dancing alone in front of a webcam, indiscrimintely all day.</p> <p>— Maureen O'Connor (@maureenoco) <a href=\"https://twitter.com/maureenoco/statuses/428203259325874176\" target=\"_blank\">January 28, 2014</a></p> </blockquote> <p>Clocking in at more than seven minutes, the video is packed with GIFable moments. Our personal favorite: Diesel channeling Elaine Benes at the 6:08 mark.</p> <figure class=\"image\"><p><img alt=\"elaine\" class=\"fullwidth\" data-caption=\"\" data-credit-link=\"http://topshelfedition.tumblr.com/\" data-credit-name=\"Tumblr, topshelfedition\" data-credit-provider=\"Select\" data-fragment=\"m!5b21\" data-image=\"http://i.amz.mshcdn.com/HVp9n8kmGgLJsVo5C_nGO8mWS8Y=/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2014%2F01%2Felaine.gif\" data-micro=\"1\" src=\"http://i.amz.mshcdn.com/Uo9QYgcIT_jmdrv0EGnmoom5k_A=/fit-in/1200x9600/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2014%2F01%2Felaine.gif\" title=\"elaine\"/></p> <div class=\"image-credit\"><p>Image: <a href=\"http://topshelfedition.tumblr.com/\" target=\"_blank\">Tumblr, topshelfedition</a></p></div></figure><p><strong><div class=\"http://mashable.com/2013/08/08/beyonce-hair-gifs/\">BONUS: <a href=\"http://mashable.com/2013/08/08/beyonce-hair-gifs/\" target=\"_blank\">A Tribute to Beyonce's Hair in 25 GIFs</a> </div></strong></p> <section class=\"gallery\" data-display-mode=\"gallery\" data-id=\"11945\" data-slide-title-pos=\"\" data-slug=\"beyonces-hair-gifs\"> <header> <h1>Beyonce's Hair GIFs</h1> </header> <ol class=\"slides\"> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a051984006a200b887\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/7hqC9S9zl5kF94PhDqUtA3vGgSA=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-hair-1.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/ov0mrlQ-I0Q8_xFeErBnRreYg98=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-hair-1.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a051984006a200b888\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/uQybEu6sczXe9DDdTSyZvTscqrI=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-hair-2.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/DSWBF7VDUBqNgYzVrH-G_92IffY=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-hair-2.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a051984006a200b889\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/CgjxkI7RaKiI6WDTxlpP_qom2m4=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fthe-live-world.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/mcKEGoIEnm27ejsuzRE74otXcDc=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fthe-live-world.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://the-live-world.tumblr.com/\" target=\"_blank\">Tumblr, the-live-world</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a051984006a200b88a\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/L5WSyMm-6JEpLLhVw_4xSpchYpE=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fponytail.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/tvzUz3VLbkXYZFItafk9eZvvYbY=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fponytail.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a051984006a200b88b\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/G6y0Y8LxqvVck9xxyvyKqiA37zE=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fteasin.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/bhulx6bwL81wpSvLUBtmGptOxbM=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fteasin.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a051984006a200b88c\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/RwaoiQJcjAQ1OE2MXrb_op9Ldxg=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-hair-flip.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/b5dkjuEDeue13rlhzxB6P5ozj-E=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-hair-flip.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b88d\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/-FL4lDKAVepvRXFkbnmpiihb1DI=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-flick-bey.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/K_KVmkS-RgNvEECN_p38qd3xxgI=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-flick-bey.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://popmusicgifs.tumblr.com/\" target=\"_blank\">Tumblr, popmusicgifs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b88e\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/mLFXeYXp6ZTpIrCyNd1t5lroX3U=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fcurly-afro-bey.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/h0wiUm5aBW9qau1cllOH7iYiDlQ=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fcurly-afro-bey.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"Tumblr,%20fiercees\" target=\"_blank\">http://fiercees.tumblr.com/</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b88f\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/tnts6B5kFek3EuMeb7wI1ff7ka8=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Flong-hair-bey.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/7BWo4BZRtBCnvhzAbjtYo1xJgGI=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Flong-hair-bey.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://popmusicgifs.tumblr.com/\" target=\"_blank\">Tumblr, popmusicgifs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b890\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/NSdN73D99Hz4ZrcIcysAE6D2PTo=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-fan.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/fbUKIGzOcda25gD9zNNP4Yo1gbg=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-fan.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"Tumblr,%20motionlmags\" target=\"_blank\">http://motionlmags.tumblr.com/</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b891\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/2bUbaTH441FeOpITBBTeWBATp8o=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fcrazy-in-love.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/TJoMAOz4ld2VfFG35A0iZ1fT0II=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fcrazy-in-love.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://snapshotof.tumblr.com/\" target=\"_blank\">Tumblr, snapshotof</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b892\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/rRzBqvO_hzYwBxL3stf_VjvAVAI=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fswing.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/psqamfSZ-tTNbtuXlgxRKq4wyy8=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fswing.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b893\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/FDcnMd3ARGjpS1-PHuH1ghG45cA=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fheadbanging.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/pNS1m89NktOxvc5AFaFH0nTs9JU=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fheadbanging.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyonceknowlesgifs.tumblr.com/\" target=\"_blank\">Tumblr, beyonceknowlesgifs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b894\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/jZD3BQ9XeR47-JEz-K-XuQML3RA=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fheadbang.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/-bLvPwbtFlyQHrUoZrxefkh3Xzs=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fheadbang.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://loljam.com/post/5852/\" target=\"_blank\">loljam</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b895\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/NfHqfAJB_uVH_vuhIo1u3eBOQos=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-swinging.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/Qkp-Hnw3mFaYbK7I17TofaBatCM=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-swinging.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://thegiflibrary.tumblr.com\" target=\"_blank\">Tumblr, The GIF Library</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b896\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/n3oiFRSWpfwLRLca17ixWNi45Ks=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Flong-straight-hair.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/zEJCJ6zCNngVj13-gbuf_853wBI=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Flong-straight-hair.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyonceblue.tumblr.com/\" target=\"_blank\">Tumblr, beyonceblue</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b897\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/CGf8hOb2nVZFenxCvw8nX3XYpRM=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fpeace-sign.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/D_XaZPyjRHlPCjGXorRB8oFFsv8=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fpeace-sign.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b898\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/Esi6Z6ZdPX-knSFqmftNhnBQhvA=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fgreen-bey.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/2SxlcwvKLiNSpcaT5Fo1p7NqcJQ=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fgreen-bey.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b899\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/jAPpcJGLWX3aLj6uLbfi8AMF1w8=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Ffro-flick.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/IFx_CzE8qo-m0c-fiMOax0ZvEic=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Ffro-flick.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b89a\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/aaPXiymQve3Z8rIh5hKioLYYK7U=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-touch.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/-9zQW45Ch8Q2fE2tpZqMsx4s2D0=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-touch.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://www.reactiongifs.com/\" target=\"_blank\">Reaction GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b89b\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/HYxI-d3dA29I4mjrEZvHJa9c87E=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-bangs.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/RB6AUzhZuOR_tO6JM5AmQC-uyhE=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fbeyonce-bangs.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoftheday.tumblr.com/\" target=\"_blank\">Tumblr, beyoftheday</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b89c\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/7M6ONIHbg17zrgXq9Vm-Grj96ms=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Ffans.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/4YcHVcB7sSpd-KlZ9evkr1Rf-x4=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Ffans.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyonceblue.tumblr.com/\" target=\"_blank\">Tumblr, beyonceblue</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b89d\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/SiIwVmDgKR8e4D-4pkg6ruCOje8=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Flong-hair.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/g_y5J4qfCdsMTi6MufZbV1n6W5w=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Flong-hair.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b89e\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/gYdcG-DZ3ASMJL8DLvbbW7r0ekE=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fdark-hair.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/Dg1QNYWfWpK_m58XqZVxZ4pDphA=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fdark-hair.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image - GIF\" data-id=\"533873a151984006a200b89f\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/A2mKvmiUon1etTASkvYlMgvO9CY=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-fix.gif\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/ZzPk8XAA1KBZ8jpc-DL0RzaXwbc=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fbeyonces-hair-gifs%2Fhair-fix.gif\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\"></h2> <div class=\"caption\"></div> <div class=\"credit\"> Image: <a href=\"http://beyoncegifs.tumblr.com/\" target=\"_blank\">Tumblr, Beyonce GIFs</a> </div> </div> </li> </ol> </section> <script>      window._msla=window.loadScriptAsync||function(src,id){if(document.getElementById(id))return;var js=document.createElement('script');js.id=id;js.src=src;document.getElementsByTagName('script')[0].parentNode.insertBefore(js,fjs);}; _msla(\"//connect.facebook.net/en_US/all.js#appId=116628718381794&xfbml=1&status=1&cookie=1\",\"facebook_jssdk\");_msla(\"//platform.twitter.com/widgets.js\",\"twitter_jssdk\");</script> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/entertainment/\">Entertainment</a>, <a href=\"/category/facebook/\">Facebook</a>, <a href=\"/category/film/\">Movies</a>, <a href=\"/category/social-media/\">Social Media</a>, <a href=\"/category/video/\">Video</a>, <a href=\"/category/videos-watercooler/\">Videos</a>, <a href=\"/category/vin-diesel/\">vin diesel</a>, <a href=\"/category/viral-video/\">viral video</a>, <a href=\"/category/watercooler/\">Watercooler</a> </footer></body></html>"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/connor-finnegan/\"><img alt=\"2016%2f06%2f30%2fb0%2f201503270cheadshot_20.5abbf.2d457\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/utF-AtG0D7uDgdK1Z4Ak7Ypvir0=/90x90/2016%2F06%2F30%2Fb0%2F201503270cHeadshot_20.5abbf.2d457.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/connor-finnegan/\">Connor Finnegan</a></span><time datetime=\"Tue, 10 Dec 2013 23:17:47 +0000\">2013-12-10 23:17:47 UTC</time></span></div></head><body><h1 class=\"title\">Vote for Your Favorite TV Show of 2013</h1><figure class=\"article-image\"><img class=\"microcontent\" data-fragment=\"lead-image\" data-image=\"http://i.amz.mshcdn.com/KVuYINwWuMBc-jh1xbrwgoBig0Y=/950x534/2013%2F12%2F10%2F24%2Ftopshows.824b1.jpg\" data-micro=\"1\" data-url=\"null\" src=\"http://i.amz.mshcdn.com/KVuYINwWuMBc-jh1xbrwgoBig0Y=/950x534/2013%2F12%2F10%2F24%2Ftopshows.824b1.jpg\"/></figure><article data-channel=\"entertainment\"><section class=\"article-content\"> <p>In many ways, the debate over which <a href=\"http://mashable.com/category/tv/\">television</a> show stole the year is the toughest of the end-of-year debates. </p> <p>Do you recognize the bittersweet ending of a beloved classic like NBC's <em>30 Rock</em>, or do you give credit to shows currently hitting their full stride of hilarity, like FOX's <em>New Girl</em>? Do you prefer being able to binge-watch entire seasons of Netflix releases like <a href=\"http://mashable.com/category/orange-is-the-new-black/\"><em>Orange Is The New Black</em></a> and <a href=\"http://mashable.com/category/house-of-cards/\"><em>House of Cards</em></a>, or do you crave the suspense that comes from waiting a week for each new episode of such dramas as AMC's <em>Breaking Bad</em> and <em>The Walking Dead</em>?</p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2013/12/03/dead-actors-on-screen/\">10 Actors Who Appeared on Screen After Their Deaths</a></p></div> <p>Regardless of your criteria, we're curious to hear what kept you glued to the screen in 2013. </p> <p>We rounded up 35 of the year's most widely-watched and critically-acclaimed shows in the poll, below. Vote for your favorite to help us determine the best show of 2013, and tell us why in the comments. You can vote for up to five shows, but if you don't see your choice feel free to add your vote in the comments. The polls will remain open all week, and we'll reveal the results Dec. 17.</p> <script charset=\"utf-8\" src=\"http://static.polldaddy.com/p/7634411.js\" type=\"text/javascript\"></script><p></p> <noscript><a href=\"http://polldaddy.com/poll/7634411/\" target=\"_blank\">What Is Your TV Show of 2013?</a></noscript> <p><strong><div class=\"bonus-content\">BONUS: <a href=\"http://mashable.com/2013/12/05/tv-characters-okcupid/\">If TV Characters Had OKCupid Profiles</a> </div></strong> </p> <section class=\"gallery\" data-display-mode=\"gallery\" data-id=\"13867\" data-slide-title-pos=\"\" data-slug=\"if-tv-characters-had-okcupid-profiles\"> <header> <h1>If TV Characters Had OKCupid Profiles </h1> </header> <ol class=\"slides\"> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"52fe6338b589e4468e0003a4\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/QGIfugcpiYLwfKSH4hKcQGCKyK4=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131202_dating-profile-sherlock.jpg\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/ZdphYOCVJcOzveCdTGywssmFgmw=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131202_dating-profile-sherlock.jpg\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">1. Sherlock Holmes from <em>Sherlock</em> </h2> <div class=\"caption\"></div> <div class=\"credit\"> Image:  <a href=\"http://www.theconnectivist.com/\" target=\"_blank\">The Connectivist</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"52fe6338b589e4468e0003a5\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/iowOzebp4zDEO4_TBaGKjsQ9prs=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131202_dating-profile-jesse.jpg\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/-4zlEEWAMf0W4TO9Ghl62YfoafM=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131202_dating-profile-jesse.jpg\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">2. Jesse Pinkman from <em>Breaking Bad</em> </h2> <div class=\"caption\"></div> <div class=\"credit\"> Image:  <a href=\"http://www.theconnectivist.com/\" target=\"_blank\">The Connectivist</a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"52fe6338b589e4468e0003a6\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/KgmLU11kY3a_tU1bSSdgr35LRJs=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131125_dating-profile-lindsay.jpg\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/eJi4ACTpJZIUIB0nJBIZMyN1ULo=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131125_dating-profile-lindsay.jpg\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">3. Lindsay Bluth Fünke from <em>Arrested Development</em> </h2> <div class=\"caption\"></div> <div class=\"credit\"> Image:  <a href=\"http://www.theconnectivist.com/\" target=\"_blank\">The Connectivist </a> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"52fe6338b589e4468e0003a7\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/HER8YZV6fSTC9J_1JSVHLwGlRN0=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131202_dating-profile-dwight.jpg\"> <figure> <img alt=\"\" src=\"http://i.amz.mshcdn.com/-xmrWAyXOYA9fFVhojCSTXFYDuw=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fif-tv-characters-had-okcupid-profiles%2F131202_dating-profile-dwight.jpg\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">4. Dwight Schrute from <em>The Office</em> </h2> <div class=\"caption\"></div> <div class=\"credit\"> Image:  <a href=\"http://www.theconnectivist.com/\" target=\"_blank\">The Connectivist </a> </div> </div> </li> </ol> </section> <p><em>Image: ABC; AMC; CBS</em></p> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/end-of-2013/\">end of 2013</a>, <a href=\"/category/entertainment/\">Entertainment</a>, <a href=\"/category/polls/\">polls</a>, <a href=\"/category/social-tv/\">Social TV</a>, <a href=\"/category/television/\">Television</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/brittany-levine/\"><img alt=\"2016%2f06%2f30%2f5d%2f201503300cheadshot_20.a69ac.90e1a\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/95Eu81GF4vPHFeM2tKbV0_9VPvQ=/90x90/2016%2F06%2F30%2F5d%2F201503300cHeadshot_20.a69ac.90e1a.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/brittany-levine/\">Brittany Levine Beckman</a></span><time datetime=\"Sun, 16 Nov 2014 03:46:24 +0000\">2014-11-16 03:46:24 UTC</time></span></div></head><body><h1 class=\"title\">Indigenous rights protesters burn Australian flags during G20 Summit</h1><figure class=\"article-image\"><img alt=\"A protester sets an Australian flag on fire.\" class=\"microcontent\" data-fragment=\"lead-image\" data-image=\"http://i.amz.mshcdn.com/F3owpv62Vx7SlBSyrTuAJCSHsTk=/950x534/2014%2F11%2F16%2Fed%2Fflagburning.1cdea.jpg\" data-micro=\"1\" data-url=\"null\" src=\"http://i.amz.mshcdn.com/F3owpv62Vx7SlBSyrTuAJCSHsTk=/950x534/2014%2F11%2F16%2Fed%2Fflagburning.1cdea.jpg\"/><figcaption class=\"image-caption\">A protester sets an Australian flag on fire.</figcaption><div class=\"image-credit\">Image: @JoelDry9</div></figure><article data-channel=\"world\"><section class=\"article-content\"> <p>Indigenous rights protesters burned several Australian flags during a protest in Brisbane as the G20 Summit continued on Sunday.</p> <p>With Australia in the international spotlight this weekend due to the political forum, indigenous protesters aired their disgust with Prime Minister Tony Abbott, who had downplayed the presence of Aboriginal people during the country's early days. </p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2014/11/15/obama-ban-ki-moon-climate-g20/\">Obama and Ban Ki-moon didn't get the memo that climate is not on G20 agenda</a></p></div> <p>Protesters chanted \"resist, revive, decolonize\" as the flags went up in flames, according to the <em><a href=\"http://www.brisbanetimes.com.au/queensland/brisbane-g20/g20-protesters-burn-australian-flag-20141116-11noxu.html\" target=\"_blank\">Brisbane Times</a></em>. </p> <p>Protesters had called for an end to the deaths of Indigenous Australians in police custody during demonstrations a few days prior.</p> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>Indigenous protest group burns the Australian flag at <a href=\"https://twitter.com/hashtag/Brisbane?src=hash\" target=\"_blank\">#Brisbane</a> <a href=\"https://twitter.com/hashtag/g20?src=hash\" target=\"_blank\">#g20</a> rally <a href=\"https://twitter.com/abcnews\" target=\"_blank\">@abcnews</a> <a href=\"http://t.co/ZaGCS97XJ1\" target=\"_blank\">pic.twitter.com/ZaGCS97XJ1</a></p> <p>— Alyse Edwards (@Alyse_Edwards) <a href=\"https://twitter.com/Alyse_Edwards/status/533811183711571969\" target=\"_blank\">November 16, 2014</a></p> </blockquote> <p>Flag burning isn't illegal in Australia. Lawmakers unsuccessfully attempted to ban the practice in 2003 and 2008.</p> <p>The roughly 50 indigenous protesters also lit up effigies of Australian Aboriginal leaders. According to news reports, the demonstrators called the leaders sellouts. </p> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>Effigies of Noel Pearson, Warren Mundine burned by indigenous protest group at <a href=\"https://twitter.com/hashtag/BrisbaneG20?src=hash\" target=\"_blank\">#BrisbaneG20</a> demonstration <a href=\"https://twitter.com/abcnews\" target=\"_blank\">@abcnews</a> <a href=\"http://t.co/94ecmEKFhi\" target=\"_blank\">pic.twitter.com/94ecmEKFhi</a></p> <p>— Alyse Edwards (@Alyse_Edwards) <a href=\"https://twitter.com/Alyse_Edwards/status/533812692838920192\" target=\"_blank\">November 16, 2014</a></p> </blockquote> <p>The provocative rally came just hours before the summit's closing speeches.</p> <p>Demonstrators allied to many causes—from climate change to freeing Tibet— have been a common sight in Brisbane during the summit despite the extreme heat.</p> <p>Thousands of police have been on hand to keep the peace.</p> <script>      window._msla=window.loadScriptAsync||function(src,id){if(document.getElementById(id))return;var js=document.createElement('script');js.id=id;js.src=src;document.getElementsByTagName('script')[0].parentNode.insertBefore(js,fjs);}; _msla(\"//platform.twitter.com/widgets.js\",\"twitter_jssdk\");</script> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/australia/\">Australia</a>, <a href=\"/category/g20/\">g20</a>, <a href=\"/category/world/\">World</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/vignesh-ramachandran/\"><img alt=\"Default-m\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/raEPzavg65Jzvjy-L4U699QBlmQ=/90x90/default-m.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/vignesh-ramachandran/\">Vignesh Ramachandran</a></span><time datetime=\"Fri, 30 Aug 2013 00:49:06 +0000\">2013-08-30 00:49:06 UTC</time></span></div></head><body><h1 class=\"title\">This Basketball Will Coach You With Audible Feedback</h1><figure class=\"article-image\"></figure><article data-channel=\"sports\"><section class=\"article-content\"> <div class=\"shift-to-hero\"><div class=\"content-mash-video\" data-autoplay=\"\" data-embedurl=\"http://mashable.com/videos/embed?video=xVnmo4vC&amp;player=offsite\" data-labels='[\"basketball\", \"kickstarter\", \"newsy\", \"sensor\", \"sports\", \"tech\"]' data-playlist=\"http://content.jwplatform.com/manifests/xVnmo4vC.m3u8\" data-template=\"postlead\" data-thumbnail=\"http://content.jwplatform.com/thumbs/xVnmo4vC.jpg\" data-title=\"Smart Basketball Could Be Your New Coach\" data-video=\"2630091127001\"></div></div> <p>If you're looking to step up your basketball game, you can now consider swapping your regular ball for one packed with smart capabilities.</p> <p><a href=\"http://www.shootersrevolution.com/\" target=\"_blank\">Shooters Revolution</a> has designed the EVO ONE smart basketball, which looks like a regular ball but has micro-sensor technology. The ball's sensor serves as a coach to help you improve your shooting technique, as explained in the video above.</p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2012/06/07/youtube-epic-nba-dunks/\">Posterized! YouTube's 12 Most Epic NBA Dunks</a></p></div> <p><img alt=\"EvoOneBasketball2\" class=\"\" data-fragment=\"m!e92d\" data-image=\"http://i.amz.mshcdn.com/iNwCsVyp0cbLIV-pp_IGiyQh87c=/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2013%2F08%2FEvoOneBasketball2.jpg\" data-micro=\"1\" src=\"http://i.amz.mshcdn.com/ZoT4qiRWoW0dF65zGBpHSjbHo4E=/fit-in/1200x9600/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2013%2F08%2FEvoOneBasketball2.jpg\"/></p> <p>Once you activate the basketball's micro-sensor, the technology can measure the backspin rate, axes and balance of the ball when you shoot it. The device's micro-speaker will emit audible feedback, if you achieve optimal backspin of plus-or-minus two revolutions per second. \"Wobbly, slow, or unorthodox backspin is not reinforced through audible feedback,\" writes the company on its Kickstarter page.</p> <p>The battery-powered sensor is removable and the basketball can be used normally. Shooters Revolution says the EVO ONE ball satisfies regulation weight and size standards.</p> <p>The company, headed by former athletes, is trying to raise $45,000 <a href=\"http://www.kickstarter.com/projects/shootersrev/evo-onetm-sensorized-basketball-performance-techno\" target=\"_blank\">on Kickstarter</a> to finalize the EVO ONE's development. With 19 days left in the Kickstarter campaign at time of writing, backers had already pledged more than $22,400. A limited number of backers can get their own EVO ONE for $59. The ball comes in an array of sizes.</p> <p>What do you think about this smart basketball? Share your thoughts in the comments.</p> <p><em>Image: Shooters Revolution <a href=\"http://www.kickstarter.com/projects/shootersrev/evo-onetm-sensorized-basketball-performance-techno\" target=\"_blank\">via Kickstarter</a></em></p> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/basketball/\">basketball</a>, <a href=\"/category/coach/\">coach</a>, <a href=\"/category/health-fitness/\">Health &amp; Fitness</a>, <a href=\"/category/kickstarter/\">kickstarter</a>, <a href=\"/category/lifestyle/\">Lifestyle</a>, <a href=\"/category/newsy/\">Newsy</a>, <a href=\"/category/sports/\">Sports</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      " The thin bar at the top of the screen indicates page load progress.\" src=\"http://i.amz.mshcdn.com/rXyfcGt8a0ZtQjHG9O80O_FX8uk=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-27.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Searching and Loading a Page</h2> <div class=\"caption\"> <p>Typing in a URL or string of text offers up real-time suggestions for an address or search query.</p> <p>The thin bar at the top of the screen indicates page load progress.</p> </div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a412d2cd7caa00041e\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/qz2MBFzxeRYNxSYywAmLoB6bNFU=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-28.png\"> <figure> <img alt=\"It's like Cover Flow, but vertical!\" src=\"http://i.amz.mshcdn.com/jddDVA_pmUzvW9ZH0e8QqfXgu_Q=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-28.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Tabs</h2> <div class=\"caption\"><p>It's like Cover Flow, but vertical!</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a412d2cd7caa00041f\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/f455gjoiLlgIVzTwC5rJ5xScMJw=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-29.png\"> <figure> <img alt=\"FaceTime can now do audio calls!\" src=\"http://i.amz.mshcdn.com/zST0VWxvG407votuigp0jW5H8XA=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-29.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">FaceTime Audio</h2> <div class=\"caption\"><p>FaceTime can now do audio calls! </p></div> <div class=\"credit\"> </div> </div> </li> </ol> </section> <p><em>Image: Apple</em></p> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/apple/\">Apple</a>, <a href=\"/category/apps-software/\">Apps and Software</a>, <a href=\"/category/backup/\">backup</a>, <a href=\"/category/icloud/\">iCloud</a>, <a href=\"/category/ios-7/\">iOS 7</a>, <a href=\"/category/iphone/\">iPhone</a>, <a href=\"/category/mobile/\">Mobile</a>, <a href=\"/category/tech/\">Tech</a> </footer></body></html> for Safari is texture free but full of color.%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-30.png\"> <figure> <img alt=\"The calendar is de-leathered and has a very Google+ look.\" src=\"http://i.amz.mshcdn.com/C6s305S3_7PL7vcVeyc5lIk5R0o=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-30.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Calendar</h2> <div class=\"caption\"><p>The calendar is de-leathered and has a very Google+ look.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a412d2cd7caa000416\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/hHT6CdISROPJTU2ULMMpx2TaLiE=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-31.png\"> <figure> <img alt=\"The calendar gets a proper inbox for incoming invitations.\" src=\"http://i.amz.mshcdn.com/gO1c5wh-qzz7RaAjsO1klwujEv4=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-31.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Calendar Inbox</h2> <div class=\"caption\"><p>The calendar gets a proper inbox for incoming invitations.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a412d2cd7caa000417\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/8DKqOF_w_9eDxcTFCFugvfiuLkk=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-21.png\"> <figure> <img alt=\"Apple Maps has toned down some of its textures but the basic app looks very similar.\" src=\"http://i.amz.mshcdn.com/pH7e3ZfaXlb4IDvhEN_-ZfEYZk0=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-21.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Maps</h2> <div class=\"caption\"><p>Apple Maps has toned down some of its textures but the basic app looks very similar.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a412d2cd7caa000418\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/3gRBUEVpTQmufKfa2oLT5weeXVw=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-22.png\"> <figure> <img alt=\"The directions page in Maps has been rebuilt and it looks much more sleek.\" src=\"http://i.amz.mshcdn.com/iWFOUY0a8Fut9vkgobH7mCWQUF0=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-22.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Directions</h2> <div class=\"caption\"><p>The directions page in Maps has been rebuilt and it looks much more sleek.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a412d2cd7caa000419\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/bTn1_M_cjcZ6NFkdCexxGaJQhLE=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-23.png\"> <figure> <img alt=\"The calculator still pays homage to the great HP calculators of the 70s, but with fewer bevels.s-on-with-ios-7%2Fios7-12.png\"> <figure> <img alt=\"The App Store structure is still the same but the look is more fresh. In addition to the general charts, users can also check to see what apps are popular in their area.\" src=\"http://i.amz.mshcdn.com/uIBP-fJrXpMdqMWXFJxEdPRE4oI=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-12.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">App Store</h2> <div class=\"caption\"><p>The App Store structure is still the same but the look is more fresh. In addition to the general charts, users can also check to see what apps are popular in their area.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a312d2cd7caa00040c\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/ME2Kxh2tyXJUxybK78BhrQL3Rds=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-13.png\"> <figure> <img alt=\"A new share button shows off what you can do with an app which includes AirDrop and adding it to a wishlist.\" src=\"http://i.amz.mshcdn.com/Ij_0VyDTYzGh99-KmLR4MlxdA3A=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-13.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Sharing Or Adding to a Wishlist</h2> <div class=\"caption\"><p>A new share button shows off what you can do with an app which includes AirDrop and adding it to a wishlist.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a312d2cd7caa00040d\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/HSH9QboYzAtEHNf9Kba5wir18ZM=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-14.png\"> <figure> <img alt=\"Both look fresh.\" src=\"http://i.amz.mshcdn.com/6mLuKkeCaLaNI-LRVw7lxFF-PAQ=/fit-in/850x850/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-14.png\"> </img></figure> <div class=\"meta\"> <h2 class=\"title\">Passbook and News Stand</h2> <div class=\"caption\"><p>Both look fresh.</p></div> <div class=\"credit\"> </div> </div> </li> <li class=\"slide\" data-content-source=\"Gallery - Image\" data-id=\"524c70a312d2cd7caa00040e\" data-skip-rerender=\"false\" data-thumb=\"http://i.amz.mshcdn.com/3LJ1zALRWkb5BpTpFYAgG2lWX04=/180x140/http%3A%2F%2Fmashable.com%2Fwp-content%2Fgallery%2Fhands-on-with-ios-7%2Fios7-15.png\"> <figure> <img alt=\"The Lock screen no longer has anything to slide to unlock, just swipe to the right.\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for c in pop_group.get_group(1).sample(n=5, random_state=1)['content']:\n",
    "    print(c)\n",
    "    print('\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><head><div class=\"article-info\"><span class=\"byline basic\"><span class=\"author_name\">By <a href=\"/author/colin-gorenstein/\">Colin Gorenstein</a></span><time datetime=\"Tue, 30 Sep 2014 16:50:49 +0000\">2014-09-30 16:50:49 UTC</time></span></div></head><body><h1 class=\"title\">Mindy Kaling and Elmo Are Undeniably Enthusiastic</h1><figure class=\"article-image\"></figure><article data-channel=\"watercooler\"><section class=\"article-content viral-video\"> <div class=\"viral-video-lead\"> <p><iframe allowfullscreen=\"\" frameborder=\"0\" height=\"360\" src=\"https://www.youtube.com/embed/em5SSa-IVUU?enablejsapi=1&amp;\" width=\"640\"></iframe></p> <script src=\"http://a.amz.mshcdn.com/assets/lib/aab-7ce243b38b9cc2caec816aff811d3153.js\" type=\"text/javascript\"></script> </div> <p>Elmo is really enthusiastic about dancing with friends. <a href=\"http://mashable.com/category/mindy-kaling/\">Mindy Kaling</a> is really enthusiastic about dancing with chickens. Chickens are, evidently, really enthusiastic about jumping.</p> <p>The crew has to meet somewhere in the middle, so the final compromise is to jump to music. </p> <p>For anyone who's dreamed of a dance party filled with chickens and Mindy Kaling (really, who hasn't?), your moment has come.</p> <figure class=\"image\"><p><img alt=\"mindy2\" class=\"fullwidth\" data-caption=\"\" data-credit-link=\"\" data-credit-name=\"\" data-credit-provider=\"Youtube\" data-fragment=\"m!e10a\" data-image=\"http://i.amz.mshcdn.com/Ed7F5KCsWgoMVwrXzm7R3zGAhjE=/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2014%2F09%2Fmindy2.gif\" data-micro=\"1\" src=\"http://i.amz.mshcdn.com/Vr_q8TL7w7dqFX_Wq3nNYrc_dqo=/fit-in/1200x9600/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2014%2F09%2Fmindy2.gif\" title=\"mindy2\"/></p> <div class=\"image-credit\"></div></figure><figure class=\"image\"><p><img alt=\"mindy1\" class=\"fullwidth\" data-caption=\"\" data-credit-link=\"\" data-credit-name=\"\" data-credit-provider=\"\" data-fragment=\"m!35ef\" data-image=\"http://i.amz.mshcdn.com/2phG-o7GZNNGEZbV3jlRlK0UYiE=/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2014%2F09%2Fmindy1.gif\" data-micro=\"1\" src=\"http://i.amz.mshcdn.com/mZfc_wqASQNVi1HX_JS366Fw9Jg=/fit-in/1200x9600/http%3A%2F%2Fmashable.com%2Fwp-content%2Fuploads%2F2014%2F09%2Fmindy1.gif\" title=\"mindy1\"/></p></figure><p><strong><div class=\"bonus-content\">BONUS: 'A Lovely Sunny Day' With Zachary Levi and Bert</div></strong></p> <p><iframe allowfullscreen=\"\" frameborder=\"0\" height=\"360\" src=\"//www.youtube.com/embed/xtvPlFty1rE?list=PLSKUhDnoJjYlXMdcdePxLzcBZl1BG1AV-\" width=\"640\"></iframe></p> <div class=\"viral-next-up\"> <div class=\"viral-video-preview story0\"><p><a href=\"http://mashable.com/2016/10/18/french-onion-barb-soup-stranger-things/?utm_cid=a-vt-nextup\"><img alt=\"Https%3a%2f%2fblueprint-api-production.s3.amazonaws.com%2fuploads%2fstory%2fthumbnail%2f24829%2f1960dc59801441feb55e2f001c201a56\" data-fragment=\"m!6cf7\" data-image=\"http://i.amz.mshcdn.com/d5XiUyUAmbIpLuniC7HIcd1N_S4=/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24829%2F1960dc59801441feb55e2f001c201a56.png\" data-micro=\"1\" data-skip-microcontent=\"true\" src=\"http://i.amz.mshcdn.com/55m25Awl05XTskH7H5dooFql80U=/410x230/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24829%2F1960dc59801441feb55e2f001c201a56.png\"/></a><a class=\"title\" href=\"http://mashable.com/2016/10/18/french-onion-barb-soup-stranger-things/\">How to make 'Stranger Things' soup, complete with floating Barb</a></p></div> <div class=\"viral-video-preview story1\"><p><a href=\"http://mashable.com/2016/10/18/emma-stone-vogue-73-questions/?utm_cid=a-vt-nextup\"><img alt=\"Https%3a%2f%2fblueprint-api-production.s3.amazonaws.com%2fuploads%2fstory%2fthumbnail%2f24813%2fscreen_shot_2016-10-18_at_12.34.08_pm\" data-fragment=\"m!a839\" data-image=\"http://i.amz.mshcdn.com/0JAAawYMPPy0Hy_RFCwmMhtUteA=/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24813%2FScreen_Shot_2016-10-18_at_12.34.08_PM.png\" data-micro=\"1\" data-skip-microcontent=\"true\" src=\"http://i.amz.mshcdn.com/Gr4kK_3PKIeQSagrdA9Vi8WKOGw=/410x230/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24813%2FScreen_Shot_2016-10-18_at_12.34.08_PM.png\"/></a><a class=\"title\" href=\"http://mashable.com/2016/10/18/emma-stone-vogue-73-questions/\">Emma Stone proves she is 'very f*cking cool' answering 73 questions from Vogue</a></p></div> <div class=\"viral-video-preview story2\"><p><a href=\"http://mashable.com/2016/10/18/first-dates-wedding-dress/?utm_cid=a-vt-nextup\"><img alt=\"Https%3a%2f%2fblueprint-api-production.s3.amazonaws.com%2fuploads%2fstory%2fthumbnail%2f24792%2fsddefault\" data-fragment=\"m!5fba\" data-image=\"http://i.amz.mshcdn.com/xo8GoMeS6SK7N_F4zag6yc_cfPc=/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24792%2Fsddefault.jpg\" data-micro=\"1\" data-skip-microcontent=\"true\" src=\"http://i.amz.mshcdn.com/f8CvWi3ztkT1nqQqR9DJvGToOPc=/410x230/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24792%2Fsddefault.jpg\"/></a><a class=\"title\" href=\"http://mashable.com/2016/10/18/first-dates-wedding-dress/\">Woman wears wedding dress on first dates, terrifies unsuspecting blokes</a></p></div> <div class=\"viral-video-preview story3\"><p><a href=\"http://mashable.com/2016/10/18/pedestrain-almost-hit-by-car/?utm_cid=a-vt-nextup\"><img alt=\"Https%3a%2f%2fblueprint-api-production.s3.amazonaws.com%2fuploads%2fstory%2fthumbnail%2f24790%2f83983a213a4049079d055ce9d4c92438\" data-fragment=\"m!51a6\" data-image=\"http://i.amz.mshcdn.com/S9hdyCimAhyJj6x6CX-aRvNVDRQ=/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24790%2F83983a213a4049079d055ce9d4c92438.png\" data-micro=\"1\" data-skip-microcontent=\"true\" src=\"http://i.amz.mshcdn.com/KJktdczWrsX-0xzFgZcfg49em3w=/410x230/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fstory%2Fthumbnail%2F24790%2F83983a213a4049079d055ce9d4c92438.png\"/></a><a class=\"title\" href=\"http://mashable.com/2016/10/18/pedestrain-almost-hit-by-car/\">Quick-thinking motorist saves pedestrian from getting hit by a car</a></p></div> </div> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/elmo/\">elmo</a>, <a href=\"/category/humor/\">humor</a>, <a href=\"/category/mindy-kaling/\">mindy kaling</a>, <a href=\"/category/sesame-street/\">sesame street</a>, <a href=\"/category/video/\">Video</a>, <a href=\"/category/videos-watercooler/\">Videos</a>, <a href=\"/category/viral-video/\">viral video</a>, <a href=\"/category/watercooler/\">Watercooler</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/chelsea-stark/\"><img alt=\"2016%2f06%2f29%2fc7%2f201506160cheadshot_20.1bc90.4faf5\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/wso9Fyhlf0QvpsKy1-Zn3puYGdo=/90x90/2016%2F06%2F29%2Fc7%2F201506160cHeadshot_20.1bc90.4faf5.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/chelsea-stark/\">Chelsea Stark</a></span><time datetime=\"Thu, 23 Oct 2014 20:17:21 +0000\">2014-10-23 20:17:21 UTC</time></span></div></head><body><h1 class=\"title\">What Happened When Felicia Day Expressed Fears Over GamerGate</h1><figure class=\"article-image\"><img alt=\"Felicia Day at Wondercon in Anaheim, California, in 2013.\" class=\"microcontent\" data-fragment=\"lead-image\" data-image=\"http://i.amz.mshcdn.com/Dlcn9DYnC8S-vgZgBlXhfbjvgIs=/950x534/2014%2F10%2F23%2F79%2Ffeliciaday.79c3e.jpg\" data-micro=\"1\" data-url=\"null\" src=\"http://i.amz.mshcdn.com/Dlcn9DYnC8S-vgZgBlXhfbjvgIs=/950x534/2014%2F10%2F23%2F79%2Ffeliciaday.79c3e.jpg\"/><figcaption class=\"image-caption\">Felicia Day at Wondercon in Anaheim, California, in 2013.</figcaption><div class=\"image-credit\">Image: Flickr, Gage Skidmore</div></figure><article data-channel=\"entertainment\"><section class=\"article-content\"> <p>Actress and geek icon Felicia Day spoke out against the Gamergate movement Wednesday night, saying she had been afraid of taking a stand on the movement because of \"self-protection and fear\" of her public details being leaked.</p> <p>Shortly after her post went up, reportedly, her fears came true. An anonymous Tumblr user posted her address and email in the Disqus comments under her post — a process known as \"doxxing\". The comments were apparently deleted by Day, but screenshots were preserved on Twitter. </p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2014/10/15/explaining-gamergate/\">Required Reading: 5 Articles to Help Demystify #Gamergate</a></p></div> <p>'Gamergate' is the name of a disparate online movement of gamers that claim to be opposed to a loosely-defined problem of \"ethics\" in games journalism. But the movement, formed in the wake of a blog post by the ex-boyfriend of a female games developer, has also been associated with violent threats against women in the industry. Feminist critic Anita Sarkeesian <a href=\"http://mashable.com/2014/10/15/anita-sarkeesian-cancels-talk-threats/\">canceled a speaking event</a> at a Utah university after an anonymous person threatened her and the school, promising \"the deadliest school shooting in American history.\" </p> <p>Day, on <a href=\"http://thisfeliciaday.tumblr.com/post/100700417809/the-only-thing-i-have-to-say-about-gamer-gate\" target=\"_blank\">her Tumblr post</a> Wednesday evening, said she had decided to remain silent despite the increasing conflict surrounding the movement.</p> <blockquote><p>I have been through a lot in my years on the internet. I have encountered a small fraction of the attacks from people like the ones who currently represent the worst of this “movement”. In the past, I worked through it alone because I felt shining a light on their words gave them exactly what they wanted: Attention and credibility. To say that their attacks and contempt didn’t set me back creatively would be a lie, but overall I got through the twists and turns, emotionally battered, but alright. My philosophy has always been, “Exist and represent yourself the way you want to exist as a woman who loves games, not as a reflection of what other people think or want of you. You will change minds by BEING. Show, don’t tell.” The attacks I experienced over the years were NOTHING compared to people who are the victims of these attacks now, but I still thought early on during the Gamer Gate phenomenon, “These trolls will dissipate into the night like they always do, it will be fine.”</p></blockquote> <p>In the Disqus comments below her post, the <em><a href=\"http://www.theguardian.com/technology/2014/oct/23/felicia-days-public-details-online-gamergate\" target=\"_blank\">Guardian</a></em> reports Day's full address was posted about an hour after her post went up. Day deleted the comment quickly, and then disabled all of Disqus, a commenting platform used on Tumblr, on her blog. </p> <p>Some Twitter users did grab <a href=\"http://imgur.com/Lrdebkq\" target=\"_blank\">screenshots</a> — with all personal details blacked out — of the comment in question, though they were made by a different username than the one reported by the <em>Guardian</em>.</p> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>I made the mistake of reading the comments on Day's <a href=\"https://twitter.com/hashtag/gamergate?src=hash\" target=\"_blank\">#gamergate</a> article. I hit \"load new comments\" and this popped up. <a href=\"http://t.co/KZOL3djOcu\" target=\"_blank\">http://t.co/KZOL3djOcu</a></p> <p>— A Man In HELL (@a_man_in_black) <a href=\"https://twitter.com/a_man_in_black/status/525067324944969728\" target=\"_blank\">October 22, 2014</a></p> </blockquote> <p>Former NFL punter and Internet star Chris Kluwe, who has recently <a href=\"https://medium.com/the-cauldron/why-gamergaters-piss-me-the-f-off-a7e4c7f6d8a6\" target=\"_blank\">become</a> an outspoken opponent of Gamergate, was one of many to express outrage over the action:</p> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>Holy shit. You fucking losers in <a href=\"https://twitter.com/hashtag/Gamergate?src=hash\" target=\"_blank\">#Gamergate</a> claim it's about ethics, and then dox a woman who speaks up in her comments section? Get fucked.</p> <p>— Chris Kluwe (@ChrisWarcraft) <a href=\"https://twitter.com/ChrisWarcraft/status/525093738264031232\" target=\"_blank\">October 23, 2014</a></p> </blockquote> <p>Others suggested that targeting Day was an example of the movement's inherent misogyny:</p> <blockquote class=\"twitter-tweet\" lang=\"en\"> <p>So they doxxed Felicia Day for saying GG was bad. Didn't dox Will Wheaton. Please, tell me more about how this isn't about misogyny.</p> <p>— Christober Howlie (@cmaue) <a href=\"https://twitter.com/cmaue/status/525079072423297024\" target=\"_blank\">October 23, 2014</a></p> </blockquote> <p>Day, who has more than 2 million Twitter followers, has not posted any tweets since the incident. We've reached out to her representatives for comment.</p> <script>      window._msla=window.loadScriptAsync||function(src,id){if(document.getElementById(id))return;var js=document.createElement('script');js.id=id;js.src=src;document.getElementsByTagName('script')[0].parentNode.insertBefore(js,fjs);}; _msla(\"//platform.twitter.com/widgets.js\",\"twitter_jssdk\");</script> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/entertainment/\">Entertainment</a>, <a href=\"/category/felicia-day/\">felicia day</a>, <a href=\"/category/gamergate/\">gamergate</a>, <a href=\"/category/gaming/\">Gaming</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/sandra-gonzalez/\"><img alt=\"2016%2f06%2f29%2f62%2fhttpsd2mhye01h4nj2n.cloudfront.netmediazgkymde0lza4.a490c\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/tNdeogX8LL5K1appzIRN425c_D0=/90x90/2016%2F06%2F29%2F62%2Fhttpsd2mhye01h4nj2n.cloudfront.netmediaZgkyMDE0LzA4.a490c.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/sandra-gonzalez/\">Sandra Gonzalez</a></span><time datetime=\"Mon, 30 Jun 2014 17:21:40 +0000\">2014-06-30 17:21:40 UTC</time></span></div></head><body><h1 class=\"title\">Denis Leary Is Heading Back to TV in 'Sex&Drugs&Rock&Roll' for FX</h1><figure class=\"article-image\"><img alt=\"(From left) Denis Leary (&quot;Johnny Rock&quot;), John Corbett (&quot;Flash&quot;), John Ales (&quot;Rehab&quot;), and Robert Kelly (&quot;Bam Bam&quot;) star in FX's new comedy, &quot;Sex&amp;Drugs&amp;Rock&amp;Roll.&quot;\" class=\"microcontent\" data-fragment=\"lead-image\" data-image=\"http://i.amz.mshcdn.com/n5fF-vsjsTXOkObWlbC7P8fPgl4=/950x534/2014%2F06%2F30%2Faa%2FFXnewshow.0e813.jpg\" data-micro=\"1\" data-url=\"null\" src=\"http://i.amz.mshcdn.com/n5fF-vsjsTXOkObWlbC7P8fPgl4=/950x534/2014%2F06%2F30%2Faa%2FFXnewshow.0e813.jpg\"/><figcaption class=\"image-caption\">(From left) Denis Leary (\"Johnny Rock\"), John Corbett (\"Flash\"), John Ales (\"Rehab\"), and Robert Kelly (\"Bam Bam\") star in FX's new comedy, \"Sex&amp;Drugs&amp;Rock&amp;Roll.\"</figcaption><div class=\"image-credit\">Image:  Patrick Harbron/FX</div></figure><article data-channel=\"entertainment\"><section class=\"article-content\"> <p>Denis Leary is heading back to FX. </p> <p>The former <em>Rescue Me</em> star's show, <em>Sex&amp;Drugs&amp;Rock&amp;Roll</em>, has been picked up for 10 episodes, set to premiere in 2015, the network announced Monday. </p> <p><strong>SEE ALSO: <a href=\"http://mashable.com/2014/06/26/summer-tv-show-recommendations/\">10 New Summer Series to Keep Your TV Addiction Alive</a></strong></p> <p>The show centers on an aging rock star named Johnny Rock (Leary), who is looking for his second chance for fame 25 years after he messed up his first shot at stardom by succumbing to booze and drugs — and sleeping with his bandmate's (John Corbett) wife. And his <em>other</em> bandmate's fiancé. The show picks up as Leary is attempting to reunite his band, The Heathens. </p> <p>\"25 years later the sex might be harder to find, the drugs might be Luvox and Crestor and Cialis — but rock 'n roll will never die.  It just has a bigger prostate,\" says the logline. </p> <p>Elizabeth Gillies, Elaine Hendrix and Bobby Kelly also star on the show, which will be filmed in New York City and feature original music and guest appearances by known rock musicians.</p> <p>The show marks Leary's reunion with the network following the 2011 conclusion of firefighter drama <em>Rescue Me</em>, which in its seven seasons earned seven Emmy nominations and one win for Michael J. Fox in the guest actor category.</p> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/denis-leary/\">Denis Leary</a>, <a href=\"/category/drama/\">drama</a>, <a href=\"/category/entertainment/\">Entertainment</a>, <a href=\"/category/fx/\">fx</a>, <a href=\"/category/television/\">Television</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/jessica-catcher/\"><img alt=\"Default-m\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/raEPzavg65Jzvjy-L4U699QBlmQ=/90x90/default-m.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/jessica-catcher/\">Jessica Catcher</a></span><time datetime=\"Tue, 08 Oct 2013 03:26:39 +0000\">2013-10-08 03:26:39 UTC</time></span></div></head><body><h1 class=\"title\">Maybe the Apple Store Can Fix the U.S. Government Shutdown</h1><figure class=\"article-image\"></figure><article data-channel=\"watercooler\"><section class=\"article-content\"> <p>Congress needs to visit the closest <a href=\"http://mashable.com/category/apple/\">Apple</a> Store stat. No stopping for donuts or bickering because the Genius Bar could be our country's only hop for a solution to the government shutdown.</p> <p>College Humor imagines what a government reboot might look like. The video shows <a href=\"http://mashable.com/category/barack-obama/\">Obama</a> going through the same steps many frustrated Apple users have dealt with when our beloved electronics let us down. Only instead of a busted iPhone, he has a busted government. </p> <p><iframe allowfullscreen=\"\" frameborder=\"0\" height=\"369\" src=\"http://www.collegehumor.com/e/6923427\" webkitallowfullscreen=\"\" width=\"600\"></iframe></p> <p>The aloof, so-called \"Genius\" doesn't help matters much as the POTUS explains that while it's an old model, it had always reliable — until he tried loading some custom programs on it. Sure he could take them off, but he shouldn't have to. \"There's a lot of European models where it works just fine!,\" the faux Obama argues.</p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2013/04/26/drawings-photographs/\">18 Drawings You'll Swear Are Actual Photographs</a></p></div> <p>Going into the first full week of the shutdown, this video puts the problem into a perspective we can all understand. Could you imagine a week without your <a href=\"http://mashable.com/category/iphone/\">iPhone</a> or Macbook?</p> <p><em>Image: <a href=\"http://www.collegehumor.com\" target=\"_blank\">College Humor</a></em></p> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/apple/\">Apple</a>, <a href=\"/category/politics/\">Politics</a>, <a href=\"/category/collegehumor/\">collegehumor</a>, <a href=\"/category/video/\">Video</a>, <a href=\"/category/videos-watercooler/\">Videos</a>, <a href=\"/category/viral-video/\">viral video</a>, <a href=\"/category/watercooler/\">Watercooler</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<html><head><div class=\"article-info\"><span class=\"byline \"><a href=\"/author/pete-pachal/\"><img alt=\"2016%2f06%2f30%2feb%2f201503270cheadshot_20.820a0.f61dd\" class=\"author_image\" src=\"http://i.amz.mshcdn.com/DLf_PsOMqPwCtvLz_lwCLCJhNBs=/90x90/2016%2F06%2F30%2Feb%2F201503270cHeadshot_20.820a0.f61dd.jpg\"/></a><span class=\"author_name\">By <a href=\"/author/pete-pachal/\">Pete Pachal</a></span><time datetime=\"Mon, 09 Jun 2014 21:44:54 +0000\">2014-06-09 21:44:54 UTC</time></span></div></head><body><h1 class=\"title\">Eye Tracking for Games Has a Name: SteelSeries Sentry</h1><figure class=\"article-image\"><img alt=\"The SteelSeries Sentry, an eye-tracking PC peripheral, will be available later in 2014.\" class=\"microcontent\" data-fragment=\"lead-image\" data-image=\"http://i.amz.mshcdn.com/9DR62Q7gnF38qY9uONipm51DfMo=/950x534/http%3A%2F%2Fadmin.mashable.com%2Fwp-content%2Fuploads%2F2014%2F06%2FHardware.jpg\" data-micro=\"1\" data-url=\"null\" src=\"http://i.amz.mshcdn.com/9DR62Q7gnF38qY9uONipm51DfMo=/950x534/http%3A%2F%2Fadmin.mashable.com%2Fwp-content%2Fuploads%2F2014%2F06%2FHardware.jpg\"/><figcaption class=\"image-caption\">The SteelSeries Sentry, an eye-tracking PC peripheral, will be available later in 2014.</figcaption><div class=\"image-credit\">Image:  SteelSeries</div></figure><article data-channel=\"gaming\"><section class=\"article-content\"> <p>To the top video-game players on the planet, fates are decided in hundredths of a second. Faster reaction time can mean the difference between finishing a level in record time, or ending up as someone's fragged lunch.</p> <p>The SteelSeries Sentry, announced Monday at <a href=\"http://mashable.com/category/e3/\">E3</a>, introduces players to a new way to improve their game: <a href=\"http://mashable.com/category/eye-tracking/\">eye tracking</a>. The product is a ruler-shaped sensor that connects to a PC's USB port, and tracks a user's eye movements. For gamers, this means they can analyze what they're looking at on the screen, for how long, and how often their gaze changes.</p> <div class=\"see-also\"><p>See also: <a href=\"http://mashable.com/2011/06/05/android-multiplayer-games/\">10 Great Android Multiplayer Games</a></p></div> <p>That last statistic, called fixations per minute (FPM), is vitally important. With the data, a serious gamer can draw correlations to actions per minute (APM). SteelSeries said it is working with some of the best gamers on the planet to establish \"peak performance goals.\" Generally, a low FPM indicates the gamer can process a lot of information, according to the company.</p> <p><iframe allowfullscreen=\"\" frameborder=\"0\" height=\"315\" src=\"//www.youtube.com/embed/Ho5lWs6ld3Q?enablejsapi=1&amp;\" width=\"560\"></iframe></p> <p>The Sentry is the first commercial product to result from a <a href=\"http://mashable.com/2014/01/02/tobii-eye-controller-gaming/\">partnership</a> between SteelSeries and <a href=\"http://mashable.com/category/tobii/\">Tobii</a>, which builds the eye-tracking technology. Although the Sentry's initial thrust will be as a training and analytics device, Tobii said it will be capable of tracking eye movement for actual gameplay in the future.</p> <p>The two companies are working with developers to build in support for eye tracking on some titles, and they're working with a third company, <a href=\"http://www.overwolf.com/\" target=\"_blank\">Overwolf</a>, to add social-media interaction. \"Staring down\" your opponent may have a whole new meaning once the Sentry really gets cooking.</p> <p>There is currently no price or release date for the Sentry yet, though Tobii said it will be available \"later this year.\" Its <a href=\"http://steelseries.com/steelseries-sentry-eye-tracker\" target=\"_blank\">product page</a> lets interested buyers sign up to be notified when the Sentry is available.</p> </section></article><footer class=\"article-topics\"> Topics: <a href=\"/category/eye-tracking/\">eye tracking</a>, <a href=\"/category/gadgets/\">Gadgets</a>, <a href=\"/category/gaming/\">Gaming</a>, <a href=\"/category/steelseries/\">SteelSeries</a>, <a href=\"/category/tech/\">Tech</a>, <a href=\"/category/tobii/\">Tobii</a> </footer></body></html>\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for c in pop_group.get_group(-1).sample(n=5, random_state=1)['content']:\n",
    "    print(c)\n",
    "    print('\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get published date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_public_datetime(content):\n",
    "    \"\"\"\n",
    "    input: raw content\n",
    "    output: article published time\n",
    "    \"\"\"\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    \n",
    "    t = soup.time\n",
    "    dt = datetime.strptime(t['datetime'], '%a, %d %b %Y %H:%M:%S %z')\n",
    "    \n",
    "    return dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def populate_public_time_data(df):\n",
    "    start = timeit.default_timer()\n",
    "    \n",
    "    for idx, r in df.iterrows():\n",
    "        dt = get_public_datetime(r['content'])\n",
    "\n",
    "        df.loc[idx, 'p_year'] = dt.year\n",
    "        df.loc[idx, 'p_month'] = dt.month\n",
    "        df.loc[idx, 'p_day'] = dt.day\n",
    "        df.loc[idx, 'p_hour'] = dt.hour\n",
    "        df.loc[idx, 'p_minute'] = dt.minute\n",
    "        df.loc[idx, 'p_second'] = dt.second\n",
    "        df.loc[idx, 'p_weekday'] = dt.weekday()\n",
    "    \n",
    "    stop = timeit.default_timer()\n",
    "    print('Time: ', stop - start)  \n",
    "    \n",
    "    display(df.head())\n",
    "    display(df.isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  184.87019739102107\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pop</th>\n",
       "      <th>content</th>\n",
       "      <th>p_year</th>\n",
       "      <th>p_month</th>\n",
       "      <th>p_day</th>\n",
       "      <th>p_hour</th>\n",
       "      <th>p_minute</th>\n",
       "      <th>p_second</th>\n",
       "      <th>p_weekday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt; &lt;span c...</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  pop                                            content  p_year  \\\n",
       "0   0   -1  <html><head><div class=\"article-info\"> <span c...  2013.0   \n",
       "1   1    1  <html><head><div class=\"article-info\"><span cl...  2013.0   \n",
       "2   2    1  <html><head><div class=\"article-info\"><span cl...  2014.0   \n",
       "3   3   -1  <html><head><div class=\"article-info\"><span cl...  2013.0   \n",
       "4   4   -1  <html><head><div class=\"article-info\"><span cl...  2014.0   \n",
       "\n",
       "   p_month  p_day  p_hour  p_minute  p_second  p_weekday  \n",
       "0      6.0   19.0    15.0       4.0      30.0        2.0  \n",
       "1      3.0   28.0    17.0      40.0      55.0        3.0  \n",
       "2      5.0    7.0    19.0      15.0      20.0        2.0  \n",
       "3     10.0   11.0     2.0      26.0      50.0        4.0  \n",
       "4      4.0   17.0     3.0      31.0      43.0        3.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "id           0\n",
       "pop          0\n",
       "content      0\n",
       "p_year       0\n",
       "p_month      0\n",
       "p_day        0\n",
       "p_hour       0\n",
       "p_minute     0\n",
       "p_second     0\n",
       "p_weekday    0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "populate_public_time_data(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  6.301303349959198\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pop</th>\n",
       "      <th>content</th>\n",
       "      <th>p_year</th>\n",
       "      <th>p_month</th>\n",
       "      <th>p_day</th>\n",
       "      <th>p_hour</th>\n",
       "      <th>p_minute</th>\n",
       "      <th>p_second</th>\n",
       "      <th>p_weekday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13516</th>\n",
       "      <td>13516</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11607</th>\n",
       "      <td>11607</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11866</th>\n",
       "      <td>11866</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25540</th>\n",
       "      <td>25540</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26611</th>\n",
       "      <td>26611</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  pop                                            content  p_year  \\\n",
       "13516  13516   -1  <html><head><div class=\"article-info\"><span cl...  2014.0   \n",
       "11607  11607   -1  <html><head><div class=\"article-info\"><span cl...  2014.0   \n",
       "11866  11866    1  <html><head><div class=\"article-info\"><span cl...  2013.0   \n",
       "25540  25540   -1  <html><head><div class=\"article-info\"><span cl...  2013.0   \n",
       "26611  26611   -1  <html><head><div class=\"article-info\"><span cl...  2014.0   \n",
       "\n",
       "       p_month  p_day  p_hour  p_minute  p_second  p_weekday  \n",
       "13516      8.0    4.0    16.0      14.0      55.0        0.0  \n",
       "11607      2.0   20.0     9.0      24.0      23.0        3.0  \n",
       "11866      4.0   24.0    17.0      43.0      38.0        2.0  \n",
       "25540     10.0   22.0    15.0      30.0      17.0        1.0  \n",
       "26611      1.0    6.0    20.0      39.0      46.0        0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "id           0\n",
       "pop          0\n",
       "content      0\n",
       "p_year       0\n",
       "p_month      0\n",
       "p_day        0\n",
       "p_hour       0\n",
       "p_minute     0\n",
       "p_second     0\n",
       "p_weekday    0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "populate_public_time_data(df_sample)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find all tags & get useful tag number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_tag(content):\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    \n",
    "    tags = set()\n",
    "    for tag in soup.find_all():\n",
    "        tags.add(tag.name)\n",
    "        \n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_df_tag(df):\n",
    "    tags = set()\n",
    "    \n",
    "    start = timeit.default_timer()\n",
    "    \n",
    "    for idx, r in df.iterrows():\n",
    "        tags = tags.union(get_all_tag(r['content']))\n",
    "        \n",
    "    stop = timeit.default_timer()\n",
    "    print('Time: ', stop - start)\n",
    "        \n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_tags = get_all_df_tag(df_sample)\n",
    "all_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a',\n",
       " 'article',\n",
       " 'b',\n",
       " 'blockquote',\n",
       " 'body',\n",
       " 'br',\n",
       " 'center',\n",
       " 'del',\n",
       " 'div',\n",
       " 'div-class',\n",
       " 'divclass',\n",
       " 'em',\n",
       " 'embed',\n",
       " 'en',\n",
       " 'figcaption',\n",
       " 'figure',\n",
       " 'footer',\n",
       " 'glowinggoodies',\n",
       " 'h1',\n",
       " 'h2',\n",
       " 'h3',\n",
       " 'head',\n",
       " 'header',\n",
       " 'hr',\n",
       " 'html',\n",
       " 'i',\n",
       " 'iframe',\n",
       " 'img',\n",
       " 'li',\n",
       " 'link',\n",
       " 'noscript',\n",
       " 'object',\n",
       " 'ol',\n",
       " 'p',\n",
       " 'param',\n",
       " 'script',\n",
       " 'section',\n",
       " 'small',\n",
       " 'span',\n",
       " 'strong',\n",
       " 'style',\n",
       " 'sub',\n",
       " 'time',\n",
       " 'ul'}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tag_num(content, tag):\n",
    "    \"\"\"\n",
    "    input: raw content\n",
    "    output: tag numbers\n",
    "    \"\"\"\n",
    "    start = timeit.default_timer()\n",
    "    \n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    \n",
    "    return len(soup.find_all(tag))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  39.66087353701005\n"
     ]
    }
   ],
   "source": [
    "start = timeit.default_timer()\n",
    "\n",
    "useful_tags = [\n",
    "    'blockquote',\n",
    "    'figcaption',\n",
    "    'figure',\n",
    "    'h1', \n",
    "    'h2',\n",
    "    'h3',\n",
    "    'iframe',\n",
    "    'img',\n",
    "    'strong'\n",
    "]\n",
    "\n",
    "for idx, r in df_sample.iterrows():\n",
    "    for tag in useful_tags:\n",
    "        col = 'num_tag_{}'.format(tag)\n",
    "        df_sample.loc[idx, col] = get_tag_num(r['content'], tag)\n",
    "    \n",
    "stop = timeit.default_timer()\n",
    "print('Time: ', stop - start)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get title of news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_title(content):\n",
    "#     soup = BeautifulSoup(content, 'html.parser')\n",
    "#     return soup.h1.string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 234.43it/s]\n"
     ]
    }
   ],
   "source": [
    "# df_sample['title'] = df_sample['content'].progress_apply(get_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27643/27643 [02:02<00:00, 226.20it/s]\n"
     ]
    }
   ],
   "source": [
    "# df_train['title'] = df_train['content'].progress_apply(get_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_title_pos_tag(title):\n",
    "    tag_pair = pos_tag(word_tokenize(clean_text(title)))\n",
    "    tags = \"\"\n",
    "    \n",
    "    for s, tag in tag_pair:\n",
    "        tags += tag + ' '\n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['title_pos_tag_seq'] = df_train['title'].progress_apply(get_title_pos_tag)\n",
    "df_test['title_pos_tag_seq'] = df_test['title'].progress_apply(get_title_pos_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_title_pos_tag_cols(df):\n",
    "    def fill(arr, i):\n",
    "        try:\n",
    "            return arr[i]\n",
    "        except:\n",
    "            return 'NONE'\n",
    "        \n",
    "    start = timeit.default_timer()\n",
    "    \n",
    "#     df['title_pos_tag_arr'] =  df['title'].apply(lambda t: get_title_pos_tag(t).split()) \n",
    "    df['title_pos_tag_arr'] =  df['title_pos_tag_seq'].apply(lambda seq: seq.split())\n",
    "    df['title_len'] =  df['title_pos_tag_seq'].apply(lambda seq: len(seq.split()))\n",
    "    MAX_LEN = 15\n",
    "        \n",
    "    for i in range(MAX_LEN):\n",
    "        col_name = 'title_pos_tag_{}'.format(i+1)\n",
    "        df[col_name] = df['title_pos_tag_arr'].apply(lambda arr: fill(arr, i))\n",
    "\n",
    "    stop = timeit.default_timer()\n",
    "    print('Time: ', stop - start)  \n",
    "    \n",
    "    display(df.head())\n",
    "    display(df.isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  0.1642745\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Popularity</th>\n",
       "      <th>Page content</th>\n",
       "      <th>article</th>\n",
       "      <th>title</th>\n",
       "      <th>topics</th>\n",
       "      <th>data_channel</th>\n",
       "      <th>see_also</th>\n",
       "      <th>p_year</th>\n",
       "      <th>p_month</th>\n",
       "      <th>...</th>\n",
       "      <th>title_pos_tag_6</th>\n",
       "      <th>title_pos_tag_7</th>\n",
       "      <th>title_pos_tag_8</th>\n",
       "      <th>title_pos_tag_9</th>\n",
       "      <th>title_pos_tag_10</th>\n",
       "      <th>title_pos_tag_11</th>\n",
       "      <th>title_pos_tag_12</th>\n",
       "      <th>title_pos_tag_13</th>\n",
       "      <th>title_pos_tag_14</th>\n",
       "      <th>title_pos_tag_15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt; &lt;span c...</td>\n",
       "      <td>There may be killer asteroids headed for Earth...</td>\n",
       "      <td>NASA's Grand Challenge: Stop Asteroids From De...</td>\n",
       "      <td>['Asteroid', 'Asteroids', 'challenge', 'Earth'...</td>\n",
       "      <td>world</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NN</td>\n",
       "      <td>IN</td>\n",
       "      <td>VBG</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Google took a stand of sorts against patent-la...</td>\n",
       "      <td>Google's New Open Source Patent Pledge: We Won...</td>\n",
       "      <td>['Apps and Software', 'Google', 'open source',...</td>\n",
       "      <td>tech</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>PRP</td>\n",
       "      <td>MD</td>\n",
       "      <td>RB</td>\n",
       "      <td>VB</td>\n",
       "      <td>IN</td>\n",
       "      <td>VBN</td>\n",
       "      <td>RB</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>You've spend countless hours training to be an...</td>\n",
       "      <td>Ballin': 2014 NFL Draft Picks Get to Choose Th...</td>\n",
       "      <td>['Entertainment', 'NFL', 'NFL Draft', 'Sports'...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>VBP</td>\n",
       "      <td>TO</td>\n",
       "      <td>VB</td>\n",
       "      <td>PRP$</td>\n",
       "      <td>JJ</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Tired of the same old sports fails and news f...</td>\n",
       "      <td>Cameraperson Fails Deliver Slapstick Laughs</td>\n",
       "      <td>['Sports', 'Video', 'Videos', 'Watercooler']</td>\n",
       "      <td>watercooler</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>At 6-foot-5 and 298 pounds, All-Pro NFL star J...</td>\n",
       "      <td>NFL Star Helps Young Fan Prove Friendship With...</td>\n",
       "      <td>['Entertainment', 'instagram', 'instagram vide...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>IN</td>\n",
       "      <td>JJ</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Popularity                                       Page content  \\\n",
       "0   0          -1  <html><head><div class=\"article-info\"> <span c...   \n",
       "1   1           1  <html><head><div class=\"article-info\"><span cl...   \n",
       "2   2           1  <html><head><div class=\"article-info\"><span cl...   \n",
       "3   3          -1  <html><head><div class=\"article-info\"><span cl...   \n",
       "4   4          -1  <html><head><div class=\"article-info\"><span cl...   \n",
       "\n",
       "                                             article  \\\n",
       "0  There may be killer asteroids headed for Earth...   \n",
       "1  Google took a stand of sorts against patent-la...   \n",
       "2  You've spend countless hours training to be an...   \n",
       "3   Tired of the same old sports fails and news f...   \n",
       "4  At 6-foot-5 and 298 pounds, All-Pro NFL star J...   \n",
       "\n",
       "                                               title  \\\n",
       "0  NASA's Grand Challenge: Stop Asteroids From De...   \n",
       "1  Google's New Open Source Patent Pledge: We Won...   \n",
       "2  Ballin': 2014 NFL Draft Picks Get to Choose Th...   \n",
       "3        Cameraperson Fails Deliver Slapstick Laughs   \n",
       "4  NFL Star Helps Young Fan Prove Friendship With...   \n",
       "\n",
       "                                              topics   data_channel  see_also  \\\n",
       "0  ['Asteroid', 'Asteroids', 'challenge', 'Earth'...          world         1   \n",
       "1  ['Apps and Software', 'Google', 'open source',...           tech         1   \n",
       "2  ['Entertainment', 'NFL', 'NFL Draft', 'Sports'...  entertainment         1   \n",
       "3       ['Sports', 'Video', 'Videos', 'Watercooler']    watercooler         1   \n",
       "4  ['Entertainment', 'instagram', 'instagram vide...  entertainment         1   \n",
       "\n",
       "   p_year  p_month  ...  title_pos_tag_6  title_pos_tag_7  title_pos_tag_8  \\\n",
       "0  2013.0      6.0  ...               NN               IN              VBG   \n",
       "1  2013.0      3.0  ...               NN               NN              PRP   \n",
       "2  2014.0      5.0  ...              VBP               TO               VB   \n",
       "3  2013.0     10.0  ...             NONE             NONE             NONE   \n",
       "4  2014.0      4.0  ...               VB               NN               IN   \n",
       "\n",
       "   title_pos_tag_9  title_pos_tag_10 title_pos_tag_11  title_pos_tag_12  \\\n",
       "0               NN              NONE             NONE              NONE   \n",
       "1               MD                RB               VB                IN   \n",
       "2             PRP$                JJ               NN                NN   \n",
       "3             NONE              NONE             NONE              NONE   \n",
       "4               JJ                NN             NONE              NONE   \n",
       "\n",
       "   title_pos_tag_13  title_pos_tag_14 title_pos_tag_15  \n",
       "0              NONE              NONE             NONE  \n",
       "1               VBN                RB             NONE  \n",
       "2              NONE              NONE             NONE  \n",
       "3              NONE              NONE             NONE  \n",
       "4              NONE              NONE             NONE  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Id                   0\n",
       "Popularity           0\n",
       "Page content         0\n",
       "article              0\n",
       "title                0\n",
       "topics               0\n",
       "data_channel         0\n",
       "see_also             0\n",
       "p_year               0\n",
       "p_month              0\n",
       "p_day                0\n",
       "p_hour               0\n",
       "p_minute             0\n",
       "p_second             0\n",
       "p_weekday            0\n",
       "author               0\n",
       "author scalar        0\n",
       "img_num              0\n",
       "link_num             0\n",
       "title_pos_tag_seq    0\n",
       "title_pos_tag_arr    0\n",
       "title_len            0\n",
       "title_pos_tag_1      0\n",
       "title_pos_tag_2      0\n",
       "title_pos_tag_3      0\n",
       "title_pos_tag_4      0\n",
       "title_pos_tag_5      0\n",
       "title_pos_tag_6      0\n",
       "title_pos_tag_7      0\n",
       "title_pos_tag_8      0\n",
       "title_pos_tag_9      0\n",
       "title_pos_tag_10     0\n",
       "title_pos_tag_11     0\n",
       "title_pos_tag_12     0\n",
       "title_pos_tag_13     0\n",
       "title_pos_tag_14     0\n",
       "title_pos_tag_15     0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  0.105737\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Page content</th>\n",
       "      <th>article</th>\n",
       "      <th>title</th>\n",
       "      <th>topics</th>\n",
       "      <th>data_channel</th>\n",
       "      <th>see_also</th>\n",
       "      <th>p_year</th>\n",
       "      <th>p_month</th>\n",
       "      <th>p_day</th>\n",
       "      <th>...</th>\n",
       "      <th>title_pos_tag_6</th>\n",
       "      <th>title_pos_tag_7</th>\n",
       "      <th>title_pos_tag_8</th>\n",
       "      <th>title_pos_tag_9</th>\n",
       "      <th>title_pos_tag_10</th>\n",
       "      <th>title_pos_tag_11</th>\n",
       "      <th>title_pos_tag_12</th>\n",
       "      <th>title_pos_tag_13</th>\n",
       "      <th>title_pos_tag_14</th>\n",
       "      <th>title_pos_tag_15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27643</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Note to humanity: One Direction fandom ain't ...</td>\n",
       "      <td>Soccer Star Gets Twitter Death Threats After T...</td>\n",
       "      <td>['Entertainment', 'Music', 'One Direction', 's...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NNS</td>\n",
       "      <td>IN</td>\n",
       "      <td>VBG</td>\n",
       "      <td>CD</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27644</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Shortly after announcing a hardware upgrade fo...</td>\n",
       "      <td>Google Glass Gets an Accessory Store</td>\n",
       "      <td>['Gadgets', 'glass', 'Google', 'Google Glass',...</td>\n",
       "      <td>tech</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27645</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Well, that was quick. Just hours after going o...</td>\n",
       "      <td>OUYA Gaming Console Already Sold Out on Amazon</td>\n",
       "      <td>['amazon', 'amazon kindle', 'Business', 'Gaming']</td>\n",
       "      <td>business</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>...</td>\n",
       "      <td>RP</td>\n",
       "      <td>IN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27646</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Between Two Ferns: Oscar Buzz Edition Part 1 ...</td>\n",
       "      <td>'Between Two Ferns' Mocks Oscar Nominees</td>\n",
       "      <td>['Between Two Ferns', 'Movies', 'The Oscars', ...</td>\n",
       "      <td>film</td>\n",
       "      <td>1</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>...</td>\n",
       "      <td>VBP</td>\n",
       "      <td>NNS</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27647</td>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;div class=\"article-info\"&gt;&lt;span cl...</td>\n",
       "      <td>Ever since The Hurt Locker it seems like we h...</td>\n",
       "      <td>'American Sniper' Trailer: Looks Like Eastwood...</td>\n",
       "      <td>['American Sniper', 'Awards', 'Bradley Cooper'...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>1</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>IN</td>\n",
       "      <td>NN</td>\n",
       "      <td>MD</td>\n",
       "      <td>VB</td>\n",
       "      <td>RB</td>\n",
       "      <td>IN</td>\n",
       "      <td>DT</td>\n",
       "      <td>NN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>NONE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Id                                       Page content  \\\n",
       "0  27643  <html><head><div class=\"article-info\"><span cl...   \n",
       "1  27644  <html><head><div class=\"article-info\"><span cl...   \n",
       "2  27645  <html><head><div class=\"article-info\"><span cl...   \n",
       "3  27646  <html><head><div class=\"article-info\"><span cl...   \n",
       "4  27647  <html><head><div class=\"article-info\"><span cl...   \n",
       "\n",
       "                                             article  \\\n",
       "0   Note to humanity: One Direction fandom ain't ...   \n",
       "1  Shortly after announcing a hardware upgrade fo...   \n",
       "2  Well, that was quick. Just hours after going o...   \n",
       "3   Between Two Ferns: Oscar Buzz Edition Part 1 ...   \n",
       "4   Ever since The Hurt Locker it seems like we h...   \n",
       "\n",
       "                                               title  \\\n",
       "0  Soccer Star Gets Twitter Death Threats After T...   \n",
       "1               Google Glass Gets an Accessory Store   \n",
       "2     OUYA Gaming Console Already Sold Out on Amazon   \n",
       "3           'Between Two Ferns' Mocks Oscar Nominees   \n",
       "4  'American Sniper' Trailer: Looks Like Eastwood...   \n",
       "\n",
       "                                              topics   data_channel  see_also  \\\n",
       "0  ['Entertainment', 'Music', 'One Direction', 's...  entertainment         1   \n",
       "1  ['Gadgets', 'glass', 'Google', 'Google Glass',...           tech         1   \n",
       "2  ['amazon', 'amazon kindle', 'Business', 'Gaming']       business         1   \n",
       "3  ['Between Two Ferns', 'Movies', 'The Oscars', ...           film         1   \n",
       "4  ['American Sniper', 'Awards', 'Bradley Cooper'...  entertainment         1   \n",
       "\n",
       "   p_year  p_month  p_day  ...  title_pos_tag_6  title_pos_tag_7  \\\n",
       "0  2013.0      9.0    9.0  ...              NNS               IN   \n",
       "1  2013.0     10.0   31.0  ...               NN             NONE   \n",
       "2  2013.0      6.0   25.0  ...               RP               IN   \n",
       "3  2013.0      2.0   13.0  ...              VBP              NNS   \n",
       "4  2014.0     10.0    3.0  ...               IN               NN   \n",
       "\n",
       "   title_pos_tag_8  title_pos_tag_9 title_pos_tag_10  title_pos_tag_11  \\\n",
       "0              VBG               CD               NN                NN   \n",
       "1             NONE             NONE             NONE              NONE   \n",
       "2               NN             NONE             NONE              NONE   \n",
       "3             NONE             NONE             NONE              NONE   \n",
       "4               MD               VB               RB                IN   \n",
       "\n",
       "   title_pos_tag_12  title_pos_tag_13 title_pos_tag_14 title_pos_tag_15  \n",
       "0              NONE              NONE             NONE             NONE  \n",
       "1              NONE              NONE             NONE             NONE  \n",
       "2              NONE              NONE             NONE             NONE  \n",
       "3              NONE              NONE             NONE             NONE  \n",
       "4                DT                NN             NONE             NONE  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Id                   0\n",
       "Page content         0\n",
       "article              1\n",
       "title                0\n",
       "topics               0\n",
       "data_channel         0\n",
       "see_also             0\n",
       "p_year               1\n",
       "p_month              1\n",
       "p_day                1\n",
       "p_hour               1\n",
       "p_minute             1\n",
       "p_second             1\n",
       "p_weekday            1\n",
       "author               0\n",
       "author_scalar        0\n",
       "img_num              0\n",
       "link_num             0\n",
       "title_pos_tag_seq    0\n",
       "title_pos_tag_arr    0\n",
       "title_len            0\n",
       "title_pos_tag_1      0\n",
       "title_pos_tag_2      0\n",
       "title_pos_tag_3      0\n",
       "title_pos_tag_4      0\n",
       "title_pos_tag_5      0\n",
       "title_pos_tag_6      0\n",
       "title_pos_tag_7      0\n",
       "title_pos_tag_8      0\n",
       "title_pos_tag_9      0\n",
       "title_pos_tag_10     0\n",
       "title_pos_tag_11     0\n",
       "title_pos_tag_12     0\n",
       "title_pos_tag_13     0\n",
       "title_pos_tag_14     0\n",
       "title_pos_tag_15     0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "get_title_pos_tag_cols(df_train)\n",
    "get_title_pos_tag_cols(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['num_topics'] = df_train['topics'].apply(lambda arr: len(arr))\n",
    "df_test['num_topics'] = df_test['topics'].apply(lambda arr: len(arr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.drop(['title_pos_tag_arr'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get first paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_paragraph(content, idx):\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    all_p = soup.find_all('p')\n",
    "    p_idx = 0\n",
    "    res = \"\"\n",
    "    \n",
    "    for i in range(idx):\n",
    "        try:\n",
    "            p = all_p[p_idx].getText().lower()\n",
    "            if 'see also' in p:\n",
    "                p_idx += 1\n",
    "                p = all_p[p_idx].getText().lower()\n",
    "                res += ' ' + p\n",
    "            else:\n",
    "                res += ' ' + p\n",
    "        except:\n",
    "            res += \"\"\n",
    "            \n",
    "        p_idx += 1\n",
    "        \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['para_1_3'] = df_train['content'].progress_apply(lambda c: get_paragraph(c, 3))\n",
    "df_test['para_1_3'] = df_test['content'].progress_apply(lambda c: get_paragraph(c, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['para_1_3_len'] = df_train['para_1_3'].progress_apply(lambda s: len(preprocessor(s).split()))\n",
    "df_test['para_1_3_len'] = df_test['para_1_3'].progress_apply(lambda s: len(preprocessor(s).split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████| 27643/27643 [00:01<00:00, 24968.33it/s]\n",
      " 39%|████████████████████████████▋                                             | 4601/11847 [00:00<00:00, 23640.60it/s]"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'int' object has no attribute 'split'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-29-bef77f96a78d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mdf_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article_len'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogress_apply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdf_test\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article_len'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_test\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogress_apply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tqdm\\_tqdm.py\u001b[0m in \u001b[0;36minner\u001b[1;34m(df, func, *args, **kwargs)\u001b[0m\n\u001b[0;32m    677\u001b[0m                 \u001b[1;31m# Apply the provided function (in **kwargs)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    678\u001b[0m                 \u001b[1;31m# on the df using our wrapper (which provides bar updating)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 679\u001b[1;33m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdf_function\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwrapper\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    680\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    681\u001b[0m                 \u001b[1;31m# Close bar and return pandas calculation result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[0;32m   3589\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3590\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3591\u001b[1;33m                 \u001b[0mmapped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3592\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3593\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tqdm\\_tqdm.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    673\u001b[0m                     \u001b[1;31m# take a fast or slow code path; so stop when t.total==t.n\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    674\u001b[0m                     \u001b[0mt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtotal\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtotal\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 675\u001b[1;33m                     \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    676\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    677\u001b[0m                 \u001b[1;31m# Apply the provided function (in **kwargs)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-29-bef77f96a78d>\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(s)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mdf_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article_len'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogress_apply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdf_test\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article_len'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_test\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'article'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogress_apply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'int' object has no attribute 'split'"
     ]
    }
   ],
   "source": [
    "df_train['article_len'] = df_train['article'].progress_apply(lambda s: s.split())\n",
    "df_test['article_len'] = df_test['article'].progress_apply(lambda s: s.split())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tokenize words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_topics = CountVectorizer(\n",
    "    ngram_range=(1, 1),\n",
    "    preprocessor=preprocessor,\n",
    "#     max_features=100\n",
    ")\n",
    "\n",
    "doc_bag = count_topics.fit_transform(df_train['para_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25618"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(count_topics.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_topics = TfidfVectorizer(\n",
    "    ngram_range=(1, 2),\n",
    "    preprocessor=preprocessor,\n",
    "    max_features=100\n",
    ")\n",
    "\n",
    "doc_tfidf = tfidf_topics.fit_transform(df_train['para_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['accord',\n",
       " 'also',\n",
       " 'announc',\n",
       " 'app',\n",
       " 'appl',\n",
       " 'around',\n",
       " 'back',\n",
       " 'big',\n",
       " 'busi',\n",
       " 'call',\n",
       " 'citi',\n",
       " 'come',\n",
       " 'compani',\n",
       " 'could',\n",
       " 'day',\n",
       " 'devic',\n",
       " 'digit',\n",
       " 'even',\n",
       " 'event',\n",
       " 'facebook',\n",
       " 'featur',\n",
       " 'final',\n",
       " 'find',\n",
       " 'first',\n",
       " 'friday',\n",
       " 'game',\n",
       " 'get',\n",
       " 'go',\n",
       " 'good',\n",
       " 'googl',\n",
       " 'help',\n",
       " 'internet',\n",
       " 'iphon',\n",
       " 'job',\n",
       " 'know',\n",
       " 'last',\n",
       " 'launch',\n",
       " 'like',\n",
       " 'live',\n",
       " 'look',\n",
       " 'made',\n",
       " 'make',\n",
       " 'mani',\n",
       " 'market',\n",
       " 'mashabl',\n",
       " 'may',\n",
       " 'media',\n",
       " 'might',\n",
       " 'million',\n",
       " 'mobil',\n",
       " 'monday',\n",
       " 'month',\n",
       " 'much',\n",
       " 'network',\n",
       " 'new',\n",
       " 'new york',\n",
       " 'news',\n",
       " 'next',\n",
       " 'night',\n",
       " 'one',\n",
       " 'onlin',\n",
       " 'peopl',\n",
       " 'phone',\n",
       " 'photo',\n",
       " 'releas',\n",
       " 'report',\n",
       " 'said',\n",
       " 'say',\n",
       " 'see',\n",
       " 'seri',\n",
       " 'servic',\n",
       " 'show',\n",
       " 'sinc',\n",
       " 'smartphon',\n",
       " 'social',\n",
       " 'space',\n",
       " 'start',\n",
       " 'state',\n",
       " 'take',\n",
       " 'technolog',\n",
       " 'thing',\n",
       " 'thursday',\n",
       " 'time',\n",
       " 'tuesday',\n",
       " 'twitter',\n",
       " 'two',\n",
       " 'updat',\n",
       " 'us',\n",
       " 'use',\n",
       " 'user',\n",
       " 'video',\n",
       " 'want',\n",
       " 'way',\n",
       " 'wednesday',\n",
       " 'week',\n",
       " 'work',\n",
       " 'world',\n",
       " 'would',\n",
       " 'year',\n",
       " 'york']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_topics.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.concat([df_train, df_train_oh_dc], axis=1)\n",
    "df_sample = df_train.sample(n=2000, random_state=4)\n",
    "df_sample.reset_index(inplace=True)\n",
    "X = df_sample.drop(['pop'], axis=1)\n",
    "y = df_sample['pop']\n",
    "\n",
    "# X = df_train.drop(['pop'], axis=1)\n",
    "# y = df_train['pop']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "TITLE_MAX_LEN = 15\n",
    "title_pos_tag_col = ['title_pos_tag_{}'.format(i+1) for i in range(TITLE_MAX_LEN)]\n",
    "\n",
    "all_pos_tag = set()\n",
    "\n",
    "for col in df_train[title_pos_tag_col]:\n",
    "    all_pos_tag = all_pos_tag.union(df_train[col].unique())     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TitlePosTagEncodeTransformer():\n",
    "    def __init__(self):\n",
    "        self.le = LabelEncoder()\n",
    "        self.le.fit(list(all_pos_tag))\n",
    "\n",
    "    def transform(self, df, **transform_params):\n",
    "        for col in df[title_pos_tag_col]:\n",
    "            df[col] = self.le.transform(df[col])\n",
    "        \n",
    "        return df\n",
    "\n",
    "    def fit(self, X, y=None, **fit_params):\n",
    "        return self\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        return {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = []\n",
    "pipes = []\n",
    "TITLE_MAX_LEN = 5\n",
    "\n",
    "date_col = ['p_year', 'p_month', 'p_day', 'p_hour', 'p_weekday']\n",
    "used_col = ['author_scalar', 'num_topics', 'title_len', 'img_num', 'see_also']\n",
    "title_pos_tag_col = ['title_pos_tag_{}'.format(i+1) for i in range(TITLE_MAX_LEN)]\n",
    "\n",
    "topic_bow_vec = CountVectorizer(\n",
    "    ngram_range=(1, 2), \n",
    "    preprocessor=preprocessor, \n",
    "    max_features=6000\n",
    ")\n",
    "\n",
    "title_bow_vec = CountVectorizer(\n",
    "    ngram_range=(1, 2), \n",
    "    preprocessor=preprocessor, \n",
    "    max_features=4000\n",
    ")\n",
    "\n",
    "article_tfidf_vec = TfidfVectorizer(\n",
    "    ngram_range=(1, 2), \n",
    "    preprocessor=preprocessor, \n",
    "    max_features=3000\n",
    ")\n",
    "\n",
    "pca_50 = TruncatedSVD(\n",
    "    n_components=30,\n",
    "    random_state=0\n",
    ")\n",
    "\n",
    "pca_300 = TruncatedSVD(\n",
    "    n_components=300,\n",
    "    random_state=0\n",
    ")\n",
    "\n",
    "data_channel_one_hot = OneHotEncoder(handle_unknown='ignore')\n",
    "std_scaler = StandardScaler()\n",
    "poly_scaler = PolynomialFeatures(2, include_bias=False)\n",
    "title_pos_tag_trans = TitlePosTagEncodeTransformer()\n",
    "\n",
    "p1 = Pipeline([\n",
    "    ('colt', make_column_transformer(\n",
    "            (data_channel_one_hot, ['data_channel']),\n",
    "            (std_scaler, date_col+used_col),\n",
    "            (poly_scaler, date_col+used_col),\n",
    "            (title_pos_tag_trans, title_pos_tag_col)\n",
    "    )),\n",
    "#     ('pca', pca_50),\n",
    "    ('clf', RidgeClassifier(alpha=10, normalize=True))\n",
    "])\n",
    "names.append('Ridge')\n",
    "pipes.append(p1)\n",
    "\n",
    "# p2 = Pipeline([\n",
    "#     ('colt', make_column_transformer(\n",
    "#             (data_channel_one_hot, ['data_channel']),\n",
    "#             (std_scaler, date_col+used_col),\n",
    "#             (poly_scaler, date_col+used_col),\n",
    "#             (title_pos_tag_trans, title_pos_tag_col)\n",
    "#     )),\n",
    "#     (\"clf\", KNeighborsClassifier(n_neighbors = 100, p = 2))\n",
    "# ]\n",
    "# # , verbose=True\n",
    "# )\n",
    "# names.append('KNN')\n",
    "# pipes.append(p2)\n",
    "\n",
    "# p3 = Pipeline([\n",
    "#     ('colt', make_column_transformer(\n",
    "#             (topic_bow_vec, 'topics'),\n",
    "#             (title_bow_vec, 'title'),\n",
    "#             (para_tfidf_vec, 'para_1_3'),\n",
    "#             (data_channel_one_hot, ['data_channel']),\n",
    "#             (std_scaler, date_col+used_col),\n",
    "#             (poly_scaler, date_col+used_col),\n",
    "#             (title_pos_tag_trans, title_pos_tag_col)\n",
    "#     )),\n",
    "#     ('clf', SVC(kernel = \"linear\", random_state = 0, gamma = 0.001, C = 100.0))\n",
    "# ]\n",
    "# , verbose=True\n",
    "# )\n",
    "# names.append('SVC')\n",
    "# pipes.append(p3)\n",
    "\n",
    "# ada_clf = DecisionTreeClassifier(max_depth=1)\n",
    "# p4 = Pipeline([\n",
    "#     ('colt', make_column_transformer(\n",
    "#             (topic_bow_vec, 'topics'),\n",
    "#             (title_bow_vec, 'title'),\n",
    "#             (para_tfidf_vec, 'para_1_3'),\n",
    "#             (data_channel_one_hot, ['data_channel']),\n",
    "#             (std_scaler, date_col+used_col),\n",
    "#             (poly_scaler, date_col+used_col),\n",
    "#             (title_pos_tag_trans, title_pos_tag_col)\n",
    "#     )),\n",
    "# #     ('pca', pca_300),\n",
    "#     ('clf', AdaBoostClassifier(ada_clf, n_estimators=500, random_state=0, learning_rate=0.1))\n",
    "# ]\n",
    "# , verbose=True\n",
    "# )\n",
    "# names.append('Ada')\n",
    "# pipes.append(p4)\n",
    "\n",
    "bag_clf = DecisionTreeClassifier(max_depth=1)\n",
    "p5 = Pipeline([\n",
    "    ('colt', make_column_transformer(\n",
    "            (topic_bow_vec, 'topics'),\n",
    "            (title_bow_vec, 'title'),\n",
    "#             (para_tfidf_vec, 'para_1_3'),\n",
    "            (article_tfidf_vec, 'article'),\n",
    "            (data_channel_one_hot, ['data_channel']),\n",
    "            (std_scaler, date_col+used_col),\n",
    "            (poly_scaler, date_col+used_col),\n",
    "            (title_pos_tag_trans, title_pos_tag_col)\n",
    "    )),\n",
    "    ('clf', BaggingClassifier(\n",
    "            base_estimator=bag_clf, \n",
    "            n_estimators=500, \n",
    "            max_samples=0.7,\n",
    "            n_jobs=8, \n",
    "            random_state=0\n",
    "    ))\n",
    "]\n",
    ", verbose=True\n",
    ")\n",
    "names.append('Bag')\n",
    "pipes.append(p5)\n",
    "\n",
    "#     (\"clf\", KNeighborsClassifier(n_neighbors = 30, p = 2))\n",
    "#     ('dt', DecisionTreeClassifier(random_state=0))\n",
    "#     ('gnb', GaussianNB())\n",
    "#     ('mnb', MultinomialNB())\n",
    "#     ('clf', RidgeClassifier(alpha=10, normalize=True))\n",
    "#     ('clf', LogisticRegression(solver = \"liblinear\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# p3.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB, ComplementNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[auc (5-fold cv)]\n",
      "Ridge: 0.584 (+/-0.025)\n",
      "Bag: 0.561 (+/-0.038)\n",
      "Time:  152.7318801\n"
     ]
    }
   ],
   "source": [
    "# CV\n",
    "K = 10\n",
    "start = timeit.default_timer()\n",
    "\n",
    "print('[auc (5-fold cv)]')\n",
    "for name, clf in zip(names, pipes):\n",
    "    scores = cross_val_score(\n",
    "        estimator=clf, \n",
    "        X=X_train, \n",
    "        y=y_train,\n",
    "        cv=K, \n",
    "        scoring='roc_auc'\n",
    "    )\n",
    "    print('%s: %.3f (+/-%.3f)' % (name, scores.mean(), scores.std()))\n",
    "    \n",
    "stop = timeit.default_timer()\n",
    "print('Time: ', stop - start) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_multiple_estimators(classifiers, X_list, y, sample_weights = None):\n",
    "\n",
    "    # Convert the labels `y` using LabelEncoder, because the predict method is using index-based pointers\n",
    "    # which will be converted back to original data later.\n",
    "    le_ = LabelEncoder()\n",
    "    le_.fit(y)\n",
    "    transformed_y = le_.transform(y)\n",
    "\n",
    "    # Fit all estimators with their respective feature arrays\n",
    "    estimators_ = [clf.fit(X, y) if sample_weights is None \\\n",
    "           else clf.fit(X, y, sample_weights) \\\n",
    "           for clf, X in zip([clf for _, clf in classifiers], X_list)]\n",
    "\n",
    "    return estimators_, le_\n",
    "\n",
    "\n",
    "def predict_from_multiple_estimator(estimators, label_encoder, X_list, weights = None):\n",
    "\n",
    "    # Predict 'soft' voting with probabilities\n",
    "\n",
    "    pred1 = np.asarray([clf.predict_proba(X) for clf, X in zip(estimators, X_list)])\n",
    "    pred2 = np.average(pred1, axis=0, weights=weights)\n",
    "    pred = np.argmax(pred2, axis=1)\n",
    "\n",
    "    # Convert integer predictions to original labels:\n",
    "    return label_encoder.inverse_transform(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Voting]\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   1.9s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "(0, 1, 2): 0.528 (+/- 0.075)\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "(0, 2, 1): 0.527 (+/- 0.076)\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   1.9s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "(1, 0, 2): 0.553 (+/- 0.040)\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   1.9s\n",
      "(1, 2, 0): 0.530 (+/- 0.076)\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   1.9s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   1.9s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "(2, 0, 1): 0.556 (+/- 0.047)\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   1.9s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   0.0s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=   2.0s\n",
      "(2, 1, 0): 0.541 (+/- 0.071)\n",
      "\n",
      "Best (2, 0, 1): 0.556\n"
     ]
    }
   ],
   "source": [
    "# Voting\n",
    "print('[Voting]')\n",
    "K = 10\n",
    "\n",
    "weights = list(itertools.permutations(range(0,3)))\n",
    "# weights = [\n",
    "#     (1, 0, 0),\n",
    "#     (4, 0, 1),\n",
    "#     (4, 1, 0),\n",
    "#     (9, 0, 1),\n",
    "#     (9, 1, 0)\n",
    "# ]\n",
    "# weights = [\n",
    "#     (1, 0),\n",
    "#     (0, 1),\n",
    "#     (1, 1),\n",
    "#     (2, 1),\n",
    "#     (1, 2),\n",
    "#     (3, 1),\n",
    "#     (1, 3),\n",
    "#     (3, 2),\n",
    "#     (2, 3),\n",
    "# ]\n",
    "\n",
    "best_vt, best_w, best_score = None, (), -1\n",
    "for weight in weights:\n",
    "    l_weight = list(weight)\n",
    "    kf = KFold(n_splits=K, random_state=0)\n",
    "    \n",
    "    scores = []\n",
    "    for train_index, test_index in kf.split(df_sample):\n",
    "\n",
    "        X_train, X_test = X.loc[train_index], X.loc[test_index]\n",
    "        y_train, y_test = y.loc[train_index], y.loc[test_index]\n",
    "    \n",
    "        y_preds = []\n",
    "        for clf in pipes:\n",
    "            clf.fit(X_train, y_train)\n",
    "            \n",
    "            try:\n",
    "                y_pred = clf.predict_proba(X_test).T[1]\n",
    "            except:\n",
    "                d = clf.decision_function(X_test)\n",
    "                y_pred = np.exp(d) / (1 + np.exp(d))\n",
    "            \n",
    "            y_preds.append(y_pred)\n",
    "        \n",
    "        y_pred = np.dot(np.transpose(y_preds), l_weight)\n",
    "            \n",
    "        score = roc_auc_score(y_test, y_pred)\n",
    "        scores.append(score)\n",
    "        \n",
    "    mean = np.mean(scores)\n",
    "    std = np.std(scores)\n",
    "    print('%s: %.3f (+/- %.3f)' % (weight, mean, std))\n",
    "    if best_score < mean:\n",
    "        best_vt, best_w, best_score = clf, weight, mean\n",
    "\n",
    "print('\\nBest %s: %.3f' % (best_w, best_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=   5.5s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total=  13.9s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5846540632777883"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p5.fit(X_train, y_train)\n",
    "\n",
    "try:\n",
    "    d = p5.decision_function(X_test)\n",
    "    y_pred = np.exp(d) / (1 + np.exp(d))\n",
    "except:\n",
    "    y_pred = p5.predict_proba(X_test).T[1]\n",
    "\n",
    "roc_auc_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train.drop(['pop'], axis=1)\n",
    "y = df_train['pop']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_f_test = pd.DataFrame(df_test, columns=X.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27643, 39)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11847, 39)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_f_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Pipeline] .............. (step 1 of 2) Processing colt, total=  44.7s\n",
      "[Pipeline] ............... (step 2 of 2) Processing clf, total= 2.3min\n"
     ]
    }
   ],
   "source": [
    "p1.fit(X, y)\n",
    "p4.fit(X, y)\n",
    "\n",
    "d = p1.decision_function(df_f_test)\n",
    "y_pred_1 = np.exp(d) / (1 + np.exp(d))\n",
    "\n",
    "y_pred_4 = p4.predict_proba(df_f_test).T[1]\n",
    "\n",
    "y_pred = y_pred_1 * 0.5 + y_pred_4 * 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Popularity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27643</td>\n",
       "      <td>0.464656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27644</td>\n",
       "      <td>0.479776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27645</td>\n",
       "      <td>0.475722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27646</td>\n",
       "      <td>0.485299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27647</td>\n",
       "      <td>0.512926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Id  Popularity\n",
       "0  27643    0.464656\n",
       "1  27644    0.479776\n",
       "2  27645    0.475722\n",
       "3  27646    0.485299\n",
       "4  27647    0.512926"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sub = pd.read_csv(p_sub)\n",
    "df_sub['Popularity'] = y_pred\n",
    "df_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Popularity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27643</td>\n",
       "      <td>0.481340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27644</td>\n",
       "      <td>0.497880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27645</td>\n",
       "      <td>0.493316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27646</td>\n",
       "      <td>0.503848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27647</td>\n",
       "      <td>0.497961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Id  Popularity\n",
       "0  27643    0.481340\n",
       "1  27644    0.497880\n",
       "2  27645    0.493316\n",
       "3  27646    0.503848\n",
       "4  27647    0.497961"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sub = pd.read_csv(p_sub)\n",
    "df_sub['Popularity'] = y_pred\n",
    "df_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sub.to_csv('jason_sub_4.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv('./data/train_p.csv', index=False)\n",
    "df_test.to_csv('./data/test_p.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}