Module-6/week-2/day-2-nlp-2/nlp-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "from progressbar import progressbar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install progressbar2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scraping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scraping Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to scrape reddit page (takes a reddit .json url)\n",
    "# returns posts \n",
    "\n",
    "headers = {'User-Agent' : 'override this bad boy!'}\n",
    "\n",
    "def scraper_bike(url):\n",
    "    posts = []\n",
    "    after = {}\n",
    "\n",
    "    for page in progressbar(range(40)):\n",
    "        params = {'after' : after}\n",
    "        url = url\n",
    "        pagepull = requests.get(url = url, params = params, headers = headers)\n",
    "        page_dict = pagepull.json()\n",
    "        posts.extend(page_dict['data']['children'])\n",
    "        after = page_dict['data']['after']\n",
    "        time.sleep(.2)\n",
    "        \n",
    "    return posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to convert posts to DataFrame - won't allow duplicate posts since unique id 'name' is set as index\n",
    "# Extract: name (as index) and subreddit, selftext, title (as columns)\n",
    "\n",
    "def posts_to_df(post_list):\n",
    "    i = 0\n",
    "    post_dict = {}\n",
    "    \n",
    "    for post in post_list:\n",
    "        ind = post_list[i]['data']\n",
    "        post_dict[ind['name']] = [ind['subreddit'], ind['title'], ind['selftext']]\n",
    "        i += 1\n",
    "\n",
    "    df_name = pd.DataFrame(post_dict)\n",
    "    df_name = df_name.T\n",
    "    df_name.columns = ['subreddit', 'title', 'selftext'] #'selftext'\n",
    "    \n",
    "    return df_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# takes scraper function and url - outputs dataframe\n",
    "\n",
    "def scrape_to_df(scrape_func, url):\n",
    "    \n",
    "    return posts_to_df(scrape_func(url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### If you want to scrape repeatedly over time and add to a csv\n",
    "# scrape, import csv, concat, drop duplicate, and output to csv\n",
    "# takes in scraper function, url, csv filename to import, csv filename to output\n",
    "# Outputs - Concatenated DataFrame as csv\n",
    "\n",
    "def scrape_add(scrape_func, url, import_file, export_file):\n",
    "    scrape_df = posts_to_df(scrape_func(url))\n",
    "    imported_df = pd.read_csv(import_file, index_col = 'Unnamed: 0')\n",
    "    concat_df = pd.concat([imported_df, scrape_df])\n",
    "    concat_df = concat_df[~concat_df.index.duplicated(keep='first')]\n",
    "    concat_df.to_csv(export_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Scrape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this and comment out pd.read_csv lines in data cleaning / preprocessing to use freshly scraped data\n",
    "# You can also put in any 2 subreddits in as the URL and get results for those\n",
    "\n",
    "nfltest = scrape_to_df(scraper_bike, 'https://www.reddit.com/r/nfl.json')\n",
    "nbatest = scrape_to_df(scraper_bike, 'https://www.reddit.com/r/nba.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "politics_test = scrape_to_df(scraper_bike, 'https://www.reddit.com/r/politics.json')\n",
    "conservative_test = scrape_to_df(scraper_bike, 'https://www.reddit.com/r/conservative.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nbatest.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nfltest.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### These scrape_add functions add to already built csvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scrape_add(scraper_bike, 'https://www.reddit.com/r/CollegeBasketball/new.json', 'NCAA_Posts_Update2.csv', 'NCAA_Posts_Update3.csv')\n",
    "# scrape_add(scraper_bike, 'https://www.reddit.com/r/AskScience/new.json', 'AskSci_Posts_Update2.csv', 'AskSci_Posts_Update3.csv')\n",
    "# scrape_add(scraper_bike, 'https://www.reddit.com/r/nba/new.json', 'NBA_Posts_Update2.csv', 'NBA_Posts_Update3.csv')\n",
    "# scrape_add(scraper_bike, 'https://www.reddit.com/r/nfl/new.json', 'NFL_Posts_Update2.csv', 'NFL_Posts_Update3.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Cleaning / Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import RegexpTokenizer, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "pd.set_option('max_colwidth', 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop column\n",
    "\n",
    "nfltest = nfltest.drop(columns = 'selftext')\n",
    "nbatest = nbatest.drop(columns = 'selftext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge subreddit data\n",
    "\n",
    "train = pd.concat([nfltest, nbatest])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Tokenize (grab only word characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_tokenizer = RegexpTokenizer(r'\\w+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['title'] = train['title'].map(lambda x: word_tokenizer.tokenize(x.lower()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rejoin list of tokenized words into single string for each row\n",
    "\n",
    "train['title'] = train['title'].map(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['title'][0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['title'][0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train test split and converting series to list of strings then to array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train[['title']]\n",
    "y = train['subreddit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=.25,\n",
    "                                                    random_state=42,\n",
    "                                                    stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# baseline is\n",
    "\n",
    "y.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create our training data list - this is a list of strings, with each string being a post title\n",
    "\n",
    "clean_train_data = []\n",
    "\n",
    "for traindata in X_train['title']:\n",
    "    clean_train_data.append(traindata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(clean_train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create test data list\n",
    "\n",
    "clean_test_data = []\n",
    "\n",
    "for testdata in X_test['title']:\n",
    "    clean_test_data.append(testdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(clean_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate our CountVectorizer. This counts the number of appearances of all the words in our training data and\n",
    "# eliminates common english stop words. 5000 max features works well for our purposes (tested various numbers). Our\n",
    "# data is already preprocessed and tokenized manually earlier. ngram_range is 1,3, although all or nearly all our\n",
    "# features are single words\n",
    "\n",
    "vectorizer = CountVectorizer(analyzer=\"word\",\n",
    "                             tokenizer=None,\n",
    "                             preprocessor=None,\n",
    "                             stop_words='english',\n",
    "                             max_features=5000,\n",
    "                             ngram_range=(1, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit our training data and test data lists to our count_vectorizer\n",
    "\n",
    "train_data_features = vectorizer.fit_transform(clean_train_data)\n",
    "\n",
    "test_data_features = vectorizer.transform(clean_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert to array\n",
    "\n",
    "train_data_features = train_data_features.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check shapes\n",
    "\n",
    "train_data_features.shape, test_data_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# I wanted check that the features corpus was as expected - removed print statement for readability\n",
    "\n",
    "vocab = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "vocab[0:200]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MODELING"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit logistic regression model\n",
    "\n",
    "lr = LogisticRegression(penalty='l2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shape check\n",
    "\n",
    "train_data_features.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.fit(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(test_data_features, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creates a dataframe that matches features to coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coef_list = lr.coef_.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coef_list = coef_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coef_df = pd.DataFrame({'features': vectorizer.get_feature_names(),\n",
    "                        'coefs': coef_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "coef_df.sort_values(by = ['coefs'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's throw out these unfair words and rerun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = set(stopwords.words('english'))\n",
    "\n",
    "extra_stopwords = ['nba', 'basketball', 'football', 'nfl']\n",
    "\n",
    "stopwords.update(extra_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(analyzer = \"word\",\n",
    "                             tokenizer = None,\n",
    "                             preprocessor = None,\n",
    "                             stop_words = stopwords,\n",
    "                             max_features = 5000,\n",
    "                             ngram_range = (1, 3))\n",
    "\n",
    "train_data_features = vectorizer.fit_transform(clean_train_data)\n",
    "\n",
    "test_data_features = vectorizer.transform(clean_test_data)\n",
    "\n",
    "train_data_features = train_data_features.toarray()\n",
    "\n",
    "train_data_features.shape, test_data_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.fit(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(test_data_features, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "coef_list = lr.coef_.tolist()\n",
    "coef_list = coef_list[0]\n",
    "\n",
    "coef_df = pd.DataFrame({'features' : vectorizer.get_feature_names(),\n",
    "                       'coefs' : coef_list})\n",
    "\n",
    "coef_df.sort_values(by = ['coefs'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree.fit(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree.score(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree.score(test_data_features, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = RandomForestClassifier(n_estimators = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest.fit(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest.score(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest.score(test_data_features, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Matrix on Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lr.predict(test_data_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_df = pd.DataFrame(cm,\n",
    "                    columns=['predict_neg', 'predict_pos'],\n",
    "                    index = ['actual_neg', 'actual_pos'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking where our model failed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comparison_df = pd.DataFrame({'y_actual' : y_test,\n",
    "             'y_predicted' : y_pred})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mismatch_df = comparison_df[comparison_df['y_actual'] != comparison_df['y_predicted']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mismatch2_df = pd.concat([mismatch_df, X_test], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All incorrect predictions with titles\n",
    "\n",
    "mismatches = mismatch2_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mismatches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's try TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Term Frequency / Inverse Document Frequency\n",
    "\n",
    "TF(w) = (Number of times term w appears in a document) / (Total number of terms in the document)\n",
    "\n",
    "IDF(w) = log_e(Total number of documents / Number of documents with term w in it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vec = TfidfVectorizer(analyzer=\"word\",\n",
    "                            tokenizer=None,\n",
    "                            preprocessor=None,\n",
    "                            stop_words=['nba', 'nfl', 'football', 'basketball'],\n",
    "                            max_features=5000,\n",
    "                            ngram_range=(1, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_features = tfidf_vec.fit_transform(clean_train_data)\n",
    "\n",
    "test_data_features = tfidf_vec.transform(clean_test_data)\n",
    "\n",
    "train_data_features = train_data_features.toarray()\n",
    "\n",
    "train_data_features.shape, test_data_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.fit(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(test_data_features, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's try on some other subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([politics_test, conservative_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train[['title']]\n",
    "y = train['subreddit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# politics_test = scrape_to_df(scraper_bike, 'https://www.reddit.com/r/politics.json')\n",
    "# conservative_test = scrape_to_df(scraper_bike, 'https://www.reddit.com/r/conservative.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 42, stratify = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "politics_test = politics_test.drop(columns = 'selftext')\n",
    "conservative_test = conservative_test.drop(columns = 'selftext')\n",
    "\n",
    "train = pd.concat([politics_test, conservative_test])\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "\n",
    "train['title'] = train['title'].map(lambda x: tokenizer.tokenize(x.lower()))\n",
    "train['title'] = train['title'].map(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create our training data list - this is a list of strings, with each string being a post title\n",
    "\n",
    "clean_train_data = []\n",
    "\n",
    "for traindata in X_train['title']:\n",
    "    clean_train_data.append(traindata)\n",
    "    \n",
    "    \n",
    "# create test data list\n",
    "\n",
    "clean_test_data = []\n",
    "\n",
    "for testdata in X_test['title']:\n",
    "    clean_test_data.append(testdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(analyzer = \"word\",\n",
    "                             tokenizer = None,\n",
    "                             preprocessor = None,\n",
    "                             stop_words = 'english',\n",
    "                             max_features = 5000,\n",
    "                             ngram_range = (1, 3))\n",
    "\n",
    "train_data_features = vectorizer.fit_transform(clean_train_data)\n",
    "\n",
    "test_data_features = vectorizer.transform(clean_test_data)\n",
    "\n",
    "train_data_features = train_data_features.toarray()\n",
    "\n",
    "train_data_features.shape, test_data_features.shape\n",
    "\n",
    "vocab = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(penalty = 'l2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_features.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.fit(train_data_features, y_train)\n",
    "\n",
    "lr.score(train_data_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.score(test_data_features, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coef_list = lr.coef_.tolist()\n",
    "\n",
    "coef_list = coef_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "coef_df = pd.DataFrame({'features' : vectorizer.get_feature_names(),\n",
    "                       'coefs' : coef_list})\n",
    "\n",
    "coef_df.sort_values(by = ['coefs'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}