data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse data files from PushShift.io for selected subreddits using PyWren"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing the necessary packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import bz2\n",
    "import pywren"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing PyWren and configuring PyWren appropriately."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/Users/angelicabosko/anaconda3/bin/pywren\", line 8, in <module>\n",
      "    sys.exit(main())\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/pywren/scripts/pywrencli.py\", line 711, in main\n",
      "    return cli() # pylint: disable=no-value-for-parameter\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/click/core.py\", line 829, in __call__\n",
      "    return self.main(*args, **kwargs)\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/click/core.py\", line 782, in main\n",
      "    rv = self.invoke(ctx)\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/click/core.py\", line 1259, in invoke\n",
      "    return _process_result(sub_ctx.command.invoke(sub_ctx))\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/click/core.py\", line 1066, in invoke\n",
      "    return ctx.invoke(self.callback, **ctx.params)\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/click/core.py\", line 610, in invoke\n",
      "    return callback(*args, **kwargs)\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/click/decorators.py\", line 21, in new_func\n",
      "    return f(get_current_context(), *args, **kwargs)\n",
      "  File \"/Users/angelicabosko/anaconda3/lib/python3.7/site-packages/pywren/scripts/pywrencli.py\", line 139, in create_config\n",
      "    filename))\n",
      "ValueError: /Users/angelicabosko/.pywren_config already exists; not overwriting (did you need --force?)\n",
      "Using existing IAM role...\n",
      "function exists, updating\n"
     ]
    }
   ],
   "source": [
    "# Creating appropriate pywren buckets\n",
    "! pywren create_config \n",
    "! pywren create_role\n",
    "! pywren create_bucket\n",
    "! pywren deploy_lambda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Listing Subreddits to pull data from."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_subreddits = ['TheRedPill', 'Feminism', 'technews']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a function to grab all the comments for the target subreddits."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# comment parser for single file (month)\n",
    "# Code adapted from: https://github.com/AhmedSoli/Reddit-Politics/blob/master/01_Content_Analysis/PreProcessing/.ipynb_checkpoints/010_ExtractCommentsTextCorpus-checkpoint.ipynb\n",
    "from IPython.display import display, clear_output\n",
    "import json\n",
    "import os\n",
    "import multiprocessing\n",
    "import pickle\n",
    "import bz2\n",
    "\n",
    "def extract_comments(file_path):\n",
    "    '''\n",
    "    '''\n",
    "    month_comments = {}\n",
    "    for sub in target_subreddits:\n",
    "        month_comments[sub] = []\n",
    "        \n",
    "    if '.bz2' in file_path:\n",
    "        o = bz2.open\n",
    "\n",
    "    with o(file_path,'rt') as comments_info:\n",
    "            for i,comment in enumerate(comments_info):\n",
    "                # load comment into json object\n",
    "                comment = json.loads(comment)\n",
    "#               # checking if comment is located in desired subreddit \n",
    "                if comment['subreddit'] in target_subreddits:\n",
    "                    # append the body of the comment and time created\n",
    "                    month_comments[comment['subreddit']].append((comment['body'], comment['created_utc']))\n",
    "\n",
    "                    \n",
    "    # Returning the month_comments dictionary\n",
    "    return month_comments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using PyWren to parallelize comment scraping per month. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pushshift_files = ['RC_2016-01.bz2', 'RC_2016-02.bz2', 'RC_2016-03.bz2',\n",
    "                   'RC_2016-04.bz2', 'RC_2016-05.bz2', 'RC_2016-06.bz2',\n",
    "                   'RC_2016-07.bz2', 'RC_2016-08.bz2', 'RC_2016-09.bz2',\n",
    "                   'RC_2016-10.bz2', 'RC_2016-11.bz2', 'RC_2016-12.bz2']\n",
    "\n",
    "pwex = pywren.default_executor()\n",
    "futures = pwex.map(func = extract_comments, iterdata = pushshift_files)\n",
    "\n",
    "monthly_files = pywren.get_all_results(futures)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse data files from PushShift.io for selected subreddits using local computer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code is adapted from:\n",
    "\n",
    "https://github.com/AhmedSoli/Reddit-Politics/blob/master/01_Content_Analysis/PreProcessing/.ipynb_checkpoints/010_ExtractCommentsTextCorpus-checkpoint.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Subreddits we are pulling data from\n",
    "target_subreddits = ['TheRedPill', 'Feminism', 'technews']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, clear_output\n",
    "import json\n",
    "import os\n",
    "import multiprocessing\n",
    "import pickle\n",
    "import bz2\n",
    "\n",
    "def extract_comments(file_path):\n",
    "    '''\n",
    "    This function extracts all of the comments necessary from\n",
    "    each subreddit from the file path given.\n",
    "    '''\n",
    "    month_comments = {}\n",
    "    for sub in target_subreddits:\n",
    "        month_comments[sub] = []\n",
    "    if '.bz2' in file_path:\n",
    "        o = bz2.open\n",
    "\n",
    "    with o(file_path,'rt') as comments_info:\n",
    "            for i,comment in enumerate(comments_info):\n",
    "                # load comment into json object\n",
    "                comment = json.loads(comment)\n",
    "                # check if comment is contained within desired subreddit\n",
    "                if comment['subreddit'] in target_subreddits:\n",
    "                    # append the body of the comment and the time created\n",
    "                    month_comments[comment['subreddit']].append((comment['body'], comment['created_utc']))\n",
    "                \n",
    "                 # display progress\n",
    "                if i % 1000000 == 0:\n",
    "                    clear_output(wait=True)\n",
    "                    print(file_path,i)\n",
    "                    \n",
    "    # serialize the result for this month through pickling\n",
    "    pickle_out = open(\"reddit_comments/comments_corpus_\" + file_path[11:-4] + \".pickle\",\"wb\")\n",
    "    pickle.dump(month_comments, pickle_out)\n",
    "    pickle_out.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-01.bz2 61000000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1099.5419058799744"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Running Locally: January 2016\n",
    "import time\n",
    "\n",
    "start = time.time()\n",
    "extract_comments('push_shift/RC_2016-01.bz2')\n",
    "end_time = time.time() - start\n",
    "\n",
    "end_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: January 2016\n",
    "Jan_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-01.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Jan_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-02.bz2 59000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: February 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-02.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: February 2016\n",
    "Feb_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-02.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Feb_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-03.bz2 63000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: March 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-03.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: March 2016\n",
    "Mar_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-03.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Mar_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-04.bz2 64000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: April 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-04.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: April 2016\n",
    "Apr_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-04.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Apr_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-05.bz2 65000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: May 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-05.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: May 2016\n",
    "May_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-05.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#May_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-06.bz2 65000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: June 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-06.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: June 2016\n",
    "Jun_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-06.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Jun_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-07.bz2 66000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: July 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-07.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: July 2016\n",
    "Jul_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-07.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Jul_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-08.bz2 69000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: August 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-08.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: August 2016\n",
    "Aug_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-08.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Aug_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-09.bz2 67000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: September 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-09.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: September 2016\n",
    "Sep_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-09.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sep_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-10.bz2 71000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: October 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-10.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: October 2016\n",
    "Oct_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-10.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Oct_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-11.bz2 71000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: November 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-11.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: November 2016\n",
    "Nov_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-11.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Nov_2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "push_shift/RC_2016-12.bz2 72000000\n"
     ]
    }
   ],
   "source": [
    "# Running Locally: December 2016\n",
    "\n",
    "extract_comments('push_shift/RC_2016-12.bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening the pickle file: December 2016\n",
    "Dec_2016 = pickle.load(open(\"reddit_comments/comments_corpus_RC_2016-12.pickle\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dec_2016"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Storing all comments into one corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TheRedPill 48964\n",
      "Feminism 2722\n",
      "technews 260\n",
      "TheRedPill 98519\n",
      "Feminism 5528\n",
      "technews 493\n",
      "TheRedPill 145073\n",
      "Feminism 8431\n",
      "technews 757\n",
      "TheRedPill 187494\n",
      "Feminism 10931\n",
      "technews 993\n",
      "TheRedPill 232865\n",
      "Feminism 14248\n",
      "technews 1234\n",
      "TheRedPill 281897\n",
      "Feminism 16884\n",
      "technews 1489\n",
      "TheRedPill 325336\n",
      "Feminism 19570\n",
      "technews 1760\n",
      "TheRedPill 366283\n",
      "Feminism 22379\n",
      "technews 2072\n",
      "TheRedPill 411257\n",
      "Feminism 25746\n",
      "technews 2405\n",
      "TheRedPill 451480\n",
      "Feminism 28512\n",
      "technews 2653\n",
      "TheRedPill 492092\n",
      "Feminism 31604\n",
      "technews 2854\n",
      "TheRedPill 535804\n",
      "Feminism 35754\n",
      "technews 3126\n"
     ]
    }
   ],
   "source": [
    "push_shift = ['RC_2016-01.bz2', 'RC_2016-02.bz2', 'RC_2016-03.bz2',\n",
    "                   'RC_2016-04.bz2', 'RC_2016-05.bz2', 'RC_2016-06.bz2',\n",
    "                   'RC_2016-07.bz2', 'RC_2016-08.bz2', 'RC_2016-09.bz2',\n",
    "                   'RC_2016-10.bz2', 'RC_2016-11.bz2', 'RC_2016-12.bz2']\n",
    "\n",
    "\n",
    "comments_corpus = {}\n",
    "# Keep file pickleing for storage reasons. \n",
    "# Storing all monthly comments into one corpus for each individual subreddit\n",
    "for file in sorted(push_shift):\n",
    "    comments_corpus_temp = pickle.load(open(\"reddit_comments/comments_corpus_\" + file[:-4] + \".pickle\",\"rb\")) # loading in monthly files without file type\n",
    "    for subreddit in comments_corpus_temp:\n",
    "        if subreddit not in comments_corpus: # some selected subreddits might not be present in files\n",
    "            comments_corpus[subreddit] = []\n",
    "        # Add comment to entire corpus of comments for a single subreddit. This might be why we want to collect utc, incase we want to run time series on the data to map key events\n",
    "        comments_corpus[subreddit].extend(\n",
    "            comments_corpus_temp[subreddit]\n",
    "        )\n",
    "        print(subreddit,len(comments_corpus[subreddit]))\n",
    "                        \n",
    "pickle_out = open(\"reddit_comments/comments_corpus_final.pickle\",\"wb\")\n",
    "pickle.dump(comments_corpus, pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}