notebooks/1.0_real_tweets_collect_cch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UNCOVERING STATE-BACKED TROLLS ON TWITTER"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NOTEOOK 1.0: USING TWEEPY TO SCRAP FOR REAL TWEETS\n",
    "This is the first of several notebooks on my capstone project for DSI 7, my 12-week Data Science Immersive bootcamp at General Assembly (Singapore). A detailed outline with index, background information, and summary results are in the Readme file.\n",
    "\n",
    "I've split key sections of this Project into different notebooks to make it easier to review key sections and make changes where needed. In 1.0, I focus on scrapping real tweets to complement the datasets of state-backed tweets released by Twitter.\n",
    "\n",
    "I'm using the Tweepy API. To run this notebook, you will need your own tokens, which are not included here for security reasons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "import tweepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv(find_dotenv(), override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "CONSUMER_KEY = os.getenv('CONSUMER_KEY')\n",
    "CONSUMER_SECRET = os.getenv('CONSUMER_SECRET')\n",
    "ACCESS_KEY = os.getenv('ACCESS_KEY')\n",
    "ACCESS_SECRET = os.getenv('ACCESS_SECRET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create function to scrape tweets via Tweepy and write them to a CSV file\n",
    "\n",
    "def get_tweets(username):\n",
    "    csv_file = open(\"../data/real_tweets.csv\", \"a\")\n",
    "    csv_writer = csv.writer(csv_file)\n",
    "\n",
    "    # Authorization to consumer key and consumer secret\n",
    "    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)\n",
    "\n",
    "    # Access to user's access key and access secret\n",
    "    auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)\n",
    "\n",
    "    # Calling api\n",
    "    api = tweepy.API(auth)\n",
    "\n",
    "    # Get tweets\n",
    "    for tweet in tweepy.Cursor(api.user_timeline, screen_name=username).items():\n",
    "        csv_writer.writerow(\n",
    "            [\n",
    "                tweet.id,\n",
    "                tweet.author.screen_name,\n",
    "                tweet.created_at,\n",
    "                tweet.lang,\n",
    "                tweet.source,\n",
    "                tweet.retweet_count,\n",
    "                tweet.favorited,\n",
    "                tweet.retweeted,\n",
    "                tweet.text\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    csv_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you are planning to re-run this, it is best to split up the following chunks when scrapping\n",
    "# This is due to the rate limits imposed by Twitter\n",
    "\n",
    "# The following accounts are verified real users/publishers, and are particularly active in the US\n",
    "\n",
    "get_tweets('nytimes')\n",
    "get_tweets('washingtonpost')\n",
    "get_tweets('Reuters')\n",
    "get_tweets('ChannelNewsAsia')\n",
    "get_tweets('STcom')\n",
    "\n",
    "get_tweets('FoxFriendsFirst')\n",
    "get_tweets('TheEconomist')\n",
    "get_tweets('politico')\n",
    "get_tweets('CNN')\n",
    "get_tweets('WSJ')\n",
    "\n",
    "get_tweets('realDonaldTrump')\n",
    "get_tweets('AOC')\n",
    "get_tweets('MichelleObama')\n",
    "get_tweets('BarackObama')\n",
    "get_tweets('HillaryClinton')\n",
    "\n",
    "get_tweets('SpeakerPelosi')\n",
    "get_tweets('RudyGiuliani')\n",
    "get_tweets('SarahPalinUSA')\n",
    "get_tweets('marcorubio')\n",
    "get_tweets('tedcruz')\n",
    "\n",
    "get_tweets('maggieNYT')\n",
    "get_tweets('ariannahuff')\n",
    "get_tweets('markknoller')\n",
    "get_tweets('jaketapper')\n",
    "get_tweets('ezraklein')\n",
    "\n",
    "get_tweets('BillKristol')\n",
    "get_tweets('glennbeck')\n",
    "get_tweets('KarlRove')\n",
    "get_tweets('BillOReilly')\n",
    "get_tweets('LouDobbs')\n",
    "\n",
    "get_tweets('BonnieGlaser')\n",
    "get_tweets('dandrezner')\n",
    "get_tweets('prchovanec')\n",
    "get_tweets('karaswisher')\n",
    "get_tweets('gtconway3d')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I'll format the scraped tweets in notebook 1.1, so that I won't have to re-run Tweepy unnecessarily. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}