DownloadTwitter.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.7.2"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "cells": [
  {
   "cell_type": "code",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "Authenticate with Twitter and get a bearer token:"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"twitter.secret.json\",encoding = 'utf-8') as f:\n",
    "    credentials = json.loads(f.read())\n",
    "\n",
    "r = requests.post('https://api.twitter.com/oauth2/token?grant_type=client_credentials', auth=(credentials['api_key'],credentials['api_secret']))\n",
    "\n",
    "access_token = r.json()[\"access_token\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "A function for downloading a person's friends (the people they follow).  The API is limited to 15 requests/15 minutes, so retry logic is included. "
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'Authorization':'Bearer {token}'.format(token=access_token)}\n",
    "\n",
    "def download_friends(screen_name,next_cursor):\n",
    "    friends = []\n",
    "    max_retry_count = 0 # create a max retry count so we don't have an infinite loop\n",
    "\n",
    "    while(next_cursor != 0 ):\n",
    "        r = requests.get('https://api.twitter.com/1.1/friends/list.json?cursor={cursor}&screen_name={screen_name}&include_user_entities=true&skip_status=true&count=200'.format(cursor=next_cursor, screen_name=screen_name), headers=headers)\n",
    "    \n",
    "        if (r.status_code == 200):\n",
    "            data = r.json()\n",
    "            data_friends = [{ \"id_str\": user[\"id_str\"], \"screen_name\":user[\"screen_name\"], \"friends_count\":user[\"friends_count\"]} for user in data[\"users\"]]\n",
    "            friends.extend(data_friends)\n",
    "\n",
    "            next_cursor = data[\"next_cursor\"]\n",
    "            max_retry_count = 0\n",
    "        elif (r.status_code == 401):\n",
    "            next_cursor = 0\n",
    "        else:\n",
    "            max_retry_count = max_retry_count + 1\n",
    "            print(r.status_code)\n",
    "            print(max_retry_count)\n",
    "            if (max_retry_count >= 20):\n",
    "                raise Exception('max_retry_count limit reached')\n",
    "            time.sleep(60)\n",
    "    \n",
    "    return friends"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "Download the friends for our initial user:"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "screen_name=\"bertwagner\"\n",
    "friends = download_friends(screen_name,-1)\n",
    "\n",
    "with open('./output_friends/{screen_name}.json'.format(screen_name=screen_name), 'w') as file:\n",
    "    file.write(json.dumps(friends))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "Download the friends of the initial user's friends.  We skip friends that have more than 3000 friends because those are probably not going to be accounts we are interested in for this analysis (someone who follows that many people proabbly isn't actually reading their Twitter feed).  This process is slow and took about 24 hours for the 450 people I follow (the Twitter API rate limits really slow things down):"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "outputs": [],
   "source": [
    "for friend in friends:\n",
    "    screen_name = friend[\"screen_name\"]\n",
    "\n",
    "    friends_count = friend[\"friends_count\"]\n",
    "    if (friends_count <= 3000):\n",
    "        indirect_friends = download_friends(screen_name,-1)\n",
    "\n",
    "        with open('./output_friends/{screen_name}.json'.format(screen_name=screen_name), 'w') as file:\n",
    "            file.write(json.dumps(indirect_friends))"
   ]
  }
 ]
}