assets/examples/example-use-twitter-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import cleaned and combined CSV data as pandas dataframe\n",
    "# Change paths and file for your situation\n",
    "import warnings # remove if you want\n",
    "warnings.filterwarnings('ignore')\n",
    "from os import listdir\n",
    "from os.path import join\n",
    "import csv\n",
    "import pandas as pd\n",
    "\n",
    "encoded_data_path = 'csv'\n",
    "\n",
    "__encoded_all_file__ = 'full_corpus_clean_urls.csv'\n",
    "\n",
    "df_all = pd.read_csv(\n",
    "    join(encoded_data_path, __encoded_all_file__),\n",
    "    delimiter=','\n",
    ")\n",
    "\n",
    "# check output, if you want\n",
    "# df_all[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cleaned_urls', 'conversation_id', 'created_at', 'date', 'hashtags',\n",
       "       'id', 'likes_count', 'link', 'location', 'mentions', 'name', 'photos',\n",
       "       'place', 'quote_url', 'replies_count', 'retweet', 'retweets_count',\n",
       "       'time', 'timezone', 'tweet', 'urls', 'user_id', 'username', 'video'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04', '2018-01-05']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example method to create date and group metadata for search queries\n",
    "import nttc\n",
    "\n",
    "# Define period ranges\n",
    "ranges = [\n",
    "    ('1', ['2018-01-01', '2018-03-30']),\n",
    "    ('2', ['2018-04-01', '2018-06-12']),\n",
    "    ('3', ['2018-06-13', '2018-07-28']),\n",
    "    ('4', ['2018-07-29', '2018-10-17']),\n",
    "    ('5', ['2018-10-18', '2018-11-24']),\n",
    "    ('6', ['2018-11-25', '2018-12-10']),\n",
    "    ('7', ['2018-12-11', '2018-12-19']),\n",
    "    ('8', ['2018-12-20', '2018-12-25']),\n",
    "    ('9', ['2018-12-26', '2019-02-13']),\n",
    "    ('10', ['2019-02-14', '2019-02-28'])\n",
    "]\n",
    "\n",
    "period_dates = nttc.period_dates_writer(ranges=ranges)\n",
    "\n",
    "# HASHTAG GROUPINGS\n",
    "btw_list = [\n",
    "    '#bordercrisis', '#bordersecurity', '#buildthewall',\n",
    "    '#caravaninvasion', '#illegals',  '#migrantcaravan',\n",
    "    '#nationalemergency', '#ronilsingh'\n",
    "]\n",
    "\n",
    "fbt_list = [\n",
    "    '#familyseparation', '#familiesbelongtogether',\n",
    "    '#felipegomez', '#keepfamiliestogether',\n",
    "    '#maquin', '#wherearethechildren',\n",
    "    'jakelin caal', 'maquín', 'maquin', 'felipe alonzo-gomez', \n",
    "    'felipe gomez'\n",
    "]\n",
    "anti_list = [\n",
    "    '#shutdownstories','#trumpshutdown','#noborderwall'\n",
    "]\n",
    "\n",
    "period_dates['1'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.\n",
      "Indices:  \n",
      "\n",
      "First 3 lines in file:  [\"# '../data/ht_groups/fbt/period_edges/edges/net/edgelists/p1_net.net ../data/ht_groups/fbt/period_infomap/ -N 10 --directed --two-level --clu --map --ftree' -> 5 nodes and 3 links partitioned in 0s from codelength 1.584962501 in one level to codelength 0.666666667 in 2 levels.\", '# path flow name node:', '1:1 0.333333 \"potus\" 4'] \n",
      "\n",
      "\n",
      "2.\n",
      "Sample hub:\n",
      " [{'module': '1', 'node': '1', 'name': 'usatoday', 'score': 0.206825}, {'module': '1', 'node': '2', 'name': 'gop', 'score': 0.0970157}, {'module': '1', 'node': '3', 'name': 'bvweir', 'score': 0.0}] \n",
      "\n",
      " [{'module': '1', 'node': '1', 'name': 'realdonaldtrump', 'score': 0.374579}, {'module': '1', 'node': '2', 'name': 'betoorourke', 'score': 0.0112137}, {'module': '1', 'node': '3', 'name': 'lindseygrahamsc', 'score': 0.00078067}] \n",
      "\n",
      " [{'module': '1', 'node': '1', 'name': 'docrocktex26', 'score': 0.0755079}, {'module': '1', 'node': '2', 'name': 'mcspocky', 'score': 0.0755079}, {'module': '1', 'node': '3', 'name': 'riotwomennn', 'score': 0.0574517}]\n"
     ]
    }
   ],
   "source": [
    "# Using nttc, import infomap commumity-detected module files in folders in batches,\n",
    "# then isolate users per period per module\n",
    "\n",
    "# 1. Retrieve directory of .ftree files and save each line of the file within a list of lists to per Period Dict\n",
    "fbt_map_path = '../data/ht_groups/fbt/period_infomap/ftree'\n",
    "btw_map_path = '../data/ht_groups/btw/period_infomap/ftree'\n",
    "anti_map_path = '../data/ht_groups/anti/period_infomap/ftree'\n",
    "\n",
    "fbt_dict_map = nttc.batch_map(regex=r\"\\d{1,2}\", path=fbt_map_path, file_type='ftree')\n",
    "btw_dict_map = nttc.batch_map(regex=r\"\\d{1,2}\", path=btw_map_path, file_type='ftree')\n",
    "anti_dict_map = nttc.batch_map(regex=r\"\\d{1,2}\", path=anti_map_path, file_type='ftree')\n",
    "\n",
    "print(\n",
    "    '1.\\nIndices: ',\n",
    "    '\\n\\nFirst 3 lines in file: ',\n",
    "    fbt_dict_map['1']['lines'][:3],\n",
    "    '\\n\\n'\n",
    ")\n",
    "\n",
    "# 2. Take full listified .ftree file and write per Period per Module hubs as a Dict\n",
    "fbt_new_dict = fbt_dict_map\n",
    "fbt_dh = nttc.infomap_hub_maker(fbt_new_dict, file_type='ftree', mod_sample_size=10, hub_sample_size=-1)\n",
    "\n",
    "btw_new_dict = btw_dict_map\n",
    "btw_dh = nttc.infomap_hub_maker(btw_new_dict, file_type='ftree', mod_sample_size=10, hub_sample_size=-1)\n",
    "\n",
    "anti_new_dict = anti_dict_map\n",
    "anti_dh = nttc.infomap_hub_maker(anti_new_dict, file_type='ftree', mod_sample_size=10, hub_sample_size=-1)\n",
    "\n",
    "print(\n",
    "    '2.\\nSample hub:\\n',\n",
    "    fbt_dh['10']['info_hub']['1'][:3],\n",
    "    '\\n\\n',\n",
    "    btw_dh['10']['info_hub']['1'][:3],\n",
    "    '\\n\\n',\n",
    "    anti_dh['10']['info_hub']['1'][:3]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dict of lists per hub with hub usernames\n",
    "# RQ: What are the top X urls shared among comms per period?\n",
    "def hub_name_lister(dh):\n",
    "    hub_lists = {}\n",
    "    for htg in dh:\n",
    "        hub_lists.update({htg[0]:{}})\n",
    "        for p in htg[1]:                \n",
    "            if 'info_hub' in htg[1][p]:\n",
    "                hub_lists[htg[0]].update({p: {}})\n",
    "                for ih in htg[1][p]['info_hub']:\n",
    "                    hub_lists[htg[0]][p].update({ih: [ ]})\n",
    "                    for h in htg[1][p]['info_hub'][ih]:\n",
    "                        hub_lists[htg[0]][p][ih].append( h['name'] )\n",
    "    return hub_lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hub_lists = hub_name_lister( [ ('btw',btw_dh), ('anti',anti_dh), ('fbt',fbt_dh) ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['senschumer',\n",
       " 'nancypelosi',\n",
       " 'hurdonthehill',\n",
       " 'centerforbiodiv',\n",
       " 'vp',\n",
       " '_political_p',\n",
       " 'ryandbeam',\n",
       " 'kimamabarbara',\n",
       " 'cantabro',\n",
       " 'emttrish1331',\n",
       " 'lifieldgg',\n",
       " 'northtexasdream',\n",
       " 'jmikeseifert',\n",
       " 'sunshinejoanie']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hub_lists['anti']['1']['1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('fbt',\n",
       " '(#familyseparation|#familiesbelongtogether|#felipegomez|#keepfamiliestogether|#maquin|#wherearethechildren|jakelin caal|maquín|maquin|felipe alonzo-gomez|felipe gomez)')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Write hashtags as regex (| | |) patterns\n",
    "import urlcounter as urlc\n",
    "htg_btw = urlc.regex_lister(btw_list,'btw')\n",
    "htg_fbt = urlc.regex_lister(fbt_list,'fbt')\n",
    "htg_anti = urlc.regex_lister(anti_list,'anti')\n",
    "\n",
    "htg_fbt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Period 1 :  ('anti', '(#shutdownstories|#trumpshutdown|#noborderwall)') : 330\n",
      "Period 1 Hub 1 Size: 14\n",
      "Period 1 Hub 1 # of Tweets: 20\n",
      "Period 1 Hub 1 URLS:\n",
      " [('https://noborderwalls.org/', 3), ('https://drive.google.com/file/d/11ZqYEEE7vbSJWmKBgKTXbITzJ5MjmOOA/view', 1), ('https://www.politico.com/story/2018/01/23/chuck-schumer-trump-wall-offer-359156', 1)]\n",
      "Period 1 Hub 1 DOMAINS:\n",
      " [('noborderwalls.org', 3), ('drive.google.com', 1), ('politico.com', 1)]\n",
      "Period 1 Hub 2 Size: 18\n",
      "Period 1 Hub 2 # of Tweets: 34\n",
      "Period 1 Hub 2 URLS:\n",
      " [('https://thinkprogress.org/congress-waive-environmental-laws-for-trumps-border-wall-2899b1200b20/', 38), ('https://www.nytimes.com/interactive/2018/01/19/us/politics/government-shutdown-employee-effects.html', 29), ('http://www.bbc.com/news/world-us-canada-43418898', 3)]\n",
      "Period 1 Hub 2 DOMAINS:\n",
      " [('thinkprogress.org', 38), ('nytimes.com', 29), ('twitter.com', 4)]\n",
      "\n",
      "Period 2 :  ('anti', '(#shutdownstories|#trumpshutdown|#noborderwall)') : 93\n",
      "Period 2 Hub 1 Size: 19\n",
      "Period 2 Hub 1 # of Tweets: 30\n",
      "Period 2 Hub 1 URLS:\n",
      " [('https://www.azcentral.com/story/opinion/op-ed/2018/05/30/park-police-mexican-border-hurts-environment/653481002/', 39), ('https://www.facebook.com/events/146158769572703/', 12), ('http://thehill.com/opinion/energy-environment/381832-border-wall-would-kill-natural-treasures-and-change-the', 9)]\n",
      "Period 2 Hub 1 DOMAINS:\n",
      " [('azcentral.com', 39), ('facebook.com', 12), ('thehill.com', 9)]\n",
      "Period 2 Hub 2 Size: 6\n",
      "Period 2 Hub 2 # of Tweets: 6\n",
      "Period 2 Hub 2 URLS:\n",
      " [('http://sc.org/2EqeTUn', 115), ('https://sc.org/2EqeTUn', 72), ('https://sc.org/2ER3OvN', 21)]\n",
      "Period 2 Hub 2 DOMAINS:\n",
      " [('sc.org', 220), ('americanrivers.org', 19), ('sierraclub.org', 2)]\n",
      "\n",
      "Period 1 :  ('btw', '(#bordercrisis|#bordersecurity|#buildthewall|#caravaninvasion|#illegals|#migrantcaravan|#nationalemergency|#ronilsingh)') : 10466\n",
      "Period 1 Hub 1 Size: 1003\n",
      "Period 1 Hub 1 # of Tweets: 4065\n",
      "Period 1 Hub 1 URLS:\n",
      " [('https://www.buzzfeed.com/adolfoflores/a-huge-caravan-of-central-americans-is-headed-for-the-us', 1074), ('https://twitter.com/RaheemKassam/status/953448571868762114', 394), ('https://twitter.com/rockprincess818/status/973583018337038337', 171)]\n",
      "Period 1 Hub 1 DOMAINS:\n",
      " [('twitter.com', 2042), ('buzzfeed.com', 1074), ('breitbart.com', 974)]\n",
      "Period 1 Hub 2 Size: 508\n",
      "Period 1 Hub 2 # of Tweets: 2149\n",
      "Period 1 Hub 2 URLS:\n",
      " [('https://www.westernjournal.com/trump-sets-date-for-border-wall-construction-to-begin/', 1299), ('http://conservativefighters.com/news/eric-bolling-wants-work-trump-asking-price-tells-everything-need-know/', 1057), ('http://totalconservative.com/democrats-dont-think-concede-anything-get-daca-amnesty/#sthash.Dg4eYdC2.uxfs', 672)]\n",
      "Period 1 Hub 2 DOMAINS:\n",
      " [('westernjournal.com', 1322), ('twitter.com', 1309), ('conservativefighters.com', 1057)]\n",
      "\n",
      "Period 2 :  ('btw', '(#bordercrisis|#bordersecurity|#buildthewall|#caravaninvasion|#illegals|#migrantcaravan|#nationalemergency|#ronilsingh)') : 8546\n",
      "Period 2 Hub 1 Size: 621\n",
      "Period 2 Hub 1 # of Tweets: 1730\n",
      "Period 2 Hub 1 URLS:\n",
      " [('https://twitter.com/MarketWatch/status/997467633669812224', 421), ('https://conservativefiringline.com/group-hosts-facebook-event-stop-caravan-illegals-moving-mexico/', 415), ('http://www.foxnews.com/politics/2018/04/03/trump-says-military-will-secure-southern-border-until-wall-can-be-built.html', 330)]\n",
      "Period 2 Hub 1 DOMAINS:\n",
      " [('twitter.com', 1674), ('breitbart.com', 1190), ('conservativefiringline.com', 415)]\n",
      "Period 2 Hub 2 Size: 291\n",
      "Period 2 Hub 2 # of Tweets: 966\n",
      "Period 2 Hub 2 URLS:\n",
      " [('https://twitter.com/inthematrixxx/status/1004692217250770950', 1017), ('https://twitter.com/GartrellLinda/status/944322367869849601', 646), ('https://www.cnbc.com/2018/04/23/ex-mexican-president-vicente-fox-border-wall-is-just-trump-stupidity.html', 578)]\n",
      "Period 2 Hub 2 DOMAINS:\n",
      " [('twitter.com', 2569), ('breitbart.com', 604), ('cnbc.com', 578)]\n",
      "\n",
      "Period 1 :  ('fbt', '(#familyseparation|#familiesbelongtogether|#felipegomez|#keepfamiliestogether|#maquin|#wherearethechildren|jakelin caal|maquín|maquin|felipe alonzo-gomez|felipe gomez)') : 10\n",
      "\n",
      "Period 2 :  ('fbt', '(#familyseparation|#familiesbelongtogether|#felipegomez|#keepfamiliestogether|#maquin|#wherearethechildren|jakelin caal|maquín|maquin|felipe alonzo-gomez|felipe gomez)') : 6349\n",
      "Period 2 Hub 1 Size: 588\n",
      "Period 2 Hub 1 # of Tweets: 2507\n",
      "Period 2 Hub 1 URLS:\n",
      " [('https://twitter.com/realDonaldTrump/status/1003969399148118016', 2614), ('https://twitter.com/NBCNews/status/988148954855608321', 2124), ('https://twitter.com/realDonaldTrump/status/1000718611688943616', 1914)]\n",
      "Period 2 Hub 1 DOMAINS:\n",
      " [('twitter.com', 8456), ('jeffmerkley.com', 1402), ('vox.com', 983)]\n",
      "Period 2 Hub 2 Size: 128\n",
      "Period 2 Hub 2 # of Tweets: 172\n",
      "Period 2 Hub 2 URLS:\n",
      " [('https://www.npr.org/2018/02/27/589079243/activists-outraged-that-u-s-border-agents-separate-immigrant-families', 500), ('https://twitter.com/search', 30), ('https://actionnetwork.org/petitions/where-are-the-children', 17)]\n",
      "Period 2 Hub 2 DOMAINS:\n",
      " [('npr.org', 500), ('twitter.com', 89), ('facebook.com', 32)]\n"
     ]
    }
   ],
   "source": [
    "# Run top_urls()\n",
    "cdf = df_all.copy()\n",
    "dict_url_counts = urlc.top_urls(\n",
    "    df=cdf,\n",
    "    periods=(1,2),\n",
    "    hubs=(1,2),\n",
    "    period_dates=period_dates, \n",
    "    list_of_regex=[htg_anti,htg_btw,htg_fbt],\n",
    "    hl=hub_lists,\n",
    "    columns=['cleaned_urls', 'retweets_count', 'hashtags', 'username', 'mentions'],\n",
    "    url_sample_size=50,\n",
    "    verbose=True # True prints out status messages, False prints nothing\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1': {'hub_domain_counts': [('noborderwalls.org', 3),\n",
       "   ('drive.google.com', 1),\n",
       "   ('politico.com', 1)],\n",
       "  'hub_sample_size': 14,\n",
       "  'hub_tweet_sample_size': 20,\n",
       "  'hub_url_counts': [('https://noborderwalls.org/', 3),\n",
       "   ('https://drive.google.com/file/d/11ZqYEEE7vbSJWmKBgKTXbITzJ5MjmOOA/view',\n",
       "    1),\n",
       "   ('https://www.politico.com/story/2018/01/23/chuck-schumer-trump-wall-offer-359156',\n",
       "    1)]},\n",
       " '2': {'hub_domain_counts': [('thinkprogress.org', 38),\n",
       "   ('nytimes.com', 29),\n",
       "   ('twitter.com', 4),\n",
       "   ('bbc.com', 3)],\n",
       "  'hub_sample_size': 18,\n",
       "  'hub_tweet_sample_size': 34,\n",
       "  'hub_url_counts': [('https://thinkprogress.org/congress-waive-environmental-laws-for-trumps-border-wall-2899b1200b20/',\n",
       "    38),\n",
       "   ('https://www.nytimes.com/interactive/2018/01/19/us/politics/government-shutdown-employee-effects.html',\n",
       "    29),\n",
       "   ('http://www.bbc.com/news/world-us-canada-43418898', 3),\n",
       "   ('https://twitter.com/washingtonpost/status/966807193713610752', 1),\n",
       "   ('https://twitter.com/Earthjustice/status/961716203751342083', 1),\n",
       "   ('https://twitter.com/roseannebennet2/status/950263818265022464', 1),\n",
       "   ('https://twitter.com/tidmarshannette/status/958819086347194368', 1)]}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict_url_counts['1']['anti']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}