AWO-128-Wikipedia pageview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wikipedia pagerank calculating"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the main functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# extract pageviews for each title\n",
    "\n",
    "def to_key_value(line):\n",
    "    params = line.split()\n",
    "    return (params[0], int(params[1])) # (title, pageviews)\n",
    "\n",
    "# sum title pageviews\n",
    "\n",
    "def sum_pageviews(item):\n",
    "    pageviews = sum([views if views else 0 for views in item[1]])\n",
    "    return (item[0], pageviews)\n",
    "\n",
    "# reduce_by_key wikipedia dump\n",
    "\n",
    "def reduce_by_key(dump):\n",
    "    return dump.map(to_key_value).reduceByKey(lambda a, b: a + b)\n",
    "            \n",
    "# detect the pages with 0 number of views\n",
    "\n",
    "def none_to_zero(x):\n",
    "    year = x[1][0] if x[1][0] else 0\n",
    "    days90 = x[1][1] if x[1][1] else 0\n",
    "    return (x[0], days90, year)\n",
    "\n",
    "# change to csv format\n",
    "\n",
    "def to_line(data):\n",
    "    return ' '.join(unicode(d) for d in data)\n",
    "\n",
    "# calculate the rank of title\n",
    "\n",
    "def rank(x):\n",
    "    global prev\n",
    "    if not prev['views_year'] or prev['views_year'] != x[0][2]:\n",
    "        prev['views_year'] = x[0][2]\n",
    "        prev['rank'] = x[1] + 1\n",
    "    return to_line([x[0][0], x[0][1], prev['views_year'], prev['rank']])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# We have scraped the needed files from Wikipedia to 2 different folders: \n",
    "# 1. For all year (10.2014 - 09.2015)\n",
    "# 2. For last 90 days (07.2015 - 09.2015)\n",
    "\n",
    "# read all files fron folders\n",
    "wikipedia_dump_year = sc.textFile(\"/vagrant/awo128/dump_month/*\")\n",
    "wikipedia_dump_90days = sc.textFile(\"/vagrant/awo128/dump_month/20150[7-9]\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of pageviews calculating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'The_Deep_End', (2, 1)), (u'Diogo_C%C3%A3o', (4, 1)), (u'Deh_Now-e_Qalandaran', (1, None)), (u'File:Ambert.jpg', (1, 1)), (u'Randy_Adams', (2, None)), (u'The_Heaven_Sword_and_Dragon_Saber', (37, 18)), (u'Roy_Scammell', (2, 2)), (u'File:Wavelets_-_DWT.png', (1, 1)), (u'Amazon.co.UK', (1, None)), (u'Hispanic_(U.S._Census)', (14, 7)), (u\"Special:WhatLinksHere/File:Gill's_Dictionary_of_the_Chinook_Jargon_01.jpg\", (1, 1)), (u'Freeze_tag/Freeze_tag', (1, None)), (u'A_little_knowledge_is_a_dangerous_thing', (1, None)), (u'Sara_Steelman', (1, 1)), (u'File:Krispy_Kreme_glazed_donut.JPG', (1, 1)), (u'97.1_WXYT', (1, 1)), (u'Talk:Pump-action', (1, 1)), (u'ZiU-9', (1, 1)), (u'File:Personality_systematics.jpg', (1, 1)), (u'naaktje', (1, 1)), (u'Modes_of_a_linear_field', (1, None)), (u'Talk:Kiril_Vajarov', (1, None)), (u'frou-frou', (3, 1)), (u'Dharmesh_Yelande', (172, 154)), (u'xenodochial', (1, 1)), (u'Huasteco_de_oriente', (1, 1)), (u'Prince_William_of_Denmark', (1, 1)), (u'Talk:Fencing_at_the_1992_Summer_Olympics_%E2%80%93_Women%27s_team_foil', (1, None)), (u'SLB_Films', (6, 1)), (u'Meriton_Civuli', (2, None)), (u'User_talk:Robertjirwin', (1, None)), (u'Griffin_Kane', (1, 1)), (u'Khattab_ibn_Nufayl', (2, 1)), (u'John_Doyle_(soccer_player_born_1946)', (1, 1)), (u'Palisade_Head', (1, None)), (u'Steph_Roberts', (2, 2)), (u'Discount_Points', (1, None)), (u'User:Jeffreywhunter', (1, 1)), (u'Sweet-and-sour', (1, None)), (u'Mormon_pioneer', (3, 3)), (u'Guard_of_Honor', (1, 1)), (u'Category:18th-century_Swedish_poets', (1, None)), (u'Granichar%2C_Dobrich_Province', (1, 1)), (u'Follicular_phase', (30, 12)), (u'United_Nations_Security_Council_Resolution_580', (1, None)), (u'Special:History/Apr%C3%A8s_toi', (1, 1)), (u'Poster_session', (7, 1)), (u'Talk:List_of_geological_features_on_Europa', (1, None)), (u'File:Kovan_double_murder_map_view.jpg', (1, 1)), (u'Category:Cumberland_School_of_Law_alumni', (1, 1)), (u'Neste_barco_%C3%A0_vela', (1, None)), (u'Ronald_breslow', (1, 1)), (u'%EF%BF%BD%D9%82%01', (3, None)), (u'Call_realty', (1, 1)), (u'Template:Medical_specialties_boards', (4, None)), (u'Kodachi', (22, 14)), (u'Category:Art_Nouveau_designers', (1, None)), (u'Dandelion_Root', (1, None)), (u'AC_Grayling', (1, 1)), (u'Special:WhatLinksHere/Category:Grand_Crosses_of_the_Order_of_Saint_Hubert', (1, None)), (u'User_talk:14.139.59.140', (1, 1)), (u'Grumman_XTSF', (1, 1)), (u'Security_and_Intelligence_Division', (9, 4)), (u'Common_bottlenose_dolphin', (8, 5)), (u'umens', (1, None)), (u'Dodge_Diplomat', (8, 3)), (u'User_talk:Loeba/Archive1', (1, None)), (u'A_Trip_Down_Market_Street', (5, 4)), (u\"Opportunities_(Let's_Make_Lots_of_Money)\", (2, 1)), (u'File:Barry_MacKenzie.png', (1, 1)), (u'File:Cooperative-Bank-House.jpg', (1, 1)), (u'Three_kingdoms', (4, 2)), (u'Galton_Laboratory', (1, None)), (u'Template:Disorders_of_the_breast', (2, None)), (u'%E5%A4%A7%E5%AF%8C%E8%B1%AA%E6%97%B6%E6%97%B6%E5%BD%A9%E5%B9%B3%E5%8F%B0%E3%80%90%E6%80%BB%E4%BB%A3QQ35767030%E3%80%91%E9%87%8D%E5%BA%86%E5%A8%B1%E4%B9%90%E5%A4%A7%E8%B1%AA%E5%BA%AD%E6%80%8E%E4%B9%88%E6%A0%B7%E4%B8%8B%E8%BD%BD%E6%8A%80%E5%B7%A7%E9%AA%97%E5%B1%80131', (1, None)), (u'Tom_Fyfe', (1, 1)), (u'List_of_Voltron_characters', (12, 9)), (u'Ranikhet', (23, 18)), (u'Croaked:_Frog_Monster_from_Hell', (2, None)), (u'File:Nagoya_Municipal_Subway_Logo.svg', (1, 1)), (u'Category:German_rapid_transit_stubs', (1, 1)), (u'2004_Paris-Nice', (1, None)), (u'Wainwright_Tomb', (1, 1)), (u'List_of_The_Honeymooners_sketches', (5, 4)), (u'1995_in_sports', (5, 3)), (u'Category:Space_colonization', (1, None)), (u'Talk:Entourage_(TV_series)', (2, None)), (u'United_Nations_headquarters', (3, None)), (u'Longmoor,_Hampshire', (2, 1)), (u'Neem-Ka-Thana', (6, 2)), (u'List_of_companies_based_in_Idaho', (1, 1)), (u'B00OMH7W08.html', (1, 1)), (u'Talk:Nicu_Constantinescu', (1, None)), (u'Great_Theatre,_Pozna%c5%84', (1, 1)), (u'Maria_Sophie_of_Bavaria', (2, 2)), (u'Linear_Algebra/Properties_of_Determinants', (1, None)), (u'Jeff_Halper', (1, None)), (u'%E5%8F%8C%E7%BB%B4%E5%BD%92%E5%9B%A0', (1, 1)), (u'Nappy_Lamare', (2, None)), (u'Best_of_the_Best_3:_No_Turning_Back', (4, 2))]\n",
      "4275219\n",
      "4275219\n"
     ]
    }
   ],
   "source": [
    "# join 90 days titles and year titles\n",
    "\n",
    "titles_joined = reduce_by_key(wikipedia_dump_year).fullOuterJoin(reduce_by_key(wikipedia_dump_90days))\n",
    "print titles_joined.take(100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pagerank calculating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'New_York', (551, 208)), (u'David_Beckham', (715, 280)), (u'Cristiano_Ronaldo', (1614, 556))]\n"
     ]
    }
   ],
   "source": [
    "# check New York, David Beckham, CRonaldo titles\n",
    "test_titles = titles_joined.filter(lambda item: 'David_Beckham' == item[0] or 'New_York' == item[0] or 'Cristiano_Ronaldo' == item[0])\n",
    "print test_titles.take(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[((u'en', 3907536, 8036991), 0), ((u'Main_Page', 719783, 1839427), 1), ((u'Novak_Djokovic', 166, 91255), 2), ((u'Oscar_Wilde', 92, 53747), 3), ((u'Special:Search', 21828, 43911), 4), ((u'Andy_Murray', 66, 40902), 5), ((u'Special:HideBanners', 36648, 37381), 6), ((u'Angelsberg', 25832, 32311), 7), ((u'Australian_Open', 30, 23283), 8), ((u'Roger_Federer', 105, 21228), 9), ((u'Rafael_Nadal', 145, 18173), 10), ((u'List_of_Grand_Slam_men%27s_singles_champions', 20, 13998), 11), ((u'United_Passions', 13030, 13037), 12), ((u'Chris_Kyle', 3931, 12631), 13), ((u'Ruby_Rose', 11328, 11395), 14), ((u'2015_Australian_Open', 21, 11324), 15), ((u'Jurassic_World', 10845, 11165), 16), ((u'Boris_Becker', 24, 10837), 17), ((u'Langston_Hughes', 31, 8589), 18), ((u'Am%C3%A9lie_Mauresmo', 10, 8227), 19), ((u'List_of_Australian_Open_men%27s_singles_champions', 8, 7599), 20), ((u'Orange_Is_the_New_Black', 6477, 7023), 21), ((u'2010_Sharm_el-Sheikh_shark_attacks', 6817, 6820), 22), ((u'Mortal_Kombat', 6530, 6657), 23), ((u'Serena_Williams', 184, 6539), 24), ((u'UFC_183', 9, 6354), 25), ((u'ATP_Tour_records', 0, 6298), 26), ((u'Super_Bowl_XLIX', 82, 6185), 27), ((u'Dennis_Rader', 6110, 6159), 28), ((u'Epidemiology_of_HIV/AIDS', 3, 5887), 29), ((u'Roy_Emerson', 5, 5462), 30), ((u'Grand_Slam_(tennis)', 82, 5238), 31), ((u'Facebook', 1560, 5186), 32), ((u'Rod_Laver', 11, 4940), 33), ((u'2015_Copa_Am%C3%A9rica', 4711, 4811), 34), ((u'Website', 1649, 4761), 35), ((u'Deaths_in_2015', 2219, 4742), 36), ((u'Special:Watchlist', 2213, 4707), 37), ((u'Game_of_Thrones', 3089, 4505), 38), ((u'Martina_Hingis', 19, 4492), 39), ((u'Plantronics_Headset', 3677, 4457), 40), ((u'Laverne_Cox', 4218, 4355), 41), ((u'Uwe_Boll', 72, 4316), 42), ((u'Islamic_State_of_Iraq_and_the_Levant', 726, 4216), 43), ((u'LaTeX/Document_Structure', 4123, 4181), 44), ((u'Alan_Shepard', 4083, 4110), 45), ((u'Stephen_Hawking', 471, 4089), 46), ((u'Chris_Pratt', 3579, 4088), 47), ((u'List_of_Bollywood_films_of_2015', 1582, 3938), 48), ((u'Wiktionary:Main_Page', 1470, 3898), 49)]\n"
     ]
    }
   ],
   "source": [
    "# sort in decreasing order, giving the same indexes for the titles with the same number of views \n",
    "\n",
    "sorted_titles = titles_joined.map(none_to_zero).sortBy(lambda x: -x[2]).zipWithIndex()\n",
    "print sorted_titles.take(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save results and calculate rank\n",
    "\n",
    "prev = {'views_year': None, 'rank': None}   \n",
    "sorted_titles.map(rank).coalesce(1,  shuffle=True).saveAsTextFile('/vagrant/awo128/Pages_final.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}