Chapter03-web-crawlers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/Wikipedia:Protection_policy#semi\n",
      "#mw-head\n",
      "#p-search\n",
      "/wiki/Kevin_Bacon_(disambiguation)\n",
      "/wiki/File:Kevin_Bacon_SDCC_2014.jpg\n",
      "/wiki/San_Diego_Comic-Con\n",
      "/wiki/Philadelphia\n",
      "/wiki/Pennsylvania\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "http://baconbros.com/\n",
      "#cite_note-1\n",
      "#cite_note-actor-2\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/JFK_(film)\n",
      "/wiki/A_Few_Good_Men\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Sleepers\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Fox_Broadcasting_Company\n",
      "/wiki/The_Following\n",
      "/wiki/HBO\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/Screen_Actors_Guild_Award\n",
      "/wiki/Primetime_Emmy_Award\n",
      "/wiki/The_Guardian\n",
      "/wiki/Academy_Award\n",
      "#cite_note-3\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "#cite_note-4\n",
      "/wiki/Social_networks\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/SixDegrees.org\n",
      "#cite_note-walk-5\n",
      "#Early_life_and_education\n",
      "#Acting_career\n",
      "#Early_work\n",
      "#1980s\n",
      "#1990s\n",
      "#2000s\n",
      "#2010s\n",
      "#Advertising_work\n",
      "#Personal_life\n",
      "#Six_Degrees_of_Kevin_Bacon\n",
      "#Music\n",
      "#Awards_and_nominations\n",
      "#Filmography\n",
      "#See_also\n",
      "#References\n",
      "#External_links\n",
      "/wiki/Philadelphia\n",
      "#cite_note-actor-2\n",
      "#cite_note-actor-2\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "#cite_note-bacon-6\n",
      "/wiki/Pennsylvania_Governor%27s_School_for_the_Arts\n",
      "/wiki/Bucknell_University\n",
      "#cite_note-7\n",
      "/wiki/Glory_Van_Scott\n",
      "#cite_note-walk-5\n",
      "#cite_note-bacon-6\n",
      "/wiki/Circle_in_the_Square\n",
      "/wiki/Nancy_Mills\n",
      "/wiki/Cosmopolitan_(magazine)\n",
      "#cite_note-cosmo91-8\n",
      "/wiki/Fraternities_and_sororities\n",
      "/wiki/Animal_House\n",
      "#cite_note-bacon-6\n",
      "/wiki/Search_for_Tomorrow\n",
      "/wiki/Guiding_Light\n",
      "/wiki/Friday_the_13th_(1980_film)\n",
      "#cite_note-9\n",
      "/wiki/Phoenix_Theater\n",
      "/wiki/Flux\n",
      "/wiki/Second_Stage_Theatre\n",
      "#cite_note-bio-10\n",
      "/wiki/Obie_Award\n",
      "/wiki/Forty_Deuce\n",
      "#cite_note-kevin-11\n",
      "/wiki/Slab_Boys\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Val_Kilmer\n",
      "/wiki/Barry_Levinson\n",
      "/wiki/Diner_(film)\n",
      "/wiki/Steve_Guttenberg\n",
      "/wiki/Daniel_Stern_(actor)\n",
      "/wiki/Mickey_Rourke\n",
      "/wiki/Tim_Daly\n",
      "/wiki/Ellen_Barkin\n",
      "#cite_note-12\n",
      "/wiki/Footloose_(1984_film)\n",
      "#cite_note-bio-10\n",
      "/wiki/James_Dean\n",
      "/wiki/Rebel_Without_a_Cause\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Judy_Garland\n",
      "#cite_note-time84-13\n",
      "#cite_note-bacon-6\n",
      "#cite_note-14\n",
      "#cite_note-15\n",
      "/wiki/People_(American_magazine)\n",
      "/wiki/Typecasting_(acting)\n",
      "/wiki/John_Hughes_(filmmaker)\n",
      "/wiki/She%27s_Having_a_Baby\n",
      "#cite_note-bio-10\n",
      "/wiki/The_Big_Picture_(1989_film)\n",
      "#cite_note-16\n",
      "/wiki/Tremors_(film)\n",
      "#cite_note-17\n",
      "/wiki/Joel_Schumacher\n",
      "/wiki/Flatliners\n",
      "#cite_note-bio-10\n",
      "/wiki/Elizabeth_Perkins\n",
      "/wiki/He_Said,_She_Said\n",
      "#cite_note-bio-10\n",
      "/wiki/The_New_York_Times\n",
      "#cite_note-nyt94-18\n",
      "/wiki/Oliver_Stone\n",
      "/wiki/JFK_(film)\n",
      "#cite_note-19\n",
      "/wiki/A_Few_Good_Men_(film)\n",
      "#cite_note-20\n",
      "/wiki/Michael_Greif\n",
      "#cite_note-bio-10\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/The_River_Wild\n",
      "#cite_note-bio-10\n",
      "/wiki/Meryl_Streep\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "#cite_note-bio-10\n",
      "/wiki/Blockbuster_(entertainment)\n",
      "/wiki/Apollo_13_(film)\n",
      "#cite_note-21\n",
      "/wiki/Sleepers_(film)\n",
      "#cite_note-22\n",
      "/wiki/Picture_Perfect_(1997_film)\n",
      "#cite_note-bio-10\n",
      "/wiki/Losing_Chase\n",
      "#cite_note-austin-23\n",
      "/wiki/Digging_to_China\n",
      "#cite_note-bio-10\n",
      "/wiki/Payola\n",
      "/wiki/Telling_Lies_in_America_(film)\n",
      "#cite_note-bio-10\n",
      "/wiki/Wild_Things_(film)\n",
      "/wiki/Stir_of_Echoes\n",
      "/wiki/David_Koepp\n",
      "#cite_note-24\n",
      "/wiki/File:KevinBaconTakingChanceFeb09.jpg\n",
      "/wiki/File:KevinBaconTakingChanceFeb09.jpg\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Paul_Verhoeven\n",
      "/wiki/Hollow_Man\n",
      "#cite_note-25\n",
      "/wiki/Colin_Firth\n",
      "/wiki/Rachel_Blanchard\n",
      "/wiki/M%C3%A9nage_%C3%A0_trois\n",
      "/wiki/Where_the_Truth_Lies\n",
      "#cite_note-26\n",
      "/wiki/Atom_Egoyan\n",
      "/wiki/MPAA\n",
      "/wiki/MPAA_film_rating_system\n",
      "#cite_note-27\n",
      "/wiki/Pedophile\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "#cite_note-28\n",
      "/wiki/HBO_Films\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Michael_Strobl\n",
      "/wiki/Desert_Storm\n",
      "#cite_note-29\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Matthew_Vaughn\n",
      "/wiki/X-Men:_First_Class\n",
      "#cite_note-30\n",
      "/wiki/Sebastian_Shaw_(comics)\n",
      "#cite_note-31\n",
      "/wiki/Dustin_Lance_Black\n",
      "/wiki/8_(play)\n",
      "/wiki/Perry_v._Brown\n",
      "/wiki/Proposition_8\n",
      "/wiki/Charles_J._Cooper\n",
      "#cite_note-8_the_play-32\n",
      "/wiki/Wilshire_Ebell_Theatre\n",
      "/wiki/American_Foundation_for_Equal_Rights\n",
      "#cite_note-8_play_video-33\n",
      "#cite_note-34\n",
      "/wiki/The_Following\n",
      "#cite_note-35\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "#cite_note-36\n",
      "/wiki/Huffington_Post\n",
      "/wiki/Wikipedia:Citation_needed\n",
      "/wiki/Tremors_(film)\n",
      "/wiki/Wikipedia:Citation_needed\n",
      "/wiki/Tremors_5:_Bloodline\n",
      "/wiki/EE_(telecommunications_company)\n",
      "/wiki/United_Kingdom\n",
      "#cite_note-37\n",
      "#cite_note-38\n",
      "/wiki/Egg_as_food\n",
      "#cite_note-39\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/PBS\n",
      "/wiki/Lanford_Wilson\n",
      "/wiki/Lemon_Sky\n",
      "#cite_note-cosmo91-8\n",
      "/wiki/Pyrates\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Upper_West_Side\n",
      "/wiki/Manhattan\n",
      "#cite_note-40\n",
      "/wiki/Tracy_Pollan\n",
      "#cite_note-41\n",
      "#cite_note-42\n",
      "#cite_note-43\n",
      "/wiki/The_Times\n",
      "#cite_note-44\n",
      "#cite_note-45\n",
      "/wiki/Will.i.am\n",
      "/wiki/It%27s_a_New_Day_(Will.i.am_song)\n",
      "/wiki/Barack_Obama\n",
      "/wiki/Ponzi_scheme\n",
      "/wiki/Bernard_Madoff\n",
      "#cite_note-financialpost-46\n",
      "#cite_note-47\n",
      "/wiki/Finding_Your_Roots\n",
      "/wiki/Henry_Louis_Gates\n",
      "#cite_note-48\n",
      "#cite_note-49\n",
      "#cite_note-50\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/Trivia\n",
      "/wiki/Big_screen\n",
      "/wiki/Six_degrees_of_separation\n",
      "/wiki/Internet_meme\n",
      "/wiki/SixDegrees.org\n",
      "#cite_note-51\n",
      "/wiki/Bacon_number\n",
      "/wiki/Internet_Movie_Database\n",
      "#cite_note-52\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Bacon_number\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "#cite_note-53\n",
      "/wiki/The_Bacon_Brothers\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/Music_album\n",
      "#cite_note-54\n",
      "/wiki/File:Question_book-new.svg\n",
      "/wiki/Wikipedia:Citing_sources\n",
      "/wiki/Wikipedia:Verifiability\n",
      "//en.wikipedia.org/w/index.php?title=Kevin_Bacon&action=edit\n",
      "/wiki/Help:Introduction_to_referencing_with_Wiki_Markup/1\n",
      "/wiki/Wikipedia:Verifiability#Burden_of_evidence\n",
      "/wiki/Help:Maintenance_template_removal\n",
      "/wiki/Golden_Globe_Awards\n",
      "/wiki/Golden_Globe_Award_for_Best_Supporting_Actor_%E2%80%93_Motion_Picture\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Broadcast_Film_Critics_Association_Awards\n",
      "/wiki/Broadcast_Film_Critics_Association_Award_for_Best_Actor\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Supporting_Role\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/MTV_Movie_Awards\n",
      "/wiki/MTV_Movie_Award_for_Best_Villain\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Boston_Society_of_Film_Critics_Awards\n",
      "/wiki/Boston_Society_of_Film_Critics_Award_for_Best_Cast\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Satellite_Awards\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Motion_Picture_Drama\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Beauty_Shop\n",
      "/wiki/Primetime_Emmy_Awards\n",
      "/wiki/Primetime_Emmy_Award_for_Outstanding_Lead_Actor_in_a_Miniseries_or_a_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Satellite_Awards\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Frost/Nixon_(film)\n",
      "/wiki/Golden_Globe_Awards\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/X-Men:_First_Class\n",
      "/wiki/Saturn_Awards\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/People%27s_Choice_Awards\n",
      "/wiki/People%27s_Choice_Awards\n",
      "/wiki/The_Following\n",
      "/wiki/Saturn_Awards\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/Golden_Globe_Awards\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "/wiki/I_Love_Dick_(TV_series)\n",
      "#cite_note-55\n",
      "#cite_note-56\n",
      "#cite_note-57\n",
      "#cite_note-58\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/List_of_actors_with_Hollywood_Walk_of_Fame_motion_picture_stars\n",
      "#cite_ref-1\n",
      "https://web.archive.org/web/20090113222205/http://www.newenglandancestors.org/research/services/articles_gbr78.asp\n",
      "http://www.newenglandancestors.org/research/services/articles_gbr78.asp\n",
      "#cite_ref-actor_2-0\n",
      "#cite_ref-actor_2-1\n",
      "#cite_ref-actor_2-2\n",
      "http://www.biography.com/people/kevin-bacon-9542173\n",
      "#cite_ref-3\n",
      "https://www.theguardian.com/film/filmblog/2009/feb/19/best-actors-never-nominated-for-oscars\n",
      "#cite_ref-4\n",
      "http://www.walkoffame.com/kevin-bacon\n",
      "#cite_ref-walk_5-0\n",
      "#cite_ref-walk_5-1\n",
      "https://web.archive.org/web/20141016202657/http://www.thebiographychannel.co.uk/biographies/kevin-bacon.html\n",
      "http://www.thebiographychannel.co.uk/biographies/kevin-bacon.html\n",
      "#cite_ref-bacon_6-0\n",
      "#cite_ref-bacon_6-1\n",
      "#cite_ref-bacon_6-2\n",
      "#cite_ref-bacon_6-3\n",
      "http://www.biography.com/news/kevin-bacon-biography-facts\n",
      "#cite_ref-7\n",
      "https://movies.yahoo.com/person/kevin-bacon/biography.html\n",
      "#cite_ref-cosmo91_8-0\n",
      "#cite_ref-cosmo91_8-1\n",
      "#cite_ref-9\n",
      "http://www.nydailynews.com/entertainment/happy-halloween-superstars-start-horror-flick-gallery-1.98345\n",
      "#cite_ref-bio_10-0\n",
      "#cite_ref-bio_10-1\n",
      "#cite_ref-bio_10-2\n",
      "#cite_ref-bio_10-3\n",
      "#cite_ref-bio_10-4\n",
      "#cite_ref-bio_10-5\n",
      "#cite_ref-bio_10-6\n",
      "#cite_ref-bio_10-7\n",
      "#cite_ref-bio_10-8\n",
      "#cite_ref-bio_10-9\n",
      "#cite_ref-bio_10-10\n",
      "https://www.pbs.org/wnet/finding-your-roots/profiles/kevin-bacon%C2%A0/\n",
      "#cite_ref-kevin_11-0\n",
      "http://www.tvguide.com/celebrities/kevin-bacon/bio/160550\n",
      "#cite_ref-12\n",
      "http://news.moviefone.com/2012/03/02/diner-30th-anniversary/\n",
      "#cite_ref-time84_13-0\n",
      "http://www.time.com/time/magazine/article/0,9171,950019,00.html\n",
      "#cite_ref-14\n",
      "http://www.huffingtonpost.com/2014/08/25/kevin-bacon-footloose_n_5710413.html\n",
      "#cite_ref-15\n",
      "https://web.archive.org/web/20090109152125/http://www.thebiographychannel.co.uk/biography_story/522%3A492/1/Kevin_Bacon.htm\n",
      "http://www.thebiographychannel.co.uk/biography_story/522:492/1/Kevin_Bacon.htm\n",
      "#cite_ref-16\n",
      "https://www.nytimes.com/1994/09/25/movies/a-second-wind-is-blowing-for-kevin-bacon.html\n",
      "#cite_ref-17\n",
      "https://www.nytimes.com/movie/review?res=9C0CE2DE1631F93AA25752C0A966958260\n",
      "#cite_ref-nyt94_18-0\n",
      "https://query.nytimes.com/gst/fullpage.html?res=9C07E6D91F3BF936A1575AC0A962958260&sec=&spon=&pagewanted=all\n",
      "#cite_ref-19\n",
      "http://www.jfk-online.com/jfkbacon.html\n",
      "#cite_ref-20\n",
      "http://www.tcm.com/this-month/article/143158%7C0/A-Few-Good-Men.html\n",
      "#cite_ref-21\n",
      "http://collider.com/kevin-bacon-commercials-footloose/\n",
      "#cite_ref-22\n",
      "http://www.rogerebert.com/reviews/sleepers-1996\n",
      "#cite_ref-austin_23-0\n",
      "http://www.austinchronicle.com/calendar/film/1997-02-07/283342/\n",
      "/wiki/The_Austin_Chronicle\n",
      "#cite_ref-24\n",
      "http://www.criminalelement.com/blogs/2013/09/under-the-raderhorror-movies-you-may-have-missed-stir-of-echoes\n",
      "#cite_ref-25\n",
      "http://www.rogerebert.com/reviews/hollow-man-2000\n",
      "#cite_ref-26\n",
      "http://movies.about.com/od/wherethetruthlies/a/truthkb101305.htm\n",
      "#cite_ref-27\n",
      "http://jam.canoe.ca/Movies/2005/09/14/1216527.html\n",
      "#cite_ref-28\n",
      "http://www.latimes.com/entertainment/la-et-kevin-bacon-photo6-photo.html\n",
      "#cite_ref-29\n",
      "http://www.nydailynews.com/entertainment/tv-movies/kevin-bacon-chance-body-fallen-marine-home-article-1.392226\n",
      "#cite_ref-30\n",
      "https://web.archive.org/web/20100722010545/http://heatvision.hollywoodreporter.com/2010/07/winters-bone-star-cast-as-mystique-in-xmen-first-class.html\n",
      "http://heatvision.hollywoodreporter.com/2010/07/winters-bone-star-cast-as-mystique-in-xmen-first-class.html\n",
      "#cite_ref-31\n",
      "https://web.archive.org/web/20100720060214/http://www.forcesofgeek.com/2010/07/kevin-bacon-playing-sebastian-shaw-in-x.html\n",
      "http://www.forcesofgeek.com/2010/07/kevin-bacon-playing-sebastian-shaw-in-x.html\n",
      "#cite_ref-8_the_play_32-0\n",
      "http://www.accesshollywood.com/jesse-tyler-ferguson/glee-stars-touched-by-brad-pitt-and-george-clooneys-support-of-8_article_61543\n",
      "/wiki/Access_Hollywood\n",
      "#cite_ref-8_play_video_33-0\n",
      "https://www.youtube.com/watch?v=qlUG8F9uVgM\n",
      "#cite_ref-34\n",
      "http://www.pinknews.co.uk/2012/03/01/youtube-to-broadcast-proposition-8-play-live/\n",
      "#cite_ref-35\n",
      "http://www.fox.com/the-following/\n",
      "#cite_ref-36\n",
      "https://news.yahoo.com/blogs/trending-now/kevin-bacon-gives-millennials-a-history-lesson-about-the--80s-162525915.html\n",
      "#cite_ref-37\n",
      "http://www.campaignlive.co.uk/news/1294856/\n",
      "#cite_ref-38\n",
      "http://parade.condenast.com/269380/ashleighschmitz/kevin-bacon-reprises-his-most-iconic-film-roles-in-british-commercial/\n",
      "#cite_ref-39\n",
      "http://money.cnn.com/2015/03/13/media/kevin-bacon-eggs/index.html?iid=HP_LN\n",
      "#cite_ref-40\n",
      "http://www.nydailynews.com/entertainment/tv-movies/kevin-bacon-loyalty-nyc-philly-origins-peace-bustling-city-article-1.147197\n",
      "#cite_ref-41\n",
      "http://www.people.com/people/archive/article/0,,20093025,00.html\n",
      "#cite_ref-42\n",
      "http://www.au.org/media/church-and-state/archives/2008/05/two-thumbs-up.html\n",
      "#cite_ref-43\n",
      "https://www.washingtonpost.com/wp-dyn/content/article/2008/03/25/AR2008032503852.html\n",
      "#cite_ref-44\n",
      "#cite_ref-45\n",
      "http://www.foxnews.com/story/0,2933,343589,00.html\n",
      "#cite_ref-financialpost_46-0\n",
      "https://web.archive.org/web/20140314085857/http://economiccrisis.us/2009/06/may-god-spare-mercy-victim-tells-madoff/\n",
      "http://economiccrisis.us/2009/06/may-god-spare-mercy-victim-tells-madoff/\n",
      "#cite_ref-47\n",
      "#cite_ref-48\n",
      "http://www.huffingtonpost.com/megan-smolenyak-smolenyak/6-degrees-of-separation-k_b_900707.html\n",
      "#cite_ref-49\n",
      "https://web.archive.org/web/20130405182304/http://www.drawtheline.org/watch-stuff/\n",
      "http://www.drawtheline.org/watch-stuff\n",
      "#cite_ref-50\n",
      "http://www.drawtheline.org/sign-now/\n",
      "#cite_ref-51\n",
      "http://www.sixdegrees.org/\n",
      "#cite_ref-52\n",
      "http://www.webmonkey.com/2012/09/easter-egg-google-connects-the-dots-for-bacon-number-search/\n",
      "#cite_ref-53\n",
      "http://www.telegraph.co.uk/science/science-news/4768389/And-the-winner-tonight-is.html\n",
      "#cite_ref-54\n",
      "http://baconbros.com/\n",
      "#cite_ref-55\n",
      "/wiki/Reuters\n",
      "https://www.cbsnews.com/pictures/golden-globes-2018-highlights/50/\n",
      "/wiki/CBS_News\n",
      "#cite_ref-56\n",
      "https://www.theverge.com/2018/1/7/16861812/golden-globes-2018-aziz-ansari-master-of-none-best-actor-tv\n",
      "/wiki/The_Verge\n",
      "#cite_ref-57\n",
      "https://www.hollywoodreporter.com/news/aziz-ansari-wins-best-performance-by-an-actor-a-tv-series-comedy-musical-golden-globes-2018-1072154\n",
      "/wiki/The_Hollywood_Reporter\n",
      "#cite_ref-58\n",
      "http://www.indiewire.com/2018/01/aziz-ansari-wins-golden-globe-best-actor-tv-comedy-1201914235/\n",
      "/wiki/Indie_Wire\n",
      "https://commons.wikimedia.org/wiki/Category:Kevin_Bacon\n",
      "http://www.imdb.com/name/nm0000102/\n",
      "/wiki/IMDb\n",
      "https://www.ibdb.com/Person/View/90569\n",
      "/wiki/Internet_Broadway_Database\n",
      "https://www.wikidata.org/wiki/Q3454165#P1220\n",
      "http://www.lortel.org/Archives/CreditableEntity/5597\n",
      "/wiki/Lortel_Archives\n",
      "https://www.allmovie.com/artist/p3164\n",
      "/wiki/AllMovie\n",
      "http://oracleofbacon.org\n",
      "/wiki/Template:Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Template_talk:Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "//en.wikipedia.org/w/index.php?title=Template:Critics%27_Choice_Movie_Award_for_Best_Actor&action=edit\n",
      "/wiki/Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Ian_McKellen\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jamie_Foxx\n",
      "/wiki/Philip_Seymour_Hoffman\n",
      "/wiki/Forest_Whitaker\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jeff_Bridges\n",
      "/wiki/Colin_Firth\n",
      "/wiki/George_Clooney\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Matthew_McConaughey\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Leonardo_DiCaprio\n",
      "/wiki/Casey_Affleck\n",
      "/wiki/Gary_Oldman\n",
      "/wiki/Template:GoldenGlobeBestActorTVMiniseriesFilm\n",
      "/wiki/Template_talk:GoldenGlobeBestActorTVMiniseriesFilm\n",
      "//en.wikipedia.org/w/index.php?title=Template:GoldenGlobeBestActorTVMiniseriesFilm&action=edit\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Anthony_Andrews\n",
      "/wiki/Richard_Chamberlain\n",
      "/wiki/Ted_Danson\n",
      "/wiki/Dustin_Hoffman\n",
      "/wiki/James_Woods\n",
      "/wiki/Randy_Quaid\n",
      "/wiki/Michael_Caine\n",
      "/wiki/Stacy_Keach\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Beau_Bridges\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Ra%C3%BAl_Juli%C3%A1\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Ving_Rhames\n",
      "/wiki/Stanley_Tucci\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/James_Franco\n",
      "/wiki/Albert_Finney\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jonathan_Rhys_Meyers\n",
      "/wiki/Bill_Nighy\n",
      "/wiki/Jim_Broadbent\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Billy_Bob_Thornton\n",
      "/wiki/Oscar_Isaac\n",
      "/wiki/Tom_Hiddleston\n",
      "/wiki/Ewan_McGregor\n",
      "/wiki/Template:Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Template_talk:Saturn_Award_for_Best_Actor_on_Television\n",
      "//en.wikipedia.org/w/index.php?title=Template:Saturn_Award_for_Best_Actor_on_Television&action=edit\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Kyle_Chandler\n",
      "/wiki/Steven_Weber_(actor)\n",
      "/wiki/Richard_Dean_Anderson\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Robert_Patrick\n",
      "/wiki/Ben_Browder\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Ben_Browder\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Michael_C._Hall\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Edward_James_Olmos\n",
      "/wiki/Josh_Holloway\n",
      "/wiki/Stephen_Moyer\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Mads_Mikkelsen\n",
      "/wiki/Hugh_Dancy\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Bruce_Campbell\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Template:ScreenActorsGuildAward_MaleTVMiniseriesMovie\n",
      "/wiki/Template_talk:ScreenActorsGuildAward_MaleTVMiniseriesMovie\n",
      "//en.wikipedia.org/w/index.php?title=Template:ScreenActorsGuildAward_MaleTVMiniseriesMovie&action=edit\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Ra%C3%BAl_Juli%C3%A1\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Christopher_Reeve\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/Ben_Kingsley\n",
      "/wiki/William_H._Macy\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Paul_Newman\n",
      "/wiki/Jeremy_Irons\n",
      "/wiki/Kevin_Kline\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Alexander_Skarsg%C3%A5rd\n",
      "/wiki/Template:ScreenActorsGuildAward_CastMotionPicture_1995%E2%80%932000\n",
      "/wiki/Template_talk:ScreenActorsGuildAward_CastMotionPicture_1995%E2%80%932000\n",
      "//en.wikipedia.org/w/index.php?title=Template:ScreenActorsGuildAward_CastMotionPicture_1995%E2%80%932000&action=edit\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Tom_Hanks\n",
      "/wiki/Ed_Harris\n",
      "/wiki/Bill_Paxton\n",
      "/wiki/Kathleen_Quinlan\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/The_Birdcage\n",
      "/wiki/Hank_Azaria\n",
      "/wiki/Christine_Baranski\n",
      "/wiki/Dan_Futterman\n",
      "/wiki/Gene_Hackman\n",
      "/wiki/Nathan_Lane\n",
      "/wiki/Dianne_Wiest\n",
      "/wiki/Robin_Williams\n",
      "/wiki/The_Full_Monty\n",
      "/wiki/Mark_Addy\n",
      "/wiki/Paul_Barber_(actor)\n",
      "/wiki/Robert_Carlyle\n",
      "/w/index.php?title=Deirdre_Costello_(actress)&action=edit&redlink=1\n",
      "/wiki/Steve_Huison\n",
      "/wiki/Bruce_Jones_(actor)\n",
      "/wiki/Lesley_Sharp\n",
      "/wiki/William_Snape\n",
      "/wiki/Hugo_Speer\n",
      "/wiki/Tom_Wilkinson\n",
      "/wiki/Emily_Woof\n",
      "/wiki/Shakespeare_in_Love\n",
      "/wiki/Ben_Affleck\n",
      "/wiki/Simon_Callow\n",
      "/wiki/Jim_Carter_(actor)\n",
      "/wiki/Martin_Clunes\n",
      "/wiki/Judi_Dench\n",
      "/wiki/Joseph_Fiennes\n",
      "/wiki/Colin_Firth\n",
      "/wiki/Gwyneth_Paltrow\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Antony_Sher\n",
      "/wiki/Imelda_Staunton\n",
      "/wiki/American_Beauty_(1999_film)\n",
      "/wiki/Annette_Bening\n",
      "/wiki/Wes_Bentley\n",
      "/wiki/Thora_Birch\n",
      "/wiki/Chris_Cooper\n",
      "/wiki/Peter_Gallagher\n",
      "/wiki/Allison_Janney\n",
      "/wiki/Kevin_Spacey\n",
      "/wiki/Mena_Suvari\n",
      "/wiki/Traffic_(2000_film)\n",
      "/wiki/Steven_Bauer\n",
      "/wiki/Benjamin_Bratt\n",
      "/wiki/James_Brolin\n",
      "/wiki/Don_Cheadle\n",
      "/wiki/Erika_Christensen\n",
      "/wiki/Clifton_Collins_Jr.\n",
      "/wiki/Benicio_del_Toro\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Miguel_Ferrer\n",
      "/wiki/Albert_Finney\n",
      "/wiki/Topher_Grace\n",
      "/wiki/Luis_Guzm%C3%A1n\n",
      "/wiki/Amy_Irving\n",
      "/wiki/Tomas_Milian\n",
      "/wiki/D._W._Moffett\n",
      "/wiki/Dennis_Quaid\n",
      "/wiki/Peter_Riegert\n",
      "/wiki/Jacob_Vargas\n",
      "/wiki/Catherine_Zeta-Jones\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Template:ScreenActorsGuildAward_CastMotionPicture_1995%E2%80%932000\n",
      "/wiki/Template:ScreenActorsGuildAward_CastMotionPicture_2001%E2%80%932010\n",
      "/wiki/Template:ScreenActorsGuildAward_CastMotionPicture_2011%E2%80%932020\n",
      "/wiki/Help:Authority_control\n",
      "https://www.worldcat.org/identities/containsVIAFID/39570812\n",
      "/wiki/Virtual_International_Authority_File\n",
      "https://viaf.org/viaf/39570812\n",
      "/wiki/Library_of_Congress_Control_Number\n",
      "http://id.loc.gov/authorities/names/n88034930\n",
      "/wiki/International_Standard_Name_Identifier\n",
      "http://isni.org/isni/0000000121291300\n",
      "/wiki/Integrated_Authority_File\n",
      "https://d-nb.info/gnd/124109659\n",
      "/wiki/Syst%C3%A8me_universitaire_de_documentation\n",
      "https://www.idref.fr/084292652\n",
      "/wiki/Biblioth%C3%A8que_nationale_de_France\n",
      "http://catalogue.bnf.fr/ark:/12148/cb139817766\n",
      "http://data.bnf.fr/ark:/12148/cb139817766\n",
      "/wiki/MusicBrainz\n",
      "https://musicbrainz.org/artist/cc0dbdfc-9b2c-4e31-8448-808412388406\n",
      "/wiki/SNAC\n",
      "http://socialarchive.iath.virginia.edu/ark:/99166/w6w67gw2\n",
      "https://en.wikipedia.org/w/index.php?title=Kevin_Bacon&oldid=821876006\n",
      "/wiki/Help:Category\n",
      "/wiki/Category:1958_births\n",
      "/wiki/Category:20th-century_American_male_actors\n",
      "/wiki/Category:21st-century_American_male_actors\n",
      "/wiki/Category:American_atheists\n",
      "/wiki/Category:American_male_film_actors\n",
      "/wiki/Category:American_male_soap_opera_actors\n",
      "/wiki/Category:American_male_television_actors\n",
      "/wiki/Category:American_male_voice_actors\n",
      "/wiki/Category:The_Bacon_Brothers_members\n",
      "/wiki/Category:Best_Miniseries_or_Television_Movie_Actor_Golden_Globe_winners\n",
      "/wiki/Category:Circle_in_the_Square_Theatre_School_alumni\n",
      "/wiki/Category:Living_people\n",
      "/wiki/Category:Male_actors_from_Philadelphia\n",
      "/wiki/Category:Obie_Award_recipients\n",
      "/wiki/Category:Outstanding_Performance_by_a_Cast_in_a_Motion_Picture_Screen_Actors_Guild_Award_winners\n",
      "/wiki/Category:Sedgwick_family\n",
      "/wiki/Category:Wikipedia_indefinitely_semi-protected_biographies_of_living_people\n",
      "/wiki/Category:Use_mdy_dates_from_October_2016\n",
      "/wiki/Category:Articles_with_hCards\n",
      "/wiki/Category:All_articles_with_unsourced_statements\n",
      "/wiki/Category:Articles_with_unsourced_statements_from_January_2016\n",
      "/wiki/Category:Articles_needing_additional_references_from_October_2017\n",
      "/wiki/Category:All_articles_needing_additional_references\n",
      "/wiki/Category:Articles_with_IBDb_links\n",
      "/wiki/Category:Wikipedia_articles_with_VIAF_identifiers\n",
      "/wiki/Category:Wikipedia_articles_with_LCCN_identifiers\n",
      "/wiki/Category:Wikipedia_articles_with_ISNI_identifiers\n",
      "/wiki/Category:Wikipedia_articles_with_GND_identifiers\n",
      "/wiki/Category:Wikipedia_articles_with_BNF_identifiers\n",
      "/wiki/Category:Wikipedia_articles_with_MusicBrainz_identifiers\n",
      "/wiki/Category:Wikipedia_articles_with_SNAC-ID_identifiers\n",
      "/wiki/Special:MyTalk\n",
      "/wiki/Special:MyContributions\n",
      "/w/index.php?title=Special:CreateAccount&returnto=Kevin+Bacon\n",
      "/w/index.php?title=Special:UserLogin&returnto=Kevin+Bacon\n",
      "/wiki/Kevin_Bacon\n",
      "/wiki/Talk:Kevin_Bacon\n",
      "/wiki/Kevin_Bacon\n",
      "/w/index.php?title=Kevin_Bacon&action=edit\n",
      "/w/index.php?title=Kevin_Bacon&action=history\n",
      "/wiki/Main_Page\n",
      "/wiki/Main_Page\n",
      "/wiki/Portal:Contents\n",
      "/wiki/Portal:Featured_content\n",
      "/wiki/Portal:Current_events\n",
      "/wiki/Special:Random\n",
      "https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en\n",
      "//shop.wikimedia.org\n",
      "/wiki/Help:Contents\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:Community_portal\n",
      "/wiki/Special:RecentChanges\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "/wiki/Special:WhatLinksHere/Kevin_Bacon\n",
      "/wiki/Special:RecentChangesLinked/Kevin_Bacon\n",
      "/wiki/Wikipedia:File_Upload_Wizard\n",
      "/wiki/Special:SpecialPages\n",
      "/w/index.php?title=Kevin_Bacon&oldid=821876006\n",
      "/w/index.php?title=Kevin_Bacon&action=info\n",
      "https://www.wikidata.org/wiki/Special:EntityPage/Q3454165\n",
      "/w/index.php?title=Special:CiteThisPage&page=Kevin_Bacon&id=821876006\n",
      "/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Kevin+Bacon\n",
      "/w/index.php?title=Special:ElectronPdf&page=Kevin+Bacon&action=show-download-screen\n",
      "/w/index.php?title=Kevin_Bacon&printable=yes\n",
      "https://commons.wikimedia.org/wiki/Category:Kevin_Bacon\n",
      "https://af.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ar.wikipedia.org/wiki/%D9%83%D9%8A%D9%81%D9%8A%D9%86_%D8%A8%D9%8A%D9%83%D9%86\n",
      "https://an.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ast.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://azb.wikipedia.org/wiki/%DA%A9%D9%88%DB%8C%D9%86_%D8%A8%DB%8C%DA%A9%D9%86\n",
      "https://zh-min-nan.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://bi.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://bg.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D0%B5%D0%B9%D0%BA%D1%8A%D0%BD\n",
      "https://bs.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ca.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://cs.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://da.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://de.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://el.wikipedia.org/wiki/%CE%9A%CE%AD%CE%B2%CE%B9%CE%BD_%CE%9C%CF%80%CE%AD%CE%B9%CE%BA%CE%BF%CE%BD\n",
      "https://eml.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://es.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://eu.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://fa.wikipedia.org/wiki/%DA%A9%D9%88%DB%8C%D9%86_%D8%A8%DB%8C%DA%A9%D9%86\n",
      "https://fr.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://gl.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ko.wikipedia.org/wiki/%EC%BC%80%EB%B9%88_%EB%B2%A0%EC%9D%B4%EC%BB%A8\n",
      "https://hy.wikipedia.org/wiki/%D5%94%D6%87%D5%AB%D5%B6_%D4%B2%D5%A5%D5%B5%D6%84%D5%B8%D5%B6\n",
      "https://hr.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://io.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://id.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://it.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://he.wikipedia.org/wiki/%D7%A7%D7%95%D7%95%D7%99%D7%9F_%D7%91%D7%99%D7%99%D7%A7%D7%95%D7%9F\n",
      "https://ka.wikipedia.org/wiki/%E1%83%99%E1%83%94%E1%83%95%E1%83%98%E1%83%9C_%E1%83%91%E1%83%94%E1%83%98%E1%83%99%E1%83%9D%E1%83%9C%E1%83%98\n",
      "https://kk.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D1%8D%D0%B9%D0%BA%D0%BE%D0%BD\n",
      "https://lv.wikipedia.org/wiki/Kevins_B%C4%93kons\n",
      "https://hu.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://xmf.wikipedia.org/wiki/%E1%83%99%E1%83%94%E1%83%95%E1%83%98%E1%83%9C_%E1%83%91%E1%83%94%E1%83%98%E1%83%99%E1%83%9D%E1%83%9C%E1%83%98\n",
      "https://mn.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D1%8D%D0%B9%D0%BA%D0%BE%D0%BD\n",
      "https://nl.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ja.wikipedia.org/wiki/%E3%82%B1%E3%83%B4%E3%82%A3%E3%83%B3%E3%83%BB%E3%83%99%E3%83%BC%E3%82%B3%E3%83%B3\n",
      "https://no.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://oc.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://pl.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://pt.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ro.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ru.wikipedia.org/wiki/%D0%91%D0%B5%D0%B9%D0%BA%D0%BE%D0%BD,_%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD\n",
      "https://sco.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://simple.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://sk.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ckb.wikipedia.org/wiki/%DA%A9%DB%8E%DA%A4%D9%86_%D8%A8%DB%95%DB%8C%DA%A9%D9%86\n",
      "https://sr.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D0%B5%D1%98%D0%BA%D0%BE%D0%BD\n",
      "https://sh.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://fi.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://sv.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://th.wikipedia.org/wiki/%E0%B9%80%E0%B8%84%E0%B8%A7%E0%B8%B4%E0%B8%99_%E0%B9%80%E0%B8%9A%E0%B8%84%E0%B8%AD%E0%B8%99\n",
      "https://tr.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://uk.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D1%96%D0%BD_%D0%91%D0%B5%D0%B9%D0%BA%D0%BE%D0%BD\n",
      "https://vi.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://zh.wikipedia.org/wiki/%E5%87%AF%E6%96%87%C2%B7%E8%B4%9D%E8%82%AF\n",
      "https://www.wikidata.org/wiki/Special:EntityPage/Q3454165#sitelinks-wikipedia\n",
      "//en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License\n",
      "//creativecommons.org/licenses/by-sa/3.0/\n",
      "//wikimediafoundation.org/wiki/Terms_of_Use\n",
      "//wikimediafoundation.org/wiki/Privacy_policy\n",
      "//www.wikimediafoundation.org/\n",
      "https://wikimediafoundation.org/wiki/Privacy_policy\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:General_disclaimer\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute\n",
      "https://wikimediafoundation.org/wiki/Cookie_statement\n",
      "//en.m.wikipedia.org/w/index.php?title=Kevin_Bacon&mobileaction=toggle_view_mobile\n",
      "https://wikimediafoundation.org/\n",
      "//www.mediawiki.org/\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup \n",
    "\n",
    "html = urlopen('http://en.wikipedia.org/wiki/Kevin_Bacon')\n",
    "bs = BeautifulSoup(html, 'html.parser')\n",
    "for link in bs.find_all('a'):\n",
    "    if 'href' in link.attrs:\n",
    "        print(link.attrs['href'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieving Articles Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/Kevin_Bacon_(disambiguation)\n",
      "/wiki/San_Diego_Comic-Con\n",
      "/wiki/Philadelphia\n",
      "/wiki/Pennsylvania\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/JFK_(film)\n",
      "/wiki/A_Few_Good_Men\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Sleepers\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Fox_Broadcasting_Company\n",
      "/wiki/The_Following\n",
      "/wiki/HBO\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/Screen_Actors_Guild_Award\n",
      "/wiki/Primetime_Emmy_Award\n",
      "/wiki/The_Guardian\n",
      "/wiki/Academy_Award\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "/wiki/Social_networks\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/SixDegrees.org\n",
      "/wiki/Philadelphia\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Pennsylvania_Governor%27s_School_for_the_Arts\n",
      "/wiki/Bucknell_University\n",
      "/wiki/Glory_Van_Scott\n",
      "/wiki/Circle_in_the_Square\n",
      "/wiki/Nancy_Mills\n",
      "/wiki/Cosmopolitan_(magazine)\n",
      "/wiki/Fraternities_and_sororities\n",
      "/wiki/Animal_House\n",
      "/wiki/Search_for_Tomorrow\n",
      "/wiki/Guiding_Light\n",
      "/wiki/Friday_the_13th_(1980_film)\n",
      "/wiki/Phoenix_Theater\n",
      "/wiki/Flux\n",
      "/wiki/Second_Stage_Theatre\n",
      "/wiki/Obie_Award\n",
      "/wiki/Forty_Deuce\n",
      "/wiki/Slab_Boys\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Val_Kilmer\n",
      "/wiki/Barry_Levinson\n",
      "/wiki/Diner_(film)\n",
      "/wiki/Steve_Guttenberg\n",
      "/wiki/Daniel_Stern_(actor)\n",
      "/wiki/Mickey_Rourke\n",
      "/wiki/Tim_Daly\n",
      "/wiki/Ellen_Barkin\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/James_Dean\n",
      "/wiki/Rebel_Without_a_Cause\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Judy_Garland\n",
      "/wiki/People_(American_magazine)\n",
      "/wiki/Typecasting_(acting)\n",
      "/wiki/John_Hughes_(filmmaker)\n",
      "/wiki/She%27s_Having_a_Baby\n",
      "/wiki/The_Big_Picture_(1989_film)\n",
      "/wiki/Tremors_(film)\n",
      "/wiki/Joel_Schumacher\n",
      "/wiki/Flatliners\n",
      "/wiki/Elizabeth_Perkins\n",
      "/wiki/He_Said,_She_Said\n",
      "/wiki/The_New_York_Times\n",
      "/wiki/Oliver_Stone\n",
      "/wiki/JFK_(film)\n",
      "/wiki/A_Few_Good_Men_(film)\n",
      "/wiki/Michael_Greif\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Meryl_Streep\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Blockbuster_(entertainment)\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Sleepers_(film)\n",
      "/wiki/Picture_Perfect_(1997_film)\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Digging_to_China\n",
      "/wiki/Payola\n",
      "/wiki/Telling_Lies_in_America_(film)\n",
      "/wiki/Wild_Things_(film)\n",
      "/wiki/Stir_of_Echoes\n",
      "/wiki/David_Koepp\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Paul_Verhoeven\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Colin_Firth\n",
      "/wiki/Rachel_Blanchard\n",
      "/wiki/M%C3%A9nage_%C3%A0_trois\n",
      "/wiki/Where_the_Truth_Lies\n",
      "/wiki/Atom_Egoyan\n",
      "/wiki/MPAA\n",
      "/wiki/MPAA_film_rating_system\n",
      "/wiki/Pedophile\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/HBO_Films\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Michael_Strobl\n",
      "/wiki/Desert_Storm\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Matthew_Vaughn\n",
      "/wiki/Sebastian_Shaw_(comics)\n",
      "/wiki/Dustin_Lance_Black\n",
      "/wiki/8_(play)\n",
      "/wiki/Perry_v._Brown\n",
      "/wiki/Proposition_8\n",
      "/wiki/Charles_J._Cooper\n",
      "/wiki/Wilshire_Ebell_Theatre\n",
      "/wiki/American_Foundation_for_Equal_Rights\n",
      "/wiki/The_Following\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Huffington_Post\n",
      "/wiki/Tremors_(film)\n",
      "/wiki/EE_(telecommunications_company)\n",
      "/wiki/United_Kingdom\n",
      "/wiki/Egg_as_food\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/PBS\n",
      "/wiki/Lanford_Wilson\n",
      "/wiki/Lemon_Sky\n",
      "/wiki/Pyrates\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Upper_West_Side\n",
      "/wiki/Manhattan\n",
      "/wiki/Tracy_Pollan\n",
      "/wiki/The_Times\n",
      "/wiki/Will.i.am\n",
      "/wiki/It%27s_a_New_Day_(Will.i.am_song)\n",
      "/wiki/Barack_Obama\n",
      "/wiki/Ponzi_scheme\n",
      "/wiki/Bernard_Madoff\n",
      "/wiki/Finding_Your_Roots\n",
      "/wiki/Henry_Louis_Gates\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/Trivia\n",
      "/wiki/Big_screen\n",
      "/wiki/Six_degrees_of_separation\n",
      "/wiki/Internet_meme\n",
      "/wiki/SixDegrees.org\n",
      "/wiki/Bacon_number\n",
      "/wiki/Internet_Movie_Database\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Bacon_number\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "/wiki/The_Bacon_Brothers\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/Music_album\n",
      "/wiki/Golden_Globe_Awards\n",
      "/wiki/Golden_Globe_Award_for_Best_Supporting_Actor_%E2%80%93_Motion_Picture\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Broadcast_Film_Critics_Association_Awards\n",
      "/wiki/Broadcast_Film_Critics_Association_Award_for_Best_Actor\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Supporting_Role\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/MTV_Movie_Awards\n",
      "/wiki/MTV_Movie_Award_for_Best_Villain\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Boston_Society_of_Film_Critics_Awards\n",
      "/wiki/Boston_Society_of_Film_Critics_Award_for_Best_Cast\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Satellite_Awards\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Motion_Picture_Drama\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Beauty_Shop\n",
      "/wiki/Primetime_Emmy_Awards\n",
      "/wiki/Primetime_Emmy_Award_for_Outstanding_Lead_Actor_in_a_Miniseries_or_a_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Satellite_Awards\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Frost/Nixon_(film)\n",
      "/wiki/Golden_Globe_Awards\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Screen_Actors_Guild_Awards\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Saturn_Awards\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/People%27s_Choice_Awards\n",
      "/wiki/People%27s_Choice_Awards\n",
      "/wiki/The_Following\n",
      "/wiki/Saturn_Awards\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/Golden_Globe_Awards\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "/wiki/I_Love_Dick_(TV_series)\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/List_of_actors_with_Hollywood_Walk_of_Fame_motion_picture_stars\n",
      "/wiki/The_Austin_Chronicle\n",
      "/wiki/Access_Hollywood\n",
      "/wiki/Reuters\n",
      "/wiki/CBS_News\n",
      "/wiki/The_Verge\n",
      "/wiki/The_Hollywood_Reporter\n",
      "/wiki/Indie_Wire\n",
      "/wiki/IMDb\n",
      "/wiki/Internet_Broadway_Database\n",
      "/wiki/Lortel_Archives\n",
      "/wiki/AllMovie\n",
      "/wiki/Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Ian_McKellen\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jamie_Foxx\n",
      "/wiki/Philip_Seymour_Hoffman\n",
      "/wiki/Forest_Whitaker\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jeff_Bridges\n",
      "/wiki/Colin_Firth\n",
      "/wiki/George_Clooney\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Matthew_McConaughey\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Leonardo_DiCaprio\n",
      "/wiki/Casey_Affleck\n",
      "/wiki/Gary_Oldman\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Anthony_Andrews\n",
      "/wiki/Richard_Chamberlain\n",
      "/wiki/Ted_Danson\n",
      "/wiki/Dustin_Hoffman\n",
      "/wiki/James_Woods\n",
      "/wiki/Randy_Quaid\n",
      "/wiki/Michael_Caine\n",
      "/wiki/Stacy_Keach\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Beau_Bridges\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Ra%C3%BAl_Juli%C3%A1\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Ving_Rhames\n",
      "/wiki/Stanley_Tucci\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/James_Franco\n",
      "/wiki/Albert_Finney\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jonathan_Rhys_Meyers\n",
      "/wiki/Bill_Nighy\n",
      "/wiki/Jim_Broadbent\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Billy_Bob_Thornton\n",
      "/wiki/Oscar_Isaac\n",
      "/wiki/Tom_Hiddleston\n",
      "/wiki/Ewan_McGregor\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Kyle_Chandler\n",
      "/wiki/Steven_Weber_(actor)\n",
      "/wiki/Richard_Dean_Anderson\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Robert_Patrick\n",
      "/wiki/Ben_Browder\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Ben_Browder\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Michael_C._Hall\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Edward_James_Olmos\n",
      "/wiki/Josh_Holloway\n",
      "/wiki/Stephen_Moyer\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Mads_Mikkelsen\n",
      "/wiki/Hugh_Dancy\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Bruce_Campbell\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Ra%C3%BAl_Juli%C3%A1\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Christopher_Reeve\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/Ben_Kingsley\n",
      "/wiki/William_H._Macy\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Paul_Newman\n",
      "/wiki/Jeremy_Irons\n",
      "/wiki/Kevin_Kline\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Alexander_Skarsg%C3%A5rd\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Tom_Hanks\n",
      "/wiki/Ed_Harris\n",
      "/wiki/Bill_Paxton\n",
      "/wiki/Kathleen_Quinlan\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/The_Birdcage\n",
      "/wiki/Hank_Azaria\n",
      "/wiki/Christine_Baranski\n",
      "/wiki/Dan_Futterman\n",
      "/wiki/Gene_Hackman\n",
      "/wiki/Nathan_Lane\n",
      "/wiki/Dianne_Wiest\n",
      "/wiki/Robin_Williams\n",
      "/wiki/The_Full_Monty\n",
      "/wiki/Mark_Addy\n",
      "/wiki/Paul_Barber_(actor)\n",
      "/wiki/Robert_Carlyle\n",
      "/wiki/Steve_Huison\n",
      "/wiki/Bruce_Jones_(actor)\n",
      "/wiki/Lesley_Sharp\n",
      "/wiki/William_Snape\n",
      "/wiki/Hugo_Speer\n",
      "/wiki/Tom_Wilkinson\n",
      "/wiki/Emily_Woof\n",
      "/wiki/Shakespeare_in_Love\n",
      "/wiki/Ben_Affleck\n",
      "/wiki/Simon_Callow\n",
      "/wiki/Jim_Carter_(actor)\n",
      "/wiki/Martin_Clunes\n",
      "/wiki/Judi_Dench\n",
      "/wiki/Joseph_Fiennes\n",
      "/wiki/Colin_Firth\n",
      "/wiki/Gwyneth_Paltrow\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Antony_Sher\n",
      "/wiki/Imelda_Staunton\n",
      "/wiki/American_Beauty_(1999_film)\n",
      "/wiki/Annette_Bening\n",
      "/wiki/Wes_Bentley\n",
      "/wiki/Thora_Birch\n",
      "/wiki/Chris_Cooper\n",
      "/wiki/Peter_Gallagher\n",
      "/wiki/Allison_Janney\n",
      "/wiki/Kevin_Spacey\n",
      "/wiki/Mena_Suvari\n",
      "/wiki/Traffic_(2000_film)\n",
      "/wiki/Steven_Bauer\n",
      "/wiki/Benjamin_Bratt\n",
      "/wiki/James_Brolin\n",
      "/wiki/Don_Cheadle\n",
      "/wiki/Erika_Christensen\n",
      "/wiki/Clifton_Collins_Jr.\n",
      "/wiki/Benicio_del_Toro\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Miguel_Ferrer\n",
      "/wiki/Albert_Finney\n",
      "/wiki/Topher_Grace\n",
      "/wiki/Luis_Guzm%C3%A1n\n",
      "/wiki/Amy_Irving\n",
      "/wiki/Tomas_Milian\n",
      "/wiki/D._W._Moffett\n",
      "/wiki/Dennis_Quaid\n",
      "/wiki/Peter_Riegert\n",
      "/wiki/Jacob_Vargas\n",
      "/wiki/Catherine_Zeta-Jones\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Virtual_International_Authority_File\n",
      "/wiki/Library_of_Congress_Control_Number\n",
      "/wiki/International_Standard_Name_Identifier\n",
      "/wiki/Integrated_Authority_File\n",
      "/wiki/Syst%C3%A8me_universitaire_de_documentation\n",
      "/wiki/Biblioth%C3%A8que_nationale_de_France\n",
      "/wiki/MusicBrainz\n",
      "/wiki/SNAC\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen \n",
    "from bs4 import BeautifulSoup \n",
    "import re\n",
    "\n",
    "html = urlopen('http://en.wikipedia.org/wiki/Kevin_Bacon')\n",
    "bs = BeautifulSoup(html, 'html.parser')\n",
    "for link in bs.find('div', {'id':'bodyContent'}).find_all(\n",
    "    'a', href=re.compile('^(/wiki/)((?!:).)*$')):\n",
    "    if 'href' in link.attrs:\n",
    "        print(link.attrs['href'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Walk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/MusicBrainz\n",
      "/wiki/ICANN\n",
      "/wiki/U.S._House_of_Representatives\n",
      "/wiki/United_States_Copyright_Office\n",
      "/wiki/U.S._representative_bibliography_(congressional_memoirs)\n",
      "/wiki/United_States_Senate\n",
      "/wiki/Richard_A._Baker_(historian)\n",
      "/wiki/Robert_Byrd\n",
      "/wiki/Washington_College_of_Law\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ee18b17d20d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mnewArticle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlinks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlinks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'href'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewArticle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mlinks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewArticle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-1-ee18b17d20d4>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(articleUrl)\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marticleUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mhtml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://en.wikipedia.org{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marticleUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'html.parser'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mbs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'div'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m'bodyContent'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhref\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'^(/wiki/)((?!:).)*$'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bs4/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, **kwargs)\u001b[0m\n\u001b[1;32m    213\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    216\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mParserRejectedMarkup\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bs4/__init__.py\u001b[0m in \u001b[0;36m_feed\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    237\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 239\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmarkup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    240\u001b[0m         \u001b[0;31m# Close out any unfinished strings and close all the open tags.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bs4/builder/_htmlparser.py\u001b[0m in \u001b[0;36mfeed\u001b[0;34m(self, markup)\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m             \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmarkup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    165\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mHTMLParseError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m             warnings.warn(RuntimeWarning(\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/html/parser.py\u001b[0m in \u001b[0;36mfeed\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    109\u001b[0m         \"\"\"\n\u001b[1;32m    110\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrawdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrawdata\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgoahead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/html/parser.py\u001b[0m in \u001b[0;36mgoahead\u001b[0;34m(self, end)\u001b[0m\n\u001b[1;32m    164\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrawdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 166\u001b[0;31m             \u001b[0mi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdatepos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    167\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m             \u001b[0mstartswith\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrawdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/_markupbase.py\u001b[0m in \u001b[0;36mupdatepos\u001b[0;34m(self, i, j)\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0mrawdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrawdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0mnlines\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrawdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mnlines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     54\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlineno\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlineno\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnlines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0mpos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrawdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Should not fail\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "import random\n",
    "import re\n",
    "\n",
    "random.seed(datetime.datetime.now())\n",
    "def getLinks(articleUrl):\n",
    "    html = urlopen('http://en.wikipedia.org{}'.format(articleUrl))\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    return bs.find('div', {'id':'bodyContent'}).find_all('a', href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "\n",
    "links = getLinks('/wiki/Kevin_Bacon')\n",
    "while len(links) > 0:\n",
    "    newArticle = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "    print(newArticle)\n",
    "    links = getLinks(newArticle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recursively crawling an entire site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/Wikipedia\n",
      "/wiki/Wikipedia:Protection_policy#semi\n",
      "/wiki/Wikipedia:Requests_for_page_protection\n",
      "/wiki/Wikipedia:Requests_for_permissions\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-94457f9052fa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-4-94457f9052fa>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-94457f9052fa>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-94457f9052fa>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-94457f9052fa>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-94457f9052fa>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpageUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mglobal\u001b[0m \u001b[0mpages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mhtml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://en.wikipedia.org{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpageUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'html.parser'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhref\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'^(/wiki/)'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    221\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0mopener\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_opener\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mopener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minstall_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprocessor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             \u001b[0mmeth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mcode\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m             response = self.parent.error(\n\u001b[0;32m--> 642\u001b[0;31m                 'http', request, response, code, msg, hdrs)\n\u001b[0m\u001b[1;32m    643\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36merror\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    562\u001b[0m             \u001b[0mhttp_err\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 564\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_chain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    754\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 756\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    757\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m     \u001b[0mhttp_error_301\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_303\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_307\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_302\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    524\u001b[0m             \u001b[0mreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 526\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m         \u001b[0;31m# post-process response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    542\u001b[0m         \u001b[0mprotocol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m         result = self._call_chain(self.handle_open, protocol, protocol +\n\u001b[0;32m--> 544\u001b[0;31m                                   '_open', req)\n\u001b[0m\u001b[1;32m    545\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttps_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1359\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mhttps_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m             return self.do_open(http.client.HTTPSConnection, req,\n\u001b[0;32m-> 1361\u001b[0;31m                 context=self._context, check_hostname=self._check_hostname)\n\u001b[0m\u001b[1;32m   1362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m         \u001b[0mhttps_request\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAbstractHTTPHandler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdo_request_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mdo_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1319\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# timeout error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mURLError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1321\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1322\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m             \u001b[0mh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1329\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1331\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1332\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    298\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    259\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1000\u001b[0m                   \u001b[0;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m                   self.__class__)\n\u001b[0;32m-> 1002\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1003\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1004\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    863\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Read on closed or unwrapped SSL socket.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    864\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 865\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    866\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mSSLError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    867\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mSSL_ERROR_EOF\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuppress_ragged_eofs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    623\u001b[0m         \"\"\"\n\u001b[1;32m    624\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 625\u001b[0;31m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    626\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "pages = set()\n",
    "def getLinks(pageUrl):\n",
    "    global pages\n",
    "    html = urlopen('http://en.wikipedia.org{}'.format(pageUrl))\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    for link in bs.find_all('a', href=re.compile('^(/wiki/)')):\n",
    "        if 'href' in link.attrs:\n",
    "            if link.attrs['href'] not in pages:\n",
    "                #We have encountered a new page\n",
    "                newPage = link.attrs['href']\n",
    "                print(newPage)\n",
    "                pages.add(newPage)\n",
    "                getLinks(newPage)\n",
    "getLinks('')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting Data Across an Entire Site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Main Page\n",
      "<p>The <b><a href=\"/wiki/Finnish_Civil_War\" title=\"Finnish Civil War\">Finnish Civil War</a></b> (27 January – 15 May 1918) marked the transition from the <a href=\"/wiki/Grand_Duchy_of_Finland\" title=\"Grand Duchy of Finland\">Grand Duchy of Finland</a>, part of the <a href=\"/wiki/Russian_Empire\" title=\"Russian Empire\">Russian Empire</a>, to an independent state. Arising during <a href=\"/wiki/World_War_I\" title=\"World War I\">World War I</a>, it was fought between the Reds, led by the <a href=\"/wiki/Social_Democratic_Party_of_Finland\" title=\"Social Democratic Party of Finland\">Social Democratic Party</a>, and the Whites, led by the conservative <a href=\"/wiki/Senate_of_Finland\" title=\"Senate of Finland\">Senate</a>. In February 1918, the Reds carried out an unsuccessful offensive, supplied with weapons by <a class=\"mw-redirect\" href=\"/wiki/Soviet_Russia\" title=\"Soviet Russia\">Soviet Russia</a>. A counteroffensive by the Whites began in March, reinforced by the <a href=\"/wiki/German_Empire\" title=\"German Empire\">German Empire</a>'s military detachments in April. The decisive engagements were the battles of <a href=\"/wiki/Battle_of_Tampere\" title=\"Battle of Tampere\">Tampere</a> and <a href=\"/wiki/Battle_of_Vyborg\" title=\"Battle of Vyborg\">Vyborg</a>, won by the Whites, and the battles of <a href=\"/wiki/Battle_of_Helsinki\" title=\"Battle of Helsinki\">Helsinki</a> and <a href=\"/wiki/Battle_of_Lahti\" title=\"Battle of Lahti\">Lahti</a>, won by German troops, leading to overall victory for the Whites and the German forces. The 39,000 casualties included <a class=\"mw-redirect\" href=\"/wiki/Political_terror\" title=\"Political terror\">political terror</a> deaths. Although the Senate and Parliament were initially pressured into accepting the <a href=\"/wiki/Prince_Frederick_Charles_of_Hesse\" title=\"Prince Frederick Charles of Hesse\">brother-in-law</a> of German Emperor <a class=\"mw-redirect\" href=\"/wiki/William_II,_German_Emperor\" title=\"William II, German Emperor\">William II</a> as the <a href=\"/wiki/Kingdom_of_Finland_(1918)\" title=\"Kingdom of Finland (1918)\">King of Finland</a>, the country emerged within a few months as an independent, democratic republic. The war would divide the nation for decades. (<a href=\"/wiki/Finnish_Civil_War\" title=\"Finnish Civil War\"><b>Full article...</b></a>)</p>\n",
      "This page is missing something! No worries, we will continue!\n",
      "--------------------\n",
      "/wiki/Wikipedia\n",
      "Wikipedia\n",
      "<p><b>Wikipedia</b> (<span class=\"nowrap\"><span class=\"IPA nopopups noexcerpt\"><a href=\"/wiki/Help:IPA/English\" title=\"Help:IPA/English\">/<span style=\"border-bottom:1px dotted\"><span title=\"/ˌ/: secondary stress follows\">ˌ</span><span title=\"'w' in 'wind'\">w</span><span title=\"/ɪ/: 'i' in 'kit'\">ɪ</span><span title=\"'k' in 'kind'\">k</span><span title=\"/ɪ/: 'i' in 'kit'\">ɪ</span><span title=\"/ˈ/: primary stress follows\">ˈ</span><span title=\"'p' in 'pie'\">p</span><span title=\"/iː/: 'ee' in 'fleece'\">iː</span><span title=\"'d' in 'dye'\">d</span><span title=\"/i/: 'y' in 'happy'\">i</span><span title=\"/ə/: 'a' in 'about'\">ə</span></span>/</a></span><small class=\"nowrap\"> (<span class=\"unicode haudio\"><span class=\"fn\"><span style=\"white-space:nowrap\"><a href=\"/wiki/File:GT_Wikipedia_BE.ogg\" title=\"About this sound\"><img alt=\"About this sound\" data-file-height=\"20\" data-file-width=\"20\" height=\"11\" src=\"//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/11px-Loudspeaker.svg.png\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/17px-Loudspeaker.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/22px-Loudspeaker.svg.png 2x\" width=\"11\"/></a> </span><a class=\"internal\" href=\"//upload.wikimedia.org/wikipedia/commons/0/01/GT_Wikipedia_BE.ogg\" title=\"GT Wikipedia BE.ogg\">listen</a></span></span>)</small></span> <a href=\"/wiki/Help:Pronunciation_respelling_key\" title=\"Help:Pronunciation respelling key\"><i title=\"English pronunciation respelling\"><span style=\"font-size:90%\">WIK</span>-i-<span style=\"font-size:90%\">PEE</span>-dee-ə</i></a> or <span class=\"nowrap\"><span class=\"IPA nopopups noexcerpt\"><a href=\"/wiki/Help:IPA/English\" title=\"Help:IPA/English\">/<span style=\"border-bottom:1px dotted\"><span title=\"/ˌ/: secondary stress follows\">ˌ</span><span title=\"'w' in 'wind'\">w</span><span title=\"/ɪ/: 'i' in 'kit'\">ɪ</span><span title=\"'k' in 'kind'\">k</span><span title=\"/i/: 'y' in 'happy'\">i</span><span title=\"/ˈ/: primary stress follows\">ˈ</span><span title=\"'p' in 'pie'\">p</span><span title=\"/iː/: 'ee' in 'fleece'\">iː</span><span title=\"'d' in 'dye'\">d</span><span title=\"/i/: 'y' in 'happy'\">i</span><span title=\"/ə/: 'a' in 'about'\">ə</span></span>/</a></span><small class=\"nowrap\"> (<span class=\"unicode haudio\"><span class=\"fn\"><span style=\"white-space:nowrap\"><a href=\"/wiki/File:GT_Wikipedia_AE.ogg\" title=\"About this sound\"><img alt=\"About this sound\" data-file-height=\"20\" data-file-width=\"20\" height=\"11\" src=\"//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/11px-Loudspeaker.svg.png\" srcset=\"//upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/17px-Loudspeaker.svg.png 1.5x, //upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Loudspeaker.svg/22px-Loudspeaker.svg.png 2x\" width=\"11\"/></a> </span><a class=\"internal\" href=\"//upload.wikimedia.org/wikipedia/commons/4/4c/GT_Wikipedia_AE.ogg\" title=\"GT Wikipedia AE.ogg\">listen</a></span></span>)</small></span> <a href=\"/wiki/Help:Pronunciation_respelling_key\" title=\"Help:Pronunciation respelling key\"><i title=\"English pronunciation respelling\"><span style=\"font-size:90%\">WIK</span>-ee-<span style=\"font-size:90%\">PEE</span>-dee-ə</i></a>) is a free <a href=\"/wiki/Online_encyclopedia\" title=\"Online encyclopedia\">online encyclopedia</a> with the mission of allowing anyone to edit articles.<sup class=\"reference\" id=\"cite_ref-6\"><a href=\"#cite_note-6\">[3]</a></sup><sup class=\"noprint Inline-Template\" style=\"white-space:nowrap;\">[<i><a href=\"/wiki/Wikipedia:Verifiability\" title=\"Wikipedia:Verifiability\"><span title=\"The material near this tag failed verification of its source citation(s). (January 2018)\">not in citation given</span></a></i>]</sup> Wikipedia is the largest and most popular general <a href=\"/wiki/Reference_work\" title=\"Reference work\">reference work</a> on the Internet,<sup class=\"reference\" id=\"cite_ref-Tancer_7-0\"><a href=\"#cite_note-Tancer-7\">[4]</a></sup><sup class=\"reference\" id=\"cite_ref-Woodson_8-0\"><a href=\"#cite_note-Woodson-8\">[5]</a></sup><sup class=\"reference\" id=\"cite_ref-9\"><a href=\"#cite_note-9\">[6]</a></sup> and is ranked the fifth-most popular website.<sup class=\"reference\" id=\"cite_ref-Alexa_siteinfo_10-0\"><a href=\"#cite_note-Alexa_siteinfo-10\">[7]</a></sup> Wikipedia is owned by the nonprofit <a href=\"/wiki/Wikimedia_Foundation\" title=\"Wikimedia Foundation\">Wikimedia Foundation</a>.<sup class=\"reference\" id=\"cite_ref-11\"><a href=\"#cite_note-11\">[8]</a></sup><sup class=\"reference\" id=\"cite_ref-12\"><a href=\"#cite_note-12\">[9]</a></sup><sup class=\"reference\" id=\"cite_ref-13\"><a href=\"#cite_note-13\">[10]</a></sup></p>\n",
      "This page is missing something! No worries, we will continue!\n",
      "--------------------\n",
      "/wiki/Wikipedia:Protection_policy#semi\n",
      "Wikipedia:Protection policy\n",
      "<p>Wikipedia is built around the principle that <a href=\"/wiki/Wiki\" title=\"Wiki\">anyone can edit it</a>, and it therefore aims to have as many of its pages as possible open for public editing so that anyone can add material and correct errors. However, in some particular circumstances, because of a specifically identified likelihood of damage resulting if editing is left open, some individual pages may need to be subject to technical restrictions (often only temporary but sometimes indefinitely) on who is permitted to modify them. The placing of such restrictions on pages is called <b>protection</b>.</p>\n",
      "This page is missing something! No worries, we will continue!\n",
      "--------------------\n",
      "/wiki/Wikipedia:Requests_for_page_protection\n",
      "Wikipedia:Requests for page protection\n",
      "<p>This page is for requesting that a page, file or template be <b>fully protected</b>, <b>create protected</b> (<a href=\"/wiki/Wikipedia:Protection_policy#Creation_protection\" title=\"Wikipedia:Protection policy\">salted</a>), <b>extended confirmed protected</b>, <b>semi-protected</b>, added to <b>pending changes</b>, <b>move-protected</b>, <b>template protected</b> (template-specific), <b>upload protected</b> (file-specific), or <b>unprotected</b>. Please read up on the <a href=\"/wiki/Wikipedia:Protection_policy\" title=\"Wikipedia:Protection policy\">protection policy</a>. Full protection is used to stop edit warring between multiple users or to prevent vandalism to <a href=\"/wiki/Wikipedia:High-risk_templates\" title=\"Wikipedia:High-risk templates\">high-risk templates</a>; semi-protection and pending changes are usually used only to prevent IP and new user vandalism (see the <a href=\"/wiki/Wikipedia:Rough_guide_to_semi-protection\" title=\"Wikipedia:Rough guide to semi-protection\">rough guide to semi-protection</a>); and move protection is used to stop <a href=\"/wiki/Wikipedia:Moving_a_page\" title=\"Wikipedia:Moving a page\">pagemove</a> revert wars. Extended confirmed protection is used where semi-protection has proved insufficient (see the <a href=\"/wiki/Wikipedia:Rough_guide_to_extended_confirmed_protection\" title=\"Wikipedia:Rough guide to extended confirmed protection\">rough guide to extended confirmed protection</a>)</p>\n",
      "/w/index.php?title=Wikipedia:Requests_for_page_protection&action=edit\n",
      "--------------------\n",
      "/wiki/Wikipedia:Requests_for_permissions\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wikipedia:Requests for permissions\n",
      "<p><span class=\"sysop-show\" id=\"coordinates\"><a href=\"/wiki/Wikipedia:Requests_for_permissions/Administrator_instructions\" title=\"Wikipedia:Requests for permissions/Administrator instructions\">Administrator instructions</a></span></p>\n",
      "This page is missing something! No worries, we will continue!\n",
      "--------------------\n",
      "/wiki/Wikipedia:Requesting_copyright_permission\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     23\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     23\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     23\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     23\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     23\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mpages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnewPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-11-79989a68f53f>\u001b[0m in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpageUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mglobal\u001b[0m \u001b[0mpages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mhtml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://en.wikipedia.org{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpageUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mbsObj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'html.parser'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    221\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0mopener\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_opener\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mopener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minstall_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprocessor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             \u001b[0mmeth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mcode\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m             response = self.parent.error(\n\u001b[0;32m--> 642\u001b[0;31m                 'http', request, response, code, msg, hdrs)\n\u001b[0m\u001b[1;32m    643\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36merror\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    562\u001b[0m             \u001b[0mhttp_err\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 564\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_chain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    754\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 756\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    757\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m     \u001b[0mhttp_error_301\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_303\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_307\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_302\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    524\u001b[0m             \u001b[0mreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 526\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m         \u001b[0;31m# post-process response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    542\u001b[0m         \u001b[0mprotocol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m         result = self._call_chain(self.handle_open, protocol, protocol +\n\u001b[0;32m--> 544\u001b[0;31m                                   '_open', req)\n\u001b[0m\u001b[1;32m    545\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttps_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1359\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mhttps_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m             return self.do_open(http.client.HTTPSConnection, req,\n\u001b[0;32m-> 1361\u001b[0;31m                 context=self._context, check_hostname=self._check_hostname)\n\u001b[0m\u001b[1;32m   1362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m         \u001b[0mhttps_request\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAbstractHTTPHandler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdo_request_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mdo_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1316\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1317\u001b[0m                 h.request(req.get_method(), req.selector, req.data, headers,\n\u001b[0;32m-> 1318\u001b[0;31m                           encode_chunked=req.has_header('Transfer-encoding'))\n\u001b[0m\u001b[1;32m   1319\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# timeout error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mURLError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1237\u001b[0m                 encode_chunked=False):\n\u001b[1;32m   1238\u001b[0m         \u001b[0;34m\"\"\"Send a complete request to the server.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1283\u001b[0m             \u001b[0;31m# default charset of iso-8859-1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1284\u001b[0m             \u001b[0mbody\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_encode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'body'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1285\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendheaders\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1287\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mendheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1232\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1233\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mCannotSendHeader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1234\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage_body\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1236\u001b[0m     def request(self, method, url, body=None, headers={}, *,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1024\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mb\"\\r\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1025\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1026\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1027\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmessage_body\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    962\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_open\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 964\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    965\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mNotConnected\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1390\u001b[0m             \u001b[0;34m\"Connect to a host on a given (SSL) port.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1392\u001b[0;31m             \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1393\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1394\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tunnel_host\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    934\u001b[0m         \u001b[0;34m\"\"\"Connect to the host and port specified in __init__.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    935\u001b[0m         self.sock = self._create_connection(\n\u001b[0;32m--> 936\u001b[0;31m             (self.host,self.port), self.timeout, self.source_address)\n\u001b[0m\u001b[1;32m    937\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetsockopt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIPPROTO_TCP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTCP_NODELAY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    938\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address)\u001b[0m\n\u001b[1;32m    711\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0msource_address\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m                 \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_address\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 713\u001b[0;31m             \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    714\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    715\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "pages = set()\n",
    "def getLinks(pageUrl):\n",
    "    global pages\n",
    "    html = urlopen('http://en.wikipedia.org{}'.format(pageUrl))\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    try:\n",
    "        print(bs.h1.get_text())\n",
    "        print(bs.find(id ='mw-content-text').find_all('p')[0])\n",
    "        print(bs.find(id='ca-edit').find('span').find('a').attrs['href'])\n",
    "    except AttributeError:\n",
    "        print('This page is missing something! Continuing.')\n",
    "    \n",
    "    for link in bs.find_all('a', href=re.compile('^(/wiki/)')):\n",
    "        if 'href' in link.attrs:\n",
    "            if link.attrs['href'] not in pages:\n",
    "                #We have encountered a new page\n",
    "                newPage = link.attrs['href']\n",
    "                print('-'*20)\n",
    "                print(newPage)\n",
    "                pages.add(newPage)\n",
    "                getLinks(newPage)\n",
    "getLinks('') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawling across the Internet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random external link is: http://twitter.com/oreillymedia\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-a23b136d3234>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0mfollowExternalOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexternalLink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m \u001b[0mfollowExternalOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-6-a23b136d3234>\u001b[0m in \u001b[0;36mfollowExternalOnly\u001b[0;34m(startingSite)\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0mexternalLink\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetRandomExternalLink\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstartingSite\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Random external link is: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexternalLink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m     \u001b[0mfollowExternalOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexternalLink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0mfollowExternalOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-a23b136d3234>\u001b[0m in \u001b[0;36mfollowExternalOnly\u001b[0;34m(startingSite)\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfollowExternalOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstartingSite\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m     \u001b[0mexternalLink\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetRandomExternalLink\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstartingSite\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Random external link is: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexternalLink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0mfollowExternalOnly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexternalLink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-a23b136d3234>\u001b[0m in \u001b[0;36mgetRandomExternalLink\u001b[0;34m(startingPage)\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetRandomExternalLink\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstartingPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0mhtml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstartingPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m     \u001b[0mbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'html.parser'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0mexternalLinks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murlparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstartingPage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetloc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexternalLinks\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bs4/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 228\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    229\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mParserRejectedMarkup\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bs4/__init__.py\u001b[0m in \u001b[0;36m_feed\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    287\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 289\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmarkup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    290\u001b[0m         \u001b[0;31m# Close out any unfinished strings and close all the open tags.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/bs4/builder/_htmlparser.py\u001b[0m in \u001b[0;36mfeed\u001b[0;34m(self, markup)\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m             \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmarkup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    216\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mHTMLParseError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m             warnings.warn(RuntimeWarning(\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/html/parser.py\u001b[0m in \u001b[0;36mfeed\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    109\u001b[0m         \"\"\"\n\u001b[1;32m    110\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrawdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrawdata\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgoahead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/html/parser.py\u001b[0m in \u001b[0;36mgoahead\u001b[0;34m(self, end)\u001b[0m\n\u001b[1;32m    169\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'<'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mstarttagopen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrawdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# < + letter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m                     \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_starttag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"</\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m                     \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_endtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/html/parser.py\u001b[0m in \u001b[0;36mparse_starttag\u001b[0;34m(self, i)\u001b[0m\n\u001b[1;32m    301\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mparse_starttag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__starttag_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 303\u001b[0;31m         \u001b[0mendpos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_for_whole_start_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    304\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mendpos\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    305\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mendpos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/html/parser.py\u001b[0m in \u001b[0;36mcheck_for_whole_start_tag\u001b[0;34m(self, i)\u001b[0m\n\u001b[1;32m    355\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m             \u001b[0mj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mnext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrawdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mnext\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\">\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mj\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from urllib.parse import urlparse\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import datetime\n",
    "import random\n",
    "\n",
    "pages = set()\n",
    "random.seed(datetime.datetime.now())\n",
    "\n",
    "#Retrieves a list of all Internal links found on a page\n",
    "def getInternalLinks(bs, includeUrl):\n",
    "    includeUrl = '{}://{}'.format(urlparse(includeUrl).scheme, urlparse(includeUrl).netloc)\n",
    "    internalLinks = []\n",
    "    #Finds all links that begin with a \"/\"\n",
    "    for link in bs.find_all('a', href=re.compile('^(/|.*'+includeUrl+')')):\n",
    "        if link.attrs['href'] is not None:\n",
    "            if link.attrs['href'] not in internalLinks:\n",
    "                if(link.attrs['href'].startswith('/')):\n",
    "                    internalLinks.append(includeUrl+link.attrs['href'])\n",
    "                else:\n",
    "                    internalLinks.append(link.attrs['href'])\n",
    "    return internalLinks\n",
    "            \n",
    "#Retrieves a list of all external links found on a page\n",
    "def getExternalLinks(bs, excludeUrl):\n",
    "    externalLinks = []\n",
    "    #Finds all links that start with \"http\" that do\n",
    "    #not contain the current URL\n",
    "    for link in bs.find_all('a', href=re.compile('^(http|www)((?!'+excludeUrl+').)*$')):\n",
    "        if link.attrs['href'] is not None:\n",
    "            if link.attrs['href'] not in externalLinks:\n",
    "                externalLinks.append(link.attrs['href'])\n",
    "    return externalLinks\n",
    "\n",
    "def getRandomExternalLink(startingPage):\n",
    "    html = urlopen(startingPage)\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    externalLinks = getExternalLinks(bs, urlparse(startingPage).netloc)\n",
    "    if len(externalLinks) == 0:\n",
    "        print('No external links, looking around the site for one')\n",
    "        domain = '{}://{}'.format(urlparse(startingPage).scheme, urlparse(startingPage).netloc)\n",
    "        internalLinks = getInternalLinks(bs, domain)\n",
    "        return getRandomExternalLink(internalLinks[random.randint(0,\n",
    "                                    len(internalLinks)-1)])\n",
    "    else:\n",
    "        return externalLinks[random.randint(0, len(externalLinks)-1)]\n",
    "    \n",
    "def followExternalOnly(startingSite):\n",
    "    externalLink = getRandomExternalLink(startingSite)\n",
    "    print('Random external link is: {}'.format(externalLink))\n",
    "    followExternalOnly(externalLink)\n",
    "            \n",
    "followExternalOnly('http://oreilly.com')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect all External Links from a Site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.oreilly.com\n",
      "http://www.oreilly.com/ideas\n",
      "https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170601+nav\n",
      "http://www.oreilly.com/conferences/\n",
      "http://shop.oreilly.com/\n",
      "http://members.oreilly.com\n",
      "https://www.oreilly.com/topics\n",
      "https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170505+homepage+get+started+now\n",
      "https://www.safaribooksonline.com/accounts/login/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170203+homepage+sign+in\n",
      "https://www.safaribooksonline.com/live-training/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+take+a+live+online+course\n",
      "https://www.safaribooksonline.com/learning-paths/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+follow+a+path\n",
      "https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170505+homepage+unlimited+access\n",
      "http://www.oreilly.com/live-training/?view=grid\n",
      "https://www.safaribooksonline.com/your-experience/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170201+homepage+safari+platform\n",
      "https://cdn.oreillystatic.com/pdf/oreilly_high_performance_organizations_whitepaper.pdf\n",
      "https://www.oreilly.com/ideas/8-data-trends-on-our-radar-for-2017?utm_medium=referral&utm_source=oreilly.com&utm_campaign=lgen&utm_content=link+2017+trends\n",
      "https://www.oreilly.com/ideas?utm_medium=referral&utm_source=oreilly.com&utm_campaign=lgen&utm_content=link+read+latest+articles\n",
      "http://www.oreilly.com/about/\n",
      "http://www.oreilly.com/work-with-us.html\n",
      "http://www.oreilly.com/careers/\n",
      "http://shop.oreilly.com/category/customer-service.do\n",
      "http://www.oreilly.com/about/contact.html\n",
      "http://twitter.com/oreillymedia\n",
      "http://fb.co/OReilly\n",
      "https://www.linkedin.com/company/oreilly-media\n",
      "https://www.youtube.com/user/OreillyMedia\n",
      "http://www.oreilly.com/emails/newsletters/\n",
      "http://www.oreilly.com/terms/\n",
      "http://www.oreilly.com/privacy.html\n",
      "http://www.oreilly.com/about/editorial_independence.html\n",
      "https://www.safaribooksonline.com/home/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170601+nav\n",
      "http://www.oreilly.com/emails/newsletters/?display=preview#ai\n",
      "https://www.oreilly.com/privacy.html\n",
      "http://www.oreilly.com/ai/building-bots-with-natural-language-processing.csp?utm_source=oreilly&utm_medium=newsite&utm_campaign=ai-topic-cta\n",
      "https://www.flickr.com/photos/ebmorse/7536436622/in/dateposted/\n",
      "http://oreilly.com/about/\n",
      "http://oreilly.com/work-with-us.html\n",
      "http://oreilly.com/careers/\n",
      "https://plus.google.com/+oreillymedia\n",
      "http://oreilly.com/terms/\n",
      "http://oreilly.com/privacy.html\n",
      "https://www.safaribooksonline.com/home/?utm_source=newsite&utm_medium=content&utm_campaign=lgen&utm_content=business-topic-cta\n",
      "https://www.flickr.com/photos/pagedooley/4253159181/\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-846c414ed600>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mgetAllExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-8-846c414ed600>\u001b[0m in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(siteUrl)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0mgetAllExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-846c414ed600>\u001b[0m in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(siteUrl)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0mgetAllExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-846c414ed600>\u001b[0m in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(siteUrl)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0mgetAllExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-846c414ed600>\u001b[0m in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(siteUrl)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0mgetAllExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mallIntLinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://oreilly.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-846c414ed600>\u001b[0m in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(siteUrl)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mallIntLinks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgetAllExternalLinks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msiteUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mhtml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msiteUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mdomain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'{}://{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murlparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msiteUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscheme\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murlparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msiteUrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetloc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'html.parser'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    221\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0mopener\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_opener\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mopener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minstall_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprocessor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             \u001b[0mmeth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mcode\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m             response = self.parent.error(\n\u001b[0;32m--> 642\u001b[0;31m                 'http', request, response, code, msg, hdrs)\n\u001b[0m\u001b[1;32m    643\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36merror\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    562\u001b[0m             \u001b[0mhttp_err\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 564\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_chain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    754\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 756\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    757\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m     \u001b[0mhttp_error_301\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_303\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_307\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_302\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprocessor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             \u001b[0mmeth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m200\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mcode\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m             response = self.parent.error(\n\u001b[0;32m--> 642\u001b[0;31m                 'http', request, response, code, msg, hdrs)\n\u001b[0m\u001b[1;32m    643\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36merror\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    562\u001b[0m             \u001b[0mhttp_err\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m         \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 564\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_chain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttp_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    754\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 756\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    757\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m     \u001b[0mhttp_error_301\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_303\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_307\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_error_302\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    524\u001b[0m             \u001b[0mreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 526\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m         \u001b[0;31m# post-process response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    542\u001b[0m         \u001b[0mprotocol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m         result = self._call_chain(self.handle_open, protocol, protocol +\n\u001b[0;32m--> 544\u001b[0;31m                                   '_open', req)\n\u001b[0m\u001b[1;32m    545\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mhttps_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1359\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mhttps_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m             return self.do_open(http.client.HTTPSConnection, req,\n\u001b[0;32m-> 1361\u001b[0;31m                 context=self._context, check_hostname=self._check_hostname)\n\u001b[0m\u001b[1;32m   1362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m         \u001b[0mhttps_request\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAbstractHTTPHandler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdo_request_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py\u001b[0m in \u001b[0;36mdo_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1319\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# timeout error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mURLError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1321\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1322\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m             \u001b[0mh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1329\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1331\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1332\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    298\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    259\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1000\u001b[0m                   \u001b[0;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m                   self.__class__)\n\u001b[0;32m-> 1002\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1003\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1004\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    863\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Read on closed or unwrapped SSL socket.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    864\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 865\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    866\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mSSLError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    867\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mSSL_ERROR_EOF\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuppress_ragged_eofs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    623\u001b[0m         \"\"\"\n\u001b[1;32m    624\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 625\u001b[0;31m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    626\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Collects a list of all external URLs found on the site\n",
    "allExtLinks = set()\n",
    "allIntLinks = set()\n",
    "\n",
    "\n",
    "def getAllExternalLinks(siteUrl):\n",
    "    html = urlopen(siteUrl)\n",
    "    domain = '{}://{}'.format(urlparse(siteUrl).scheme,\n",
    "                              urlparse(siteUrl).netloc)\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    internalLinks = getInternalLinks(bs, domain)\n",
    "    externalLinks = getExternalLinks(bs, domain)\n",
    "\n",
    "    for link in externalLinks:\n",
    "        if link not in allExtLinks:\n",
    "            allExtLinks.add(link)\n",
    "            print(link)\n",
    "    for link in internalLinks:\n",
    "        if link not in allIntLinks:\n",
    "            allIntLinks.add(link)\n",
    "            getAllExternalLinks(link)\n",
    "\n",
    "\n",
    "allIntLinks.add('http://oreilly.com')\n",
    "getAllExternalLinks('http://oreilly.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}