Web scraping action.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import urllib\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = urllib.urlopen('https://en.wikipedia.org/wiki/Web_scraping').read()\n",
    "soup = BeautifulSoup(r,'lxml')\n",
    "type(soup)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html class=\"client-nojs\" dir=\"ltr\" lang=\"en\">\n",
      " <head>\n",
      "  <meta charset=\"utf-8\"/>\n",
      "  <\n"
     ]
    }
   ],
   "source": [
    "print soup.prettify()[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "#mw-head\n",
      "#p-search\n",
      "/wiki/File:Question_book-new.svg\n",
      "/wiki/Wikipedia:Verifiability\n",
      "//en.wikipedia.org/w/index.php?title=Web_scraping&action=edit\n",
      "/wiki/Help:Introduction_to_referencing_with_Wiki_Markup/1\n",
      "/wiki/Help:Maintenance_template_removal\n",
      "/wiki/Data_scraping\n",
      "/wiki/Data_scraping\n",
      "/wiki/Data_extraction\n",
      "/wiki/Website\n",
      "#cite_note-Boeing2016JPER-1\n",
      "/wiki/Hypertext_Transfer_Protocol\n",
      "/wiki/Internet_bot\n",
      "/wiki/Web_crawler\n",
      "/wiki/Database\n",
      "/wiki/Data_retrieval\n",
      "/wiki/Data_analysis\n",
      "#cite_note-Boeing2016JPER-1\n",
      "#cite_note-2\n",
      "/wiki/Parsing\n",
      "/wiki/Contact_scraping\n",
      "/wiki/Web_indexing\n",
      "/wiki/Web_mining\n",
      "/wiki/Data_mining\n",
      "/wiki/Comparison_shopping_website\n",
      "/wiki/Change_detection_and_notification\n",
      "/wiki/Web_mashup\n",
      "/wiki/Web_page\n",
      "/wiki/HTML\n",
      "/wiki/XHTML\n",
      "/wiki/End-user_(computer_science)\n",
      "/wiki/Application_Programming_Interface\n",
      "/wiki/Amazon_AWS\n",
      "/wiki/Google\n",
      "/wiki/JSON\n",
      "/wiki/Document_Object_Model\n",
      "/wiki/Computer_vision\n",
      "/wiki/Natural_language_processing\n",
      "#Techniques\n",
      "#Human_copy-and-paste\n",
      "#Text_pattern_matching\n",
      "#HTTP_programming\n",
      "#HTML_parsing\n",
      "#DOM_parsing\n",
      "#Vertical_aggregation\n",
      "#Semantic_annotation_recognizing\n",
      "#Computer_vision_web-page_analysis\n",
      "#Software\n",
      "#Example_tools\n",
      "#Javascript_tools\n",
      "#SaaS_version\n",
      "#Web_crawling_frameworks\n",
      "#Legal_issues\n",
      "#United_States\n",
      "#The_EU\n",
      "#Australia\n",
      "#Methods_to_prevent_web_scraping\n",
      "#See_also\n",
      "#References\n",
      "/w/index.php?title=Web_scraping&action=edit&section=1\n",
      "/wiki/Semantic_web\n",
      "/wiki/Human-computer_interaction\n",
      "/w/index.php?title=Web_scraping&action=edit&section=2\n",
      "/w/index.php?title=Web_scraping&action=edit&section=3\n",
      "/wiki/Grep\n",
      "/wiki/Regular_expression\n",
      "/wiki/Perl\n",
      "/wiki/Python_(programming_language)\n",
      "/w/index.php?title=Web_scraping&action=edit&section=4\n",
      "/wiki/Static_web_page\n",
      "/wiki/Dynamic_web_page\n",
      "/wiki/Socket_programming\n",
      "/w/index.php?title=Web_scraping&action=edit&section=5\n",
      "/wiki/Wrapper_(data_mining)\n",
      "#cite_note-3\n",
      "/wiki/Semi-structured_data\n",
      "/wiki/XQuery\n",
      "/w/index.php?title=Web_scraping&action=edit&section=6\n",
      "/wiki/Document_Object_Model\n",
      "/wiki/Internet_Explorer\n",
      "/wiki/Mozilla\n",
      "/w/index.php?title=Web_scraping&action=edit&section=7\n",
      "/wiki/Long_Tail\n",
      "/w/index.php?title=Web_scraping&action=edit&section=8\n",
      "/wiki/Metadata\n",
      "/wiki/Microformat\n",
      "#cite_note-4\n",
      "/w/index.php?title=Web_scraping&action=edit&section=9\n",
      "/wiki/Machine_learning\n",
      "/wiki/Computer_vision\n",
      "#cite_note-5\n",
      "/w/index.php?title=Web_scraping&action=edit&section=10\n",
      "/w/index.php?title=Web_scraping&action=edit&section=11\n",
      "/wiki/CURL\n",
      "/wiki/Data_Toolbar\n",
      "/wiki/Diffbot\n",
      "/wiki/Heritrix\n",
      "/wiki/Wayback_Machine\n",
      "/wiki/HtmlUnit\n",
      "/wiki/HTTrack\n",
      "/wiki/IMacros\n",
      "/wiki/Selenium_(software)\n",
      "/wiki/Aptana#Aptana_Jaxer\n",
      "/wiki/Nokogiri_(software)\n",
      "/wiki/OutWit_Hub\n",
      "/wiki/Watir\n",
      "/wiki/Wget\n",
      "/wiki/WSO2_Mashup_Server\n",
      "/wiki/Yahoo!_Query_Language\n",
      "/w/index.php?title=Web_scraping&action=edit&section=12\n",
      "/wiki/Greasemonkey\n",
      "/wiki/Node.js\n",
      "/wiki/PhantomJS\n",
      "/wiki/Headless_browser\n",
      "/wiki/JQuery\n",
      "/w/index.php?title=Web_scraping&action=edit&section=13\n",
      "https://www.diggernaut.com/\n",
      "/w/index.php?title=FScraper&action=edit&redlink=1\n",
      "/wiki/Import.io\n",
      "https://listly.io/\n",
      "/wiki/Mozenda\n",
      "/w/index.php?title=UScraper&action=edit&redlink=1\n",
      "/w/index.php?title=Web_scraping&action=edit&section=14\n",
      "/wiki/Scrapy\n",
      "/w/index.php?title=Web_scraping&action=edit&section=15\n",
      "/wiki/Wikipedia:WikiProject_Countering_systemic_bias\n",
      "//en.wikipedia.org/w/index.php?title=Web_scraping&action=edit\n",
      "/wiki/Talk:Web_scraping\n",
      "/wiki/Wikipedia:Article_wizard\n",
      "/wiki/Help:Maintenance_template_removal\n",
      "/wiki/Terms_of_use\n",
      "#cite_note-6\n",
      "/w/index.php?title=Web_scraping&action=edit&section=16\n",
      "/wiki/Cause_of_action\n",
      "/wiki/Computer_Fraud_and_Abuse_Act\n",
      "/wiki/Trespass_to_chattels\n",
      "#cite_note-7\n",
      "/wiki/Feist_Publications,_Inc.,_v._Rural_Telephone_Service_Co.\n",
      "/wiki/Trespass_to_chattels\n",
      "#cite_note-8\n",
      "#cite_note-9\n",
      "/wiki/EBay_v._Bidder%27s_Edge\n",
      "/wiki/Auction_sniping\n",
      "/wiki/Personal_property\n",
      "/wiki/Plaintiff\n",
      "/wiki/Defendant\n",
      "#cite_note-10\n",
      "/wiki/Screen_scraping\n",
      "/wiki/American_Airlines\n",
      "#cite_note-11\n",
      "/wiki/Injunction\n",
      "#cite_note-12\n",
      "/wiki/Southwest_Airlines\n",
      "/wiki/US_Copyright_law\n",
      "/wiki/Supreme_Court_of_the_United_States\n",
      "/wiki/Yahoo!\n",
      "#cite_note-impervawp2011-13\n",
      "/wiki/Craigslist_v._3Taps\n",
      "/wiki/Computer_Fraud_and_Abuse_Act\n",
      "#cite_note-14\n",
      "/wiki/Cvent,_Inc.\n",
      "/wiki/Eventbrite\n",
      "/wiki/Browse_wrap\n",
      "#cite_note-15\n",
      "/wiki/United_States_District_Court_for_the_Eastern_District_of_Pennsylvania\n",
      "#cite_note-16\n",
      "/wiki/QVC\n",
      "#cite_note-17\n",
      "#cite_note-18\n",
      "/wiki/Facebook,_Inc._v._Power_Ventures,_Inc.\n",
      "/wiki/Electronic_Frontier_Foundation\n",
      "#cite_note-19\n",
      "#cite_note-20\n",
      "/wiki/Associated_Press_v._Meltwater_U.S._Holdings,_Inc.\n",
      "/w/index.php?title=Web_scraping&action=edit&section=17\n",
      "#cite_note-21\n",
      "/w/index.php?title=Inchoate&action=edit&redlink=1\n",
      "#cite_note-22\n",
      "#cite_note-23\n",
      "/w/index.php?title=Web_scraping&action=edit&section=18\n",
      "/wiki/Spam_Act_2003\n",
      "#cite_note-24\n",
      "#cite_note-25\n",
      "/w/index.php?title=Web_scraping&action=edit&section=19\n",
      "/wiki/IP_address\n",
      "/wiki/Geolocation\n",
      "/wiki/DNSBL\n",
      "/wiki/Web_service\n",
      "/wiki/Application_programming_interface\n",
      "/wiki/User_agent\n",
      "/wiki/String_(computer_science)\n",
      "/wiki/Robots_exclusion_standard\n",
      "/wiki/Googlebot\n",
      "/wiki/CAPTCHA\n",
      "/wiki/Application_firewall\n",
      "#cite_note-26\n",
      "/wiki/Honeypot_(computing)\n",
      "/wiki/Obfuscation\n",
      "/wiki/CSS_sprite\n",
      "/wiki/Web_accessibility\n",
      "/wiki/Screen_reader\n",
      "https://www.promptcloud.com/blog/how-to-read-and-respect-robots-file\n",
      "/w/index.php?title=Web_scraping&action=edit&section=20\n",
      "/wiki/Archive.is\n",
      "/wiki/Comparison_of_feed_aggregators\n",
      "/wiki/Data_scraping\n",
      "/wiki/Data_wrangling\n",
      "/wiki/Importer_(computing)\n",
      "/wiki/Job_wrapping\n",
      "/wiki/Knowledge_extraction\n",
      "/wiki/OpenSocial\n",
      "/wiki/Scraper_site\n",
      "/wiki/Fake_news_website\n",
      "/wiki/Blog_scraping\n",
      "/wiki/Spamdexing\n",
      "/wiki/Domain_name_drop_list\n",
      "/wiki/Text_corpus\n",
      "/wiki/Web_archiving\n",
      "/wiki/Blog_network\n",
      "/wiki/Search_Engine_Scraping\n",
      "/wiki/Category:Web_crawlers\n",
      "/w/index.php?title=Web_scraping&action=edit&section=21\n",
      "#cite_ref-Boeing2016JPER_1-0\n",
      "#cite_ref-Boeing2016JPER_1-1\n",
      "/wiki/Digital_object_identifier\n",
      "//doi.org/10.1177%2F0739456X16664789\n",
      "#cite_ref-2\n",
      "/wiki/Digital_object_identifier\n",
      "//doi.org/10.5430%2Fair.v2n1p44\n",
      "#cite_ref-3\n",
      "https://pdfs.semanticscholar.org/4fb4/3c5a212df751e84c3b2f8d29fabfe56c3616.pdf\n",
      "#cite_ref-4\n",
      "http://www.gooseeker.com/en/node/knowledgebase/freeformat\n",
      "#cite_ref-5\n",
      "http://www.xconomy.com/san-francisco/2012/07/25/diffbot-is-using-computer-vision-to-reinvent-the-semantic-web/\n",
      "#cite_ref-6\n",
      "https://web.archive.org/web/20020308222536/http://www.chillingeffects.org/linking/faq.cgi#QID596\n",
      "http://www.chillingeffects.org/linking/faq.cgi#QID596\n",
      "#cite_ref-7\n",
      "http://scholarship.law.berkeley.edu/btlj/vol29/iss4/16/\n",
      "/wiki/Digital_object_identifier\n",
      "//doi.org/10.15779%2FZ38B39B\n",
      "/wiki/International_Standard_Serial_Number\n",
      "//www.worldcat.org/issn/1086-3818\n",
      "#cite_ref-8\n",
      "http://www.tomwbell.com/NetLaw/Ch06.html\n",
      "#cite_ref-9\n",
      "https://web.archive.org/web/20020308222536/http://www.chillingeffects.org/linking/faq.cgi#QID460\n",
      "http://www.chillingeffects.org/linking/faq.cgi#QID460\n",
      "#cite_ref-10\n",
      "http://www.tomwbell.com/NetLaw/Ch07/Ticketmaster.html\n",
      "#cite_ref-11\n",
      "https://web.archive.org/web/20110723131832/http://www.fornova.net/documents/AAFareChase.pdf\n",
      "http://www.fornova.net/documents/AAFareChase.pdf\n",
      "#cite_ref-12\n",
      "http://www.thefreelibrary.com/American+Airlines,+FareChase+Settle+Suit.-a0103213546\n",
      "#cite_ref-impervawp2011_13-0\n",
      "http://www.imperva.com/docs/WP_Detecting_and_Blocking_Site_Scraping_Attacks.pdf\n",
      "#cite_ref-14\n",
      "http://library.findlaw.com/2003/Jul/29/132944.html\n",
      "#cite_ref-15\n",
      "http://www.fornova.net/documents/Cvent.pdf\n",
      "#cite_ref-16\n",
      "https://www.scribd.com/doc/249068700/LinkedIn-v-Resultly-LLC-Complaint?secret_password=pEVKDbnvhQL52oKfdrmT\n",
      "#cite_ref-17\n",
      "http://newmedialaw.proskauer.com/2014/12/05/qvc-sues-shopping-app-for-web-scraping-that-allegedly-triggered-site-outage/\n",
      "#cite_ref-18\n",
      "http://www.fornova.net/documents/pblog-bna-com.pdf\n",
      "#cite_ref-19\n",
      "https://www.techdirt.com/articles/20090605/2228205147.shtml\n",
      "#cite_ref-20\n",
      "https://www.eff.org/cases/facebook-v-power-ventures\n",
      "#cite_ref-21\n",
      "https://web.archive.org/web/20071012005033/http://www.bvhd.dk/uploads/tx_mocarticles/S_-_og_Handelsrettens_afg_relse_i_Ofir-sagen.pdf\n",
      "http://www.bvhd.dk/uploads/tx_mocarticles/S_-_og_Handelsrettens_afg_relse_i_Ofir-sagen.pdf\n",
      "#cite_ref-22\n",
      "http://www.bailii.org/ie/cases/IEHC/2010/H47.html\n",
      "#cite_ref-23\n",
      "http://www.lkshields.ie/htmdocs/publications/newsletters/update26/update26_03.htm\n",
      "#cite_ref-24\n",
      "https://www.lloyds.com/~/media/5880dae185914b2487bed7bd63b96286.ashx\n",
      "#cite_ref-25\n",
      "http://www.webstartdesign.com.au/spam_business_practical_guide.pdf\n",
      "#cite_ref-26\n",
      "https://s3.us-west-2.amazonaws.com/research-papers-mynk/Breaking-Fraud-And-Bot-Detection-Solutions.pdf\n",
      "https://en.wikipedia.org/w/index.php?title=Web_scraping&oldid=830441507\n",
      "/wiki/Help:Category\n",
      "/wiki/Category:Web_scraping\n",
      "/wiki/Category:World_Wide_Web\n",
      "/wiki/Category:Spamming\n",
      "/wiki/Category:CS1_Danish-language_sources_(da)\n",
      "/wiki/Category:Articles_needing_additional_references_from_June_2017\n",
      "/wiki/Category:All_articles_needing_additional_references\n",
      "/wiki/Category:Articles_with_limited_geographic_scope_from_October_2015\n",
      "/wiki/Category:USA-centric\n",
      "/wiki/Category:Pages_using_div_col_with_deprecated_parameters\n",
      "/wiki/Special:MyTalk\n",
      "/wiki/Special:MyContributions\n",
      "/w/index.php?title=Special:CreateAccount&returnto=Web+scraping\n",
      "/w/index.php?title=Special:UserLogin&returnto=Web+scraping\n",
      "/wiki/Web_scraping\n",
      "/wiki/Talk:Web_scraping\n",
      "/wiki/Web_scraping\n",
      "/w/index.php?title=Web_scraping&action=edit\n",
      "/w/index.php?title=Web_scraping&action=history\n",
      "/wiki/Main_Page\n",
      "/wiki/Main_Page\n",
      "/wiki/Portal:Contents\n",
      "/wiki/Portal:Featured_content\n",
      "/wiki/Portal:Current_events\n",
      "/wiki/Special:Random\n",
      "https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en\n",
      "//shop.wikimedia.org\n",
      "/wiki/Help:Contents\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:Community_portal\n",
      "/wiki/Special:RecentChanges\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "/wiki/Special:WhatLinksHere/Web_scraping\n",
      "/wiki/Special:RecentChangesLinked/Web_scraping\n",
      "/wiki/Wikipedia:File_Upload_Wizard\n",
      "/wiki/Special:SpecialPages\n",
      "/w/index.php?title=Web_scraping&oldid=830441507\n",
      "/w/index.php?title=Web_scraping&action=info\n",
      "https://www.wikidata.org/wiki/Special:EntityPage/Q665452\n",
      "/w/index.php?title=Special:CiteThisPage&page=Web_scraping&id=830441507\n",
      "/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Web+scraping\n",
      "/w/index.php?title=Special:ElectronPdf&page=Web+scraping&action=show-download-screen\n",
      "/w/index.php?title=Web_scraping&printable=yes\n",
      "https://ar.wikipedia.org/wiki/%D8%A5%D8%B3%D8%AA%D8%AE%D9%84%D8%A7%D8%B5_%D8%A7%D9%84%D9%85%D9%88%D8%A7%D9%82%D8%B9\n",
      "https://ca.wikipedia.org/wiki/Web_scraping\n",
      "https://de.wikipedia.org/wiki/Screen_Scraping\n",
      "https://es.wikipedia.org/wiki/Web_scraping\n",
      "https://eu.wikipedia.org/wiki/Web_scraping\n",
      "https://fr.wikipedia.org/wiki/Web_scraping\n",
      "https://is.wikipedia.org/wiki/Vefs%C3%B6fnun\n",
      "https://it.wikipedia.org/wiki/Web_scraping\n",
      "https://lv.wikipedia.org/wiki/Rasmo%C5%A1ana\n",
      "https://nl.wikipedia.org/wiki/Scrapen\n",
      "https://ja.wikipedia.org/wiki/%E3%82%A6%E3%82%A7%E3%83%96%E3%82%B9%E3%82%AF%E3%83%AC%E3%82%A4%E3%83%94%E3%83%B3%E3%82%B0\n",
      "https://sr.wikipedia.org/wiki/Web_scraping\n",
      "https://tr.wikipedia.org/wiki/Web_kaz%C4%B1ma\n",
      "https://uk.wikipedia.org/wiki/Web_scraping\n",
      "https://zh.wikipedia.org/wiki/%E7%BD%91%E9%A1%B5%E6%8A%93%E5%8F%96\n",
      "https://www.wikidata.org/wiki/Special:EntityPage/Q665452#sitelinks-wikipedia\n",
      "//en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License\n",
      "//creativecommons.org/licenses/by-sa/3.0/\n",
      "//wikimediafoundation.org/wiki/Terms_of_Use\n",
      "//wikimediafoundation.org/wiki/Privacy_policy\n",
      "//www.wikimediafoundation.org/\n",
      "https://wikimediafoundation.org/wiki/Privacy_policy\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:General_disclaimer\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute\n",
      "https://wikimediafoundation.org/wiki/Cookie_statement\n",
      "//en.m.wikipedia.org/w/index.php?title=Web_scraping&mobileaction=toggle_view_mobile\n",
      "https://wikimediafoundation.org/\n",
      "//www.mediawiki.org/\n"
     ]
    }
   ],
   "source": [
    "for link in soup.find_all('a'):\n",
    "    print link.get('href')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a class=\"external text\" href=\"https://www.diggernaut.com/\" rel=\"nofollow\">diggernaut.com</a>\n",
      "<a class=\"external text\" href=\"https://listly.io/\" rel=\"nofollow\">Listly.io</a>\n",
      "<a class=\"external text\" href=\"https://www.promptcloud.com/blog/how-to-read-and-respect-robots-file\" rel=\"nofollow\">robots.txt</a>\n",
      "<a class=\"external text\" href=\"https://pdfs.semanticscholar.org/4fb4/3c5a212df751e84c3b2f8d29fabfe56c3616.pdf\" rel=\"nofollow\">\"Joint Optimization of Wrapper Generation and Template Detection\"</a>\n",
      "<a class=\"external text\" href=\"http://www.gooseeker.com/en/node/knowledgebase/freeformat\" rel=\"nofollow\">Semantic annotation based web scraping</a>\n",
      "<a class=\"external text\" href=\"http://www.xconomy.com/san-francisco/2012/07/25/diffbot-is-using-computer-vision-to-reinvent-the-semantic-web/\" rel=\"nofollow\">\"Diffbot Is Using Computer Vision to Reinvent the Semantic Web\"</a>\n",
      "<a class=\"external text\" href=\"https://web.archive.org/web/20020308222536/http://www.chillingeffects.org/linking/faq.cgi#QID596\" rel=\"nofollow\">\"FAQ about linking – Are website terms of use binding contracts?\"</a>\n",
      "<a class=\"external text\" href=\"http://www.chillingeffects.org/linking/faq.cgi#QID596\" rel=\"nofollow\">the original</a>\n",
      "<a class=\"external text\" href=\"http://scholarship.law.berkeley.edu/btlj/vol29/iss4/16/\" rel=\"nofollow\">\"Symbiotic Relationships: Pragmatic Acceptance of Data Scraping\"</a>\n",
      "<a class=\"external text\" href=\"http://www.tomwbell.com/NetLaw/Ch06.html\" rel=\"nofollow\">\"Internet Law, Ch. 06: Trespass to Chattels\"</a>\n",
      "<a class=\"external text\" href=\"https://web.archive.org/web/20020308222536/http://www.chillingeffects.org/linking/faq.cgi#QID460\" rel=\"nofollow\">\"What are the \"trespass to chattels\" claims some companies or website owners have brought?\"</a>\n",
      "<a class=\"external text\" href=\"http://www.chillingeffects.org/linking/faq.cgi#QID460\" rel=\"nofollow\">the original</a>\n",
      "<a class=\"external text\" href=\"http://www.tomwbell.com/NetLaw/Ch07/Ticketmaster.html\" rel=\"nofollow\">\"Ticketmaster Corp. v. Tickets.com, Inc.\"</a>\n",
      "<a class=\"external text\" href=\"https://web.archive.org/web/20110723131832/http://www.fornova.net/documents/AAFareChase.pdf\" rel=\"nofollow\">\"American Airlines v. FareChase\"</a>\n",
      "<a class=\"external text\" href=\"http://www.fornova.net/documents/AAFareChase.pdf\" rel=\"nofollow\">the original</a>\n",
      "<a class=\"external text\" href=\"http://www.thefreelibrary.com/American+Airlines,+FareChase+Settle+Suit.-a0103213546\" rel=\"nofollow\">\"American Airlines, FareChase Settle Suit\"</a>\n",
      "<a class=\"external text\" href=\"http://www.imperva.com/docs/WP_Detecting_and_Blocking_Site_Scraping_Attacks.pdf\" rel=\"nofollow\">Detecting and Blocking Site Scraping Attacks</a>\n",
      "<a class=\"external text\" href=\"http://library.findlaw.com/2003/Jul/29/132944.html\" rel=\"nofollow\">\"Controversy Surrounds 'Screen Scrapers': Software Helps Users Access Web Sites But Activity by Competitors Comes Under Scrutiny\"</a>\n",
      "<a class=\"external text\" href=\"http://www.fornova.net/documents/Cvent.pdf\" rel=\"nofollow\">\"QVC Inc. v. Resultly LLC, No. 14-06714 (E.D. Pa. filed Nov. 24, 2014)\"</a>\n",
      "<a class=\"external text\" href=\"https://www.scribd.com/doc/249068700/LinkedIn-v-Resultly-LLC-Complaint?secret_password=pEVKDbnvhQL52oKfdrmT\" rel=\"nofollow\">\"QVC Inc. v. Resultly LLC, No. 14-06714 (E.D. Pa. filed Nov. 24, 2014)\"</a>\n",
      "<a class=\"external text\" href=\"http://newmedialaw.proskauer.com/2014/12/05/qvc-sues-shopping-app-for-web-scraping-that-allegedly-triggered-site-outage/\" rel=\"nofollow\">\"QVC Sues Shopping App for Web Scraping That Allegedly Triggered Site Outage\"</a>\n",
      "<a class=\"external text\" href=\"http://www.fornova.net/documents/pblog-bna-com.pdf\" rel=\"nofollow\">\"Did Iqbal/Twombly Raise the Bar for Browsewrap Claims?\"</a>\n",
      "<a class=\"external text\" href=\"https://www.techdirt.com/articles/20090605/2228205147.shtml\" rel=\"nofollow\">\"Can Scraping Non-Infringing Content Become Copyright Infringement... Because Of How Scrapers Work? | Techdirt\"</a>\n",
      "<a class=\"external text\" href=\"https://www.eff.org/cases/facebook-v-power-ventures\" rel=\"nofollow\">\"Facebook v. Power Ventures\"</a>\n",
      "<a class=\"external text\" href=\"https://web.archive.org/web/20071012005033/http://www.bvhd.dk/uploads/tx_mocarticles/S_-_og_Handelsrettens_afg_relse_i_Ofir-sagen.pdf\" rel=\"nofollow\">\"UDSKRIFT AF SØ- &amp; HANDELSRETTENS DOMBOG\"</a>\n",
      "<a class=\"external text\" href=\"http://www.bvhd.dk/uploads/tx_mocarticles/S_-_og_Handelsrettens_afg_relse_i_Ofir-sagen.pdf\" rel=\"nofollow\">the original</a>\n",
      "<a class=\"external text\" href=\"http://www.bailii.org/ie/cases/IEHC/2010/H47.html\" rel=\"nofollow\">\"High Court of Ireland Decisions &gt;&gt; Ryanair Ltd -v- Billigfluege.de GMBH 2010 IEHC 47 (26 February 2010)\"</a>\n",
      "<a class=\"external text\" href=\"http://www.lkshields.ie/htmdocs/publications/newsletters/update26/update26_03.htm\" rel=\"nofollow\">\"Intellectual Property: Website Terms of Use\"</a>\n",
      "<a class=\"external text\" href=\"https://www.lloyds.com/~/media/5880dae185914b2487bed7bd63b96286.ashx\" rel=\"nofollow\">\"Spam Act 2003: An overview for business\"</a>\n",
      "<a class=\"external text\" href=\"http://www.webstartdesign.com.au/spam_business_practical_guide.pdf\" rel=\"nofollow\">\"Spam Act 2003: A practical guide for business\"</a>\n",
      "<a class=\"external text\" href=\"https://s3.us-west-2.amazonaws.com/research-papers-mynk/Breaking-Fraud-And-Bot-Detection-Solutions.pdf\" rel=\"nofollow\">Breaking Fraud &amp; Bot Detection Solutions</a>\n",
      "<a dir=\"ltr\" href=\"https://en.wikipedia.org/w/index.php?title=Web_scraping&amp;oldid=830441507\">https://en.wikipedia.org/w/index.php?title=Web_scraping&amp;oldid=830441507</a>\n",
      "<a href=\"https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&amp;utm_medium=sidebar&amp;utm_campaign=C13_en.wikipedia.org&amp;uselang=en\" title=\"Support us\">Donate to Wikipedia</a>\n",
      "<a accesskey=\"g\" href=\"https://www.wikidata.org/wiki/Special:EntityPage/Q665452\" title=\"Link to connected data repository item [g]\">Wikidata item</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://ar.wikipedia.org/wiki/%D8%A5%D8%B3%D8%AA%D8%AE%D9%84%D8%A7%D8%B5_%D8%A7%D9%84%D9%85%D9%88%D8%A7%D9%82%D8%B9\" hreflang=\"ar\" lang=\"ar\" title=\"إستخلاص المواقع – Arabic\">العربية</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://ca.wikipedia.org/wiki/Web_scraping\" hreflang=\"ca\" lang=\"ca\" title=\"Web scraping – Catalan\">Català</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://de.wikipedia.org/wiki/Screen_Scraping\" hreflang=\"de\" lang=\"de\" title=\"Screen Scraping – German\">Deutsch</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://es.wikipedia.org/wiki/Web_scraping\" hreflang=\"es\" lang=\"es\" title=\"Web scraping – Spanish\">Español</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://eu.wikipedia.org/wiki/Web_scraping\" hreflang=\"eu\" lang=\"eu\" title=\"Web scraping – Basque\">Euskara</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://fr.wikipedia.org/wiki/Web_scraping\" hreflang=\"fr\" lang=\"fr\" title=\"Web scraping – French\">Français</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://is.wikipedia.org/wiki/Vefs%C3%B6fnun\" hreflang=\"is\" lang=\"is\" title=\"Vefsöfnun – Icelandic\">Íslenska</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://it.wikipedia.org/wiki/Web_scraping\" hreflang=\"it\" lang=\"it\" title=\"Web scraping – Italian\">Italiano</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://lv.wikipedia.org/wiki/Rasmo%C5%A1ana\" hreflang=\"lv\" lang=\"lv\" title=\"Rasmošana – Latvian\">Latviešu</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://nl.wikipedia.org/wiki/Scrapen\" hreflang=\"nl\" lang=\"nl\" title=\"Scrapen – Dutch\">Nederlands</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://ja.wikipedia.org/wiki/%E3%82%A6%E3%82%A7%E3%83%96%E3%82%B9%E3%82%AF%E3%83%AC%E3%82%A4%E3%83%94%E3%83%B3%E3%82%B0\" hreflang=\"ja\" lang=\"ja\" title=\"ウェブスクレイピング – Japanese\">日本語</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://sr.wikipedia.org/wiki/Web_scraping\" hreflang=\"sr\" lang=\"sr\" title=\"Web scraping – Serbian\">Српски / srpski</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://tr.wikipedia.org/wiki/Web_kaz%C4%B1ma\" hreflang=\"tr\" lang=\"tr\" title=\"Web kazıma – Turkish\">Türkçe</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://uk.wikipedia.org/wiki/Web_scraping\" hreflang=\"uk\" lang=\"uk\" title=\"Web scraping – Ukrainian\">Українська</a>\n",
      "<a class=\"interlanguage-link-target\" href=\"https://zh.wikipedia.org/wiki/%E7%BD%91%E9%A1%B5%E6%8A%93%E5%8F%96\" hreflang=\"zh\" lang=\"zh\" title=\"网页抓取 – Chinese\">中文</a>\n",
      "<a class=\"wbc-editpage\" href=\"https://www.wikidata.org/wiki/Special:EntityPage/Q665452#sitelinks-wikipedia\" title=\"Edit interlanguage links\">Edit links</a>\n",
      "<a class=\"extiw\" href=\"https://wikimediafoundation.org/wiki/Privacy_policy\" title=\"wmf:Privacy policy\">Privacy policy</a>\n",
      "<a href=\"https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute\">Developers</a>\n",
      "<a href=\"https://wikimediafoundation.org/wiki/Cookie_statement\">Cookie statement</a>\n",
      "<a href=\"https://wikimediafoundation.org/\"><img alt=\"Wikimedia Foundation\" height=\"31\" src=\"/static/images/wikimedia-button.png\" srcset=\"/static/images/wikimedia-button-1.5x.png 1.5x, /static/images/wikimedia-button-2x.png 2x\" width=\"88\"/></a>\n"
     ]
    }
   ],
   "source": [
    "for link in soup.find_all('a', attrs = {'href': re.compile(\"http\")}):\n",
    "    print link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Saving in a file\n",
    "\n",
    "file = open('ParsedText.txt','wb' )\n",
    "for link in soup.find_all('a', attrs = {'href': re.compile(\"http\")}):\n",
    "    ## First convert to string\n",
    "    link = str(link)\n",
    "    file.write(link)\n",
    "file.flush()\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/Users/Adhira/Untitled Folder 2'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}