Chapter10-CrawlingThroughFormsAndLogins.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello there, Ryan Mitchell!\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "params = {'firstname': 'Ryan', 'lastname': 'Mitchell'}\n",
    "r = requests.post(\"http://pythonscraping.com/pages/processing.php\", data=params)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "<!doctype html>\n",
      "<html>\n",
      "<head>\n",
      "  <meta charset=\"UTF-8\">\n",
      "\t<title>O&rsquo;Reilly Media - Email Subscription Center</title>\n",
      "\n",
      "  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n",
      "  <meta name=\"apple-mobile-web-app-title\" content=\"O'Reilly Email Center\">\n",
      "\n",
      "  <!--[if lt IE 9]>\n",
      "  <script>\n",
      "    'article aside footer figure header nav section address time'.replace(/\\w+/g,function(n){document.createElement(n)})\n",
      "  </script>\n",
      "\t<![endif]-->\n",
      "\n",
      "\t<script type=\"text/javascript\" src=\"//fonts.oreillystatic.com/bmj2nsh.js\"></script>\n",
      "  <script type=\"text/javascript\">try{Typekit.load();}catch(e){}</script>\n",
      "\n",
      "  <link type=\"text/css\" rel=\"stylesheet\" href=\"http://img.ipost.com/client/o/oreilly/css/styles.css\" />\n",
      "<style type=\"text/css\">\n",
      "  .error { color:red; font-weight:bold; }\n",
      "  form #submit-row { min-height:34px; }\n",
      "  form #submit-row-fixed { display:none; position:fixed; left:0; bottom:0; width:100%; max-width:100%; margin-bottom:0; overflow:visible; }\n",
      "  form #submit-row-fixed .max-width { background-color:rgb(34,34,34); background-color:rgba(34,34,34,.95); padding:10px 2.0833%; overflow:hidden; -webkit-box-shadow:0px 0px 5px 0px rgba(0, 0, 0, 0.5); -moz-box-shadow:0px 0px 5px 0px rgba(0, 0, 0, 0.5); box-shadow:0px 0px 5px 0px rgba(0, 0, 0, 0.5); }\n",
      "  form #submit-row-fixed button { vertical-align:top; margin-right:20px; margin-left:-4px; }\n",
      "  form #submit-row-fixed p { display:inline-block; margin:6px 0 0 0; color:rgb(255,255,255); vertical-align:top; }\n",
      "\n",
      "  @media screen and (max-width:958px) {\n",
      "    form #submit-row-fixed button { margin-left:0; }\n",
      "  }\n",
      "</style>\n",
      "<script language=\"JavaScript\" type=\"text/javascript\"><!--\n",
      "var s_account=\"ororeilly,orglobal\"\n",
      "//--></script>\n",
      "<!-- SiteCatalyst code version: H.20.2. Copyright 1997-2009 Omniture, Inc. More info available at http://www.omniture.com  -->\n",
      "<script language=\"JavaScript\" type=\"text/javascript\" src=\"http://cdn.oreillystatic.com/assets/js/s_code.js\"></script>\n",
      "<script language=\"JavaScript\" type=\"text/javascript\"><!--\n",
      "s.pageName=\"oreilly:email signup:email complete-update profile\"\n",
      "s.channel=\"oreilly\"\n",
      "s.prop1=\"oreilly:email signup\"\n",
      "s.prop2=\"oreilly:email signup:email signup\"\n",
      "s.prop3=\"oreilly:email signup:email signup:email signup\"\n",
      "s.prop5=\"email complete-update profile\"\n",
      "s.prop24=\"08/01/2014\"\n",
      "s.events=\"event2,event9\"\n",
      "/************* DO NOT ALTER ANYTHING BELOW THIS LINE ! **************/\n",
      "var s_code=s.t();if(s_code)document.write(s_code)//--></script>\n",
      "<!-- End SiteCatalyst code version: H.20.2. -->\n",
      "\n",
      "</head>\n",
      "\n",
      "<body>\n",
      "\t<header class=\"max-width\">\n",
      "\t\t<a class=\"logo\" href=\"http://oreilly.com\" title=\"O'Reilly\"></a>\n",
      "\t</header>\n",
      "\n",
      "\t<article class=\"page max-width\" id=\"subscription\">\n",
      "      <form id=\"email-subscription\" action=\"/form/oreilly/signup4/9z1zkgbefi3bkmpu0h3tijdk5dptvk3kjv251gd32g8\" method=\"post\">\n",
      "  \t\t<section>\n",
      "  \t\t\t<h1>Your Email Subscription</h1>\n",
      "\n",
      "        <aside class=\"legend\">\n",
      "          <ul>\n",
      "            <li class=\"option unspecified\">Preference unspecified</li>\n",
      "            <li class=\"option yes\">Email from this category</li>\n",
      "            <li class=\"option no\">No email from this category</li>\n",
      "          </ul>\n",
      "        </aside>\n",
      "\n",
      "  \t\t\t<p class=\"intro\">We want to stay in touch, but only in ways that you find helpful. Select your preferences by using the form below and saving your changes.</p>\n",
      "\n",
      "<h2 class=\"highlight profile\">You&rsquo;re Almost Finished</h2>\n",
      "\n",
      "        <p class=\"intro\">One more click and you&rsquo;ll be all set. To confirm your subscription, please add to your profile below, or simply confirm by clicking this button:</p>\n",
      "\n",
      "        <button type=\"submit\" name=\"submit\" class=\"button\">Yes, subscribe me to the oreilly.com newsletter</button>\n",
      "      </section>\n",
      "      \n",
      "      <section>\n",
      "\n",
      "        <h2 class=\"highlight profile\">Complete Your Profile</h2>\n",
      "<div class=\"row\">\n",
      "            <label for=\"email_addr\">Email</label><br>\n",
      "\t<input type=\"hidden\" value=\"ryan.e.mitchell@gmail.com\" name=\"email_addr\">\n",
      "\tryan.e.mitchell@gmail.com &nbsp; (<a href=\"/form/oreilly/signup3\" style=\"text-decoration:none;\">Not ryan.e.mitchell@gmail.com?</a>)\n",
      "\t </div>\n",
      "        <div class=\"row\">\n",
      "          <div class=\"inline\">\n",
      "            <label for=\"first_name\">First Name</label>\n",
      "<input type=\"text\" name=\"first_name\" class=\"field\" value=\"\">\n",
      "          </div>\n",
      "\n",
      "          <div class=\"inline\">\n",
      "            <label for=\"last_name\">Last Name</label>\n",
      "<input type=\"text\" name=\"last_name\" class=\"field\" value=\"\">\n",
      "          </div>\n",
      "        </div>\n",
      "  \t\t</section>\n",
      "\n",
      "  \t\t<section class=\"selection\" style=\"padding-bottom:30px;\">\n",
      "        <h2 class=\"highlight\">Choose Your Email Preferences</h2>\n",
      "\n",
      "        <div class=\"col x2\">\n",
      "    \t\t\t<h3>Training</h3>\n",
      "          <div class=\"col\">\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_video\">New video training alerts</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_training\">New live training alerts</a>\n",
      "          </div>\n",
      "\n",
      "    \t\t\t<h3>Webcasts</h3>\n",
      "          <div class=\"col\">\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_webcasts\">Weekly notifications about upcoming, live webcasts</a>\n",
      "          </div>\n",
      "\n",
      "    \t\t\t<h3>Product and Promotion Alerts</h3>\n",
      "          <div class=\"col x2\">\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_business\">Business &amp; Leadership</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_data\">Data</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_design\">Design</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_photo\">Digital Photography</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_diy\">DIY</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_games\">Game Design &amp; Development</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_iot\">IoT</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_linux_unix\">Linux &amp; Unix</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_math_science\">Math and Science</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_mobile\">Mobile Design &amp; Development</a>\n",
      "          </div>\n",
      "\n",
      "          <div class=\"col x2\">\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_open_source\">Open Source Programming</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_perf\">Performance</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_security\">Security &amp; Cryptography</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_software_eng\">Software Engineering</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_tech_culture\">Tech Culture</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_user_guides\">User Guides</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_webdev\">Web Development</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_webops\">WebOps</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_winprog\">Windows Programming</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"topic_winadmin\">Windows Admin</a>\n",
      "          </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"col x2\">\n",
      "    \t\t\t<h3>Newsletters</h3>\n",
      "          <div class=\"col\">\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_ai\">Artifical Intelligence</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_data\">Data</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_design\">Design</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_money\">FinTech</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_4sl\">Four short links</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_economy\">Next:Economy</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_prog\">Programming</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_security\">Security</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_webops_perf\">Systems Engineering and Operations</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"nl_web\">Web</a>\n",
      " \t  </div>\n",
      "\n",
      "\n",
      "          <h3>Conference Bulletins</h3>\n",
      "          <div class=\"col\">\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_fluent\">Fluent, the O'Reilly Web Platform Conference</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_jupyter\">JupyterCon</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_oscon\">OSCON, the O'Reilly Open Source Convention</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_ai\">O'Reilly Artificial Intelligence Conference</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_design\">O'Reilly Design Conference</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_economy\">O'Reilly Next:Economy Summit</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_security\">O'Reilly Security Conference</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_software_architecture\">O'Reilly Software Architecture Conference</a> \n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_strata\">Strata Data Conference</a>\n",
      "            <a href=\"#\" class=\"option unspecified\" data-field=\"con_velocity\">Velocity, the O'Reilly Systems Engineering Conference</a>\n",
      "          </div>\n",
      "\n",
      "        </div>\n",
      "\n",
      " <input type=\"hidden\" name=\"subscribe\" value=\"optin\">\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_training\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_training\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_video\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_video\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_webcasts\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_webcasts\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_business\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_business\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_data\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_data\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_design\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_design\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_photo\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_photo\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_diy\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_diy\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_games\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_games\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_iot\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_iot\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_linux_unix\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_linux_unix\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_math_science\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_math_science\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_mobile\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_mobile\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_open_source\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_open_source\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_perf\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_perf\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_security\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_security\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_software_eng\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_software_eng\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_tech_culture\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_tech_culture\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_user_guides\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_user_guides\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_webdev\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_webdev\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_webops\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_webops\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_winprog\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_winprog\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"topic_winadmin\" class=\"form-option\"><input type=\"hidden\" name=\"pref_topic_winadmin\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_prog\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_prog\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_security\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_security\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_ai\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_ai\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_data\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_data\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_web\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_web\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_webops_perf\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_webops_perf\" value=\"1\" />\n",
      "\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_economy\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_economy\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_design\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_design\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_money\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_money\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"nl_4sl\" class=\"form-option\"><input type=\"hidden\" name=\"pref_nl_4sl\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_fluent\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_fluent\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_jupyter\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_jupyter\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_oscon\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_oscon\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_ai\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_ai\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_design\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_design\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_economy\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_economy\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_security\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_security\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_software_architecture\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_software_architecture\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_strata\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_strata\" value=\"1\" />\n",
      "\n",
      "        <input type=\"hidden\" value=\"\" name=\"con_velocity\" class=\"form-option\"><input type=\"hidden\" name=\"pref_con_velocity\" value=\"1\" />\n",
      "\n",
      "        <div class=\"row\" id=\"submit-row-fixed\">\n",
      "          <div class=\"max-width\">\n",
      "            <p>Scroll to manage all subscription options.</p>\n",
      "          </div>\n",
      "        </div>\n",
      "        <div class=\"row\" id=\"submit-row-disabled\">\n",
      "          <button type=\"submit\" name=\"submit\" class=\"button\">Save Subscription</button>\n",
      "        </div>\n",
      "\n",
      "          <p class=\"privacy\">We take <a href=\"http://oreilly.com/oreilly/privacy.html\">your privacy seriously</a> and will not sell or distribute your email.</p>\n",
      "        </div>\n",
      "\t\t  </section>\n",
      "    \n",
      "    </form>\n",
      "\t</article>\n",
      "\n",
      "\t<footer class=\"max-width\">\n",
      "\t\t<div class=\"col\">&copy;2018, O'Reilly Media, Inc.</div>\n",
      "\t\t<address>\n",
      "\t\t\t<a class=\"col\" href=\"tel:+17078277019\">(707) 827-7019</a>\n",
      "\t\t\t<a class=\"col\" href=\"tel:+18008898969\">(800) 889-8969</a>\n",
      "\t\t</address>\n",
      "\t</footer>\n",
      "\n",
      "  <script src=\"//ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js\"></script>\n",
      "\n",
      "  <script type=\"text/javascript\">\n",
      "  <!--\n",
      " jQuery(document).ready(function($) {\n",
      "    var options = $('.selection').find('.option');\n",
      "    var formOptions = $('#email-subscription').find('.form-option');\n",
      "    \n",
      "    //Populate selection list with values from hidden input fields\n",
      "    formOptions.each(function() {\n",
      "      var that = $(this);\n",
      "      if (that.val() != '0') {\n",
      "        var option = options.filter(\"[data-field|='\" + that.attr('name') + \"']\");\n",
      "\n",
      "        if (that.val() == '1') {\n",
      "          option.removeClass('unspecified').addClass('yes');\n",
      "        }\n",
      "        else if (that.val() == '-1') {\n",
      "          option.removeClass('unspecified').addClass('no');\n",
      "        }\n",
      "      }\n",
      "    });\n",
      "\n",
      "    options.on('click', function(e) {\n",
      "      e.preventDefault();\n",
      "      var that = $(this);\n",
      "      var field = formOptions.filter(\"[name|='\" + that.attr('data-field') + \"']\");\n",
      "\n",
      "      if (that.hasClass('unspecified')) {\n",
      "        that.removeClass('unspecified').addClass('yes');\n",
      "        field.val('1');\n",
      "      }\n",
      "      else if (that.hasClass('yes')) {\n",
      "        that.removeClass('yes').addClass('no');\n",
      "        field.val('-1');\n",
      "      }\n",
      "      else if (that.hasClass('no')) {\n",
      "        that.removeClass('no').addClass('unspecified');\n",
      "        field.val('0');\n",
      "      }\n",
      "    });\n",
      "\n",
      "    //Show save button at bottom of screen\n",
      "    var submitStatic = $('#submit-row'),\n",
      "      submitFixed = $('#submit-row-fixed');\n",
      "      buttonFixed = false;\n",
      "\n",
      "    if ($(window).height() + $(window).scrollTop() - 43 < submitStatic.offset().top) {\n",
      "      submitStatic.children('.button').prependTo(submitFixed.children('.max-width'));\n",
      "      submitFixed.css({'display':'block', 'opacity':'0'}).stop().animate({'opacity':'1'},300);\n",
      "      buttonFixed = true;\n",
      "    }\n",
      "\n",
      "    $(window).on('scroll', function() {\n",
      "      if (buttonFixed) {\n",
      "        if ($(window).height() + $(window).scrollTop() - 43 >= submitStatic.offset().top) {\n",
      "          buttonFixed = false;\n",
      "          submitFixed.stop().animate({'opacity':'0'},200, function() {\n",
      "            submitFixed.css('display','none');\n",
      "          }).find('.button').appendTo(submitStatic);\n",
      "        }\n",
      "      }\n",
      "      else {\n",
      "        if ($(window).height() + $(window).scrollTop() - 43 < submitStatic.offset().top) {\n",
      "          submitStatic.children('.button').prependTo(submitFixed.children('.max-width'));\n",
      "          submitFixed.css({'display':'block', 'opacity':'0'}).stop().animate({'opacity':'1'},300);\n",
      "          buttonFixed = true;\n",
      "        }\n",
      "      }\n",
      "    });\n",
      "  });\n",
      "  -->\n",
      "  </script>\n",
      "\n",
      "\n",
      "</body>\n",
      "</html>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "params = {'email_addr': 'ryan.e.mitchell@gmail.com'}\n",
    "r = requests.post(\"http://post.oreilly.com/client/o/oreilly/forms/quicksignup.cgi\",\n",
    "                   data=params)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'files/Python-logo.png'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-d5ec9af2a4b5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'uploadFile'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'files/Python-logo.png'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://pythonscraping.com/pages/processing2.php'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'files/Python-logo.png'"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "files = {'uploadFile': open('files/Python-logo.png', 'rb')}\n",
    "r = requests.post('http://pythonscraping.com/pages/processing2.php', files=files)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cookie is set to:\n",
      "{'loggedin': '1', 'username': 'Ryan'}\n",
      "-----------\n",
      "Going to profile page...\n",
      "Hey Ryan! Looks like you're still logged into the site!\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "params = {'username': 'Ryan', 'password': 'password'}\n",
    "r = requests.post('http://pythonscraping.com/pages/cookies/welcome.php', params)\n",
    "print('Cookie is set to:')\n",
    "print(r.cookies.get_dict())\n",
    "print('Going to profile page...')\n",
    "r = requests.get('http://pythonscraping.com/pages/cookies/profile.php', \n",
    "                 cookies=r.cookies)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cookie is set to:\n",
      "{'loggedin': '1', 'username': 'username'}\n",
      "Going to profile page...\n",
      "Hey username! Looks like you're still logged into the site!\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "session = requests.Session()\n",
    "\n",
    "params = {'username': 'username', 'password': 'password'}\n",
    "s = session.post('http://pythonscraping.com/pages/cookies/welcome.php', params)\n",
    "print(\"Cookie is set to:\")\n",
    "print(s.cookies.get_dict())\n",
    "print('Going to profile page...')\n",
    "s = session.get('http://pythonscraping.com/pages/cookies/profile.php')\n",
    "print(s.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>Hello ryan.</p><p>You entered password as your password.</p>\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from requests.auth import AuthBase\n",
    "from requests.auth import HTTPBasicAuth\n",
    "\n",
    "auth = HTTPBasicAuth('ryan', 'password')\n",
    "r = requests.post(\n",
    "    url='http://pythonscraping.com/pages/auth/login.php', auth=auth)\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}