CHANGELOG.txt EXAMPLES.txt FAQ.txt GUIDE.txt LICENSE.txt Manifest.txt NOTES.txt README.txt lib/mechanize/history.rb lib/mechanize/monkey_patch.rb lib/mechanize/parsers/rexml_page.rb lib/mechanize/rexml.rb test/htdocs/empty_form.html test/htdocs/link with space.html test/htdocs/relative/tc_relative_links.html test/htdocs/tc_base_link.html test/htdocs/tc_blank_form.html test/htdocs/tc_encoded_links.html test/htdocs/tc_follow_meta.html test/htdocs/tc_form_action.html test/htdocs/tc_links.html test/htdocs/tc_referer.html test/htdocs/tc_relative_links.html test/htdocs/unusual______.html test/tc_blank_form.rb test/tc_encoded_links.rb test/tc_follow_meta.rb test/tc_form_action.rb test/tc_form_as_hash.rb test/tc_form_button.rb test/tc_history.rb test/tc_html_unscape_forms.rb test/tc_if_modified_since.rb test/tc_keep_alive.rb test/tc_referer.rb test/tc_relative_links.rb test/tc_subclass.rb test/test_all.rb test/test_mechanize_file.rb test/test_servlets.rb @@ -1,101 +1,60 @@ require 'rubygems' -require 'rake' -require 'rake/testtask' -require 'rake/gempackagetask' -require 'rake/rdoctask' -require 'rake/contrib/sshpublisher' - -def announce(msg='') - STDERR.puts msg -end - -PKG_BUILD = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : '' -PKG_NAME = 'mechanize' -PKG_VERSION = '0.6.0' + PKG_BUILD -PKG_FILES = FileList["{doc,lib,test}/**/*"].exclude("rdoc").to_a - -spec = Gem::Specification.new do |s| - s.name = PKG_NAME - s.version = PKG_VERSION - s.author = "Aaron Patterson" - s.email = "aaronp@rubyforge.org" - s.homepage = "#{PKG_NAME}.rubyforge.org" - s.platform = Gem::Platform::RUBY - s.summary = "Mechanize provides automated web-browsing" - s.files = Dir.glob("{bin,test,lib,doc}/**/*").delete_if {|item| item.include?(".svn") } - s.require_path = "lib" - s.has_rdoc = true - s.extra_rdoc_files = ["README", "EXAMPLES", "CHANGELOG", "LICENSE", "NOTES", - "GUIDE"] - s.rdoc_options << "--main" << 'README' << "--title" << "'WWW::Mechanize RDoc'" - s.rubyforge_project = PKG_NAME - s.add_dependency('hpricot') -end - -Rake::GemPackageTask.new(spec) do |p| - p.gem_spec = spec - p.need_tar = true - p.need_zip = true -end - -Rake::RDocTask.new do |p| - p.main = "README" - p.rdoc_dir = "doc" - p.rdoc_files.include("README", "CHANGELOG", "LICENSE", "EXAMPLES", "NOTES", - "GUIDE", "lib/**/*.rb") - p.options << "--main" << 'README' << "--title" << "WWW::Mechanize RDoc" -end - -desc "Publish the API documentation" -task :pubrdoc => [ :rdoc ] do - Rake::SshDirPublisher.new( - "#{ENV['USER']}@rubyforge.org", - "/var/www/gforge-projects/#{PKG_NAME}/", - "doc" ).upload -end - -task :update_version do - announce "Updating Mechanize Version to #{PKG_VERSION}" - File.open("lib/mechanize/mech_version.rb", "w") do |f| - f.puts "module WWW" - f.puts " class Mechanize" - f.puts " Version = '#{PKG_VERSION}'" - f.puts " end" - f.puts "end" +require 'hoe' + +$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib") +require 'mechanize' + +class MechHoe < Hoe + def define_tasks + super + + desc "Tag code" + task :tag do |p| + abort "Must supply VERSION=x.y.z" unless ENV['VERSION'] + v = ENV['VERSION'].gsub(/\./, '_') + + rf = RubyForge.new + user = rf.userconfig['username'] + + baseurl = "svn+ssh://#{user}@rubyforge.org//var/svn/#{name}" + sh "svn cp -m 'tagged REL-#{v}' . #{ baseurl }/tags/REL-#{ v }" + end + + desc "Branch code" + Rake::Task.define_task("branch") do |p| + abort "Must supply VERSION=x.y.z" unless ENV['VERSION'] + v = ENV['VERSION'].split(/\./)[0..1].join('_') + + rf = RubyForge.new + user = rf.userconfig['username'] + + baseurl = "svn+ssh://#{user}@rubyforge.org/var/svn/#{name}" + sh "svn cp -m'branched #{v}' #{baseurl}/trunk #{baseurl}/branches/RB-#{v}" + end + + desc "Update SSL Certificate" + Rake::Task.define_task('ssl_cert') do |p| + sh "openssl genrsa -des3 -out server.key 1024" + sh "openssl req -new -key server.key -out server.csr" + sh "cp server.key server.key.org" + sh "openssl rsa -in server.key.org -out server.key" + sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt" + sh "cp server.key server.pem" + sh "mv server.key server.csr server.crt server.pem test/data/" + sh "rm server.key.org" + end end - sh 'svn commit -m"updating version" lib/mechanize/mech_version.rb' end -desc "Create a new release" -task :release => [ :clobber, :update_version, :package, :tag ] do - announce - announce "**************************************************************" - announce "* Release #{PKG_VERSION} Complete." - announce "* Packages ready to upload." - announce "**************************************************************" - announce +MechHoe.new('mechanize', WWW::Mechanize::VERSION) do |p| + p.rubyforge_name = 'mechanize' + p.author = 'Aaron Patterson' + p.email = 'aaronp@rubyforge.org' + p.summary = "Mechanize provides automated web-browsing" + p.description = p.paragraphs_of('README.txt', 3).join("\n\n") + p.url = p.paragraphs_of('README.txt', 1).first.strip + p.changes = p.paragraphs_of('CHANGELOG.txt', 0..2).join("\n\n") + p.extra_deps = [['hpricot', '>= 0.5.0']] end -desc "Tag code" -Rake::Task.define_task("tag") do |p| - baseurl = "svn+ssh://#{ENV['USER']}@rubyforge.org//var/svn/#{PKG_NAME}" - sh "svn cp -m 'tagged #{ PKG_VERSION }' . #{ baseurl }/tags/REL-#{ PKG_VERSION }" -end - -desc "Branch code" -Rake::Task.define_task("branch") do |p| - baseurl = "svn+ssh://#{ENV['USER']}@rubyforge.org/var/svn/#{PKG_NAME}" - sh "svn cp -m 'branched #{ PKG_VERSION }' #{baseurl}/trunk #{ baseurl }/branches/RB-#{ PKG_VERSION }" -end -desc "Update SSL Certificate" -Rake::Task.define_task('ssl_cert') do |p| - sh "openssl genrsa -des3 -out server.key 1024" - sh "openssl req -new -key server.key -out server.csr" - sh "cp server.key server.key.org" - sh "openssl rsa -in server.key.org -out server.key" - sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt" - sh "cp server.key server.pem" - sh "mv server.key server.csr server.crt server.pem test/data/" - sh "rm server.key.org" -end Rakefile @@ -11,20 +11,31 @@ unless RUBY_VERSION > "1.8.2" $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "mechanize", "net-overrides") end + require 'net/http' require 'net/https' +# Monkey patch for ruby 1.8.4 +unless RUBY_VERSION > "1.8.4" +module Net # :nodoc: + class HTTPResponse # :nodoc: + CODE_TO_OBJ['500'] = HTTPInternalServerError + end +end +end + require 'uri' require 'webrick/httputils' require 'zlib' require 'stringio' -require 'mechanize/hpricot' -require 'mechanize/mech_version' +require 'digest/md5' +require 'mechanize/monkey_patch' require 'mechanize/cookie' require 'mechanize/errors' require 'mechanize/pluggable_parsers' require 'mechanize/form' require 'mechanize/form_elements' +require 'mechanize/history' require 'mechanize/list' require 'mechanize/page' require 'mechanize/page_elements' @@ -50,20 +61,27 @@ module WWW # search_results = agent.submit(search_form) # puts search_results.body class Mechanize + ## + # The version of Mechanize you are using. + + VERSION = '0.6.11' + + ## + # User Agent aliases AGENT_ALIASES = { 'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', + 'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', 'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6', 'Mac Safari' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3', 'Mac FireFox' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3', 'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401', 'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624', 'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)', - 'Mechanize' => "WWW-Mechanize/#{Version} (http://rubyforge.org/projects/mechanize/)" + 'Mechanize' => "WWW-Mechanize/#{VERSION} (http://rubyforge.org/projects/mechanize/)" } attr_accessor :cookie_jar attr_accessor :log - attr_accessor :max_history attr_accessor :open_timeout, :read_timeout attr_accessor :user_agent attr_accessor :watch_for_set @@ -71,15 +89,24 @@ class Mechanize attr_accessor :key attr_accessor :cert attr_accessor :pass + attr_accessor :redirect_ok + attr_accessor :keep_alive_time + attr_accessor :keep_alive + attr_accessor :conditional_requests + attr_accessor :follow_meta_refresh attr_reader :history attr_reader :pluggable_parser + alias :follow_redirect? :redirect_ok + + @@nonce_count = -1 + CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535))) + def initialize # attr_accessors - @cookie_jar = CookieJar.new + @cookie_jar = CookieJar.new @log = nil - @max_history = nil @open_timeout = nil @read_timeout = nil @user_agent = AGENT_ALIASES['Mechanize'] @@ -88,14 +115,17 @@ class Mechanize @cert = nil # OpenSSL Certificate @key = nil # OpenSSL Private Key @pass = nil # OpenSSL Password + @redirect_ok = true # Should we follow redirects? # attr_readers - @history = [] + @history = WWW::Mechanize::History.new @pluggable_parser = PluggableParser.new - # Basic Auth variables - @user = nil # Basic Auth User - @password = nil # Basic Auth Password + # Auth variables + @user = nil # Auth User + @password = nil # Auth Password + @digest = nil # DigestAuth Digest + @auth_hash = {} # Keep track of urls for sending auth # Proxy settings @proxy_addr = nil @@ -103,9 +133,21 @@ class Mechanize @proxy_port = nil @proxy_user = nil + @conditional_requests = true + + @follow_meta_refresh = false + + # Connection Cache & Keep alive + @connection_cache = {} + @keep_alive_time = 300 + @keep_alive = true + yield self if block_given? end + def max_history=(length); @history.max_size = length; end + def max_history; @history.max_size; end + # Sets the proxy address, port, user, and password def set_proxy(addr, port, user = nil, pass = nil) @proxy_addr, @proxy_port, @proxy_user, @proxy_pass = addr, port, user, pass @@ -124,18 +166,23 @@ class Mechanize # Sets the user and password to be used for basic authentication. def basic_auth(user, password) - @user = user - @password = password + auth(user, password) + end + + def auth(user, password) + @user = user + @password = password end # Fetches the URL passed in and returns a page. - def get(url) - cur_page = current_page || Page.new( nil, {'content-type'=>'text/html'}) + def get(url, referer=nil, &block) + cur_page = referer || current_page || + Page.new( nil, {'content-type'=>'text/html'}) # fetch the page abs_uri = to_absolute_uri(url, cur_page) request = fetch_request(abs_uri) - page = fetch_page(abs_uri, request, cur_page) + page = fetch_page(abs_uri, request, cur_page, &block) add_to_history(page) page end @@ -149,10 +196,17 @@ class Mechanize # Clicks the WWW::Mechanize::Link object passed in and returns the # page fetched. def click(link) + referer = + begin + link.page + rescue + nil + end uri = to_absolute_uri( - link.attributes['href'] || link.attributes['src'] || link.href + link.attributes['href'] || link.attributes['src'] || link.href, + referer || current_page() ) - get(uri) + get(uri, referer) end # Equivalent to the browser back button. Returns the most recent page @@ -169,9 +223,8 @@ class Mechanize # agent.post('http://example.com/', [ ["foo", "bar"] ]) def post(url, query={}) node = Hpricot::Elem.new(Hpricot::STag.new('form')) - node.attributes = {} - node.attributes['method'] = 'POST' - node.attributes['enctype'] = 'application/x-www-form-urlencoded' + node['method'] = 'POST' + node['enctype'] = 'application/x-www-form-urlencoded' form = Form.new(node) query.each { |k,v| @@ -188,17 +241,12 @@ class Mechanize # agent.submit(page.forms.first, page.forms.first.buttons.first) def submit(form, button=nil) form.add_button_to_query(button) if button - uri = to_absolute_uri(form.action) + uri = to_absolute_uri(form.action, form.page) case form.method.upcase when 'POST' post_form(uri, form) when 'GET' - if uri.query.nil? - uri.query = WWW::Mechanize.build_query_string(form.build_query) - else - uri.query = uri.query + "&" + - WWW::Mechanize.build_query_string(form.build_query) - end + uri.query = WWW::Mechanize.build_query_string(form.build_query) get(uri) else raise "unsupported method: #{form.method.upcase}" @@ -212,9 +260,15 @@ class Mechanize # Returns whether or not a url has been visited def visited?(url) - url = url.uri if url.respond_to? :uri - uri = to_absolute_uri(url).to_s - ! @history.find { |h| h.uri.to_s == uri }.nil? + ! visited_page(url).nil? + end + + # Returns a visited page for the url passed in, otherwise nil + def visited_page(url) + if url.respond_to? :href + url = url.href + end + @history.visited_page(to_absolute_uri(url)) end # Runs given block, then resets the page history as it was before. self is @@ -230,22 +284,136 @@ class Mechanize alias :page :current_page + protected + def set_headers(uri, request, cur_page) + if @keep_alive + request.add_field('Connection', 'keep-alive') + request.add_field('Keep-Alive', keep_alive_time.to_s) + else + request.add_field('Connection', 'close') + end + request.add_field('Accept-Encoding', 'gzip,identity') + request.add_field('Accept-Language', 'en-us,en;q0.5') + request.add_field('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7') + + unless @cookie_jar.empty?(uri) + cookies = @cookie_jar.cookies(uri) + cookie = cookies.length > 0 ? cookies.join("; ") : nil + if log + cookies.each do |c| + log.debug("using cookie: #{c}") + end + end + request.add_field('Cookie', cookie) + end + + # Add Referer header to request + unless cur_page.uri.nil? + request.add_field('Referer', cur_page.uri.to_s) + end + + # Add User-Agent header to request + request.add_field('User-Agent', @user_agent) if @user_agent + + # Add If-Modified-Since if page is in history + if @conditional_requests + if( (page = visited_page(uri)) && page.response['Last-Modified'] ) + request.add_field('If-Modified-Since', page.response['Last-Modified']) + end + end + + if( @auth_hash[uri.host] ) + case @auth_hash[uri.host] + when :basic + request.basic_auth(@user, @password) + when :digest + @digest_response ||= nil + @digest_response = self.gen_auth_header(uri,request,@digest) if @digest + request.add_field('Authorization', @digest_response) if @digest_response + end + end + + request + end + + def gen_auth_header(uri, request, auth_header, is_IIS = false) + @@nonce_count += 1 + + user = @digest_user + password = @digest_password + + auth_header =~ /^(\w+) (.*)/ + + params = {} + $2.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 } + + a_1 = "#{@user}:#{params['realm']}:#{@password}" + a_2 = "#{request.method}:#{uri.path}" + request_digest = '' + request_digest << Digest::MD5.hexdigest(a_1) + request_digest << ':' << params['nonce'] + request_digest << ':' << ('%08x' % @@nonce_count) + request_digest << ':' << CNONCE + request_digest << ':' << params['qop'] + request_digest << ':' << Digest::MD5.hexdigest(a_2) + + header = '' + header << "Digest username=\"#{@user}\", " + header << "realm=\"#{params['realm']}\", " + if is_IIS then + header << "qop=\"#{params['qop']}\", " + else + header << "qop=#{params['qop']}, " + end + header << "uri=\"#{uri.path}\", " + header << "algorithm=MD5, " + header << "nonce=\"#{params['nonce']}\", " + header << "nc=#{'%08x' % @@nonce_count}, " + header << "cnonce=\"#{CNONCE}\", " + header << "response=\"#{Digest::MD5.hexdigest(request_digest)}\"" + + return header + end + private def to_absolute_uri(url, cur_page=current_page()) - url = URI.parse(URI.escape(url.to_s.strip)) unless url.is_a? URI + unless url.is_a? URI + url = url.to_s.strip.gsub(/[^#{0.chr}-#{125.chr}]/) { |match| + sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'c')[0]) + } + + url = URI.parse( + Util.html_unescape( + url.split(/%[0-9A-Fa-f]{2}|#/).zip( + url.scan(/%[0-9A-Fa-f]{2}|#/) + ).map { |x,y| + "#{URI.escape(x)}#{y}" + }.join('') + ) + ) + end + + url.path = '/' if url.path.length == 0 # construct an absolute uri if url.relative? raise 'no history. please specify an absolute URL' unless cur_page.uri + base = cur_page.respond_to?(:bases) ? cur_page.bases.last : nil + url = ((base && base.uri && base.uri.absolute?) ? + base.uri : + cur_page.uri) + url url = cur_page.uri + url + # Strip initial "/.." bits from the path + url.path.sub!(/^(\/\.\.)+(?=\/)/, '') end return url end def post_form(url, form) - cur_page = current_page || Page.new(nil, {'content-type'=>'text/html'}) + cur_page = form.page || current_page || + Page.new( nil, {'content-type'=>'text/html'}) request_data = form.request_data @@ -276,19 +444,32 @@ class Mechanize def fetch_page(uri, request, cur_page=current_page(), request_data=[]) raise "unsupported scheme" unless ['http', 'https'].include?(uri.scheme) - log.info("#{ request.class }: #{ uri.to_s }") if log + log.info("#{ request.class }: #{ request.path }") if log page = nil - http_obj = Net::HTTP.new( uri.host, - uri.port, - @proxy_addr, - @proxy_port, - @proxy_user, - @proxy_pass - ) + cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= { + :connection => nil, + :keep_alive_options => {}, + }) + http_obj = cache_obj[:connection] + if http_obj.nil? || ! http_obj.started? + http_obj = cache_obj[:connection] = + Net::HTTP.new( uri.host, + uri.port, + @proxy_addr, + @proxy_port, + @proxy_user, + @proxy_pass + ) + cache_obj[:keep_alive_options] = {} + + # Specify timeouts if given + http_obj.open_timeout = @open_timeout if @open_timeout + http_obj.read_timeout = @read_timeout if @read_timeout + end - if uri.scheme == 'https' + if uri.scheme == 'https' && ! http_obj.started? http_obj.use_ssl = true http_obj.verify_mode = OpenSSL::SSL::VERIFY_NONE if @ca_file @@ -301,28 +482,24 @@ class Mechanize end end - request.add_field('Accept-Encoding', 'gzip,identity') + # If we're keeping connections alive and the last request time is too + # long ago, stop the connection. Or, if the max requests left is 1, + # reset the connection. + if @keep_alive && http_obj.started? + opts = cache_obj[:keep_alive_options] + if((opts[:timeout] && + Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) || + opts[:max] && opts[:max].to_i == 1) - unless @cookie_jar.empty?(uri) - cookies = @cookie_jar.cookies(uri) - cookie = cookies.length > 0 ? cookies.join("; ") : nil - if log - cookies.each do |c| - log.debug("using cookie: #{c}") - end - end - request.add_field('Cookie', cookie) - end + log.debug('Finishing stale connection') if log + http_obj.finish - # Add Referer header to request - unless cur_page.uri.nil? - request.add_field('Referer', cur_page.uri.to_s) + end end - # Add User-Agent header to request - request.add_field('User-Agent', @user_agent) if @user_agent + http_obj.start unless http_obj.started? - request.basic_auth(@user, @password) if @user || @password + request = set_headers(uri, request, cur_page) # Log specified headers for the request if log @@ -331,76 +508,115 @@ class Mechanize end end - http_obj.start { |http| - # Specify timeouts if given - http.open_timeout = @open_timeout if @open_timeout - http.read_timeout = @read_timeout if @read_timeout - - # Send the request - http.request(request, *request_data) {|response| - if log - response.each_header {|k,v| - log.debug("response-header: #{ k } => #{ v }") - } - end + cache_obj[:last_request_time] = Time.now.to_i - (response.get_fields('Set-Cookie')||[]).each do |cookie| - Cookie::parse(uri, cookie) { |c| - log.debug("saved cookie: #{c}") if log - @cookie_jar.add(uri, c) - } - end + # Send the request + response = http_obj.request(request, *request_data) {|response| + + body = StringIO.new + total = 0 + response.read_body { |part| + total += part.length + body.write(part) + log.debug("Read #{total} bytes") if log + } + body.rewind - response.read_body + response.each_header { |k,v| + log.debug("response-header: #{ k } => #{ v }") + } if log - content_type = nil - unless response['Content-Type'].nil? - data = response['Content-Type'].match(/^([^;]*)/) - content_type = data[1].downcase unless data.nil? - end + content_type = nil + unless response['Content-Type'].nil? + data = response['Content-Type'].match(/^([^;]*)/) + content_type = data[1].downcase unless data.nil? + end - response_body = - if encoding = response['Content-Encoding'] - case encoding.downcase - when 'gzip' - log.debug('gunzip body') if log - Zlib::GzipReader.new(StringIO.new(response.body)).read - else - raise 'Unsupported content encoding' - end + response_body = + if encoding = response['Content-Encoding'] + case encoding.downcase + when 'gzip' + log.debug('gunzip body') if log + Zlib::GzipReader.new(body).read + when 'x-gzip' + body.read else - response.body + raise 'Unsupported content encoding' end + else + body.read + end - # Find our pluggable parser - page = @pluggable_parser.parser(content_type).new( - uri, - response, - response_body, - response.code - ) + # Find our pluggable parser + page = @pluggable_parser.parser(content_type).new( + uri, + response, + response_body, + response.code + ) { |parser| + parser.mech = self if parser.respond_to? :mech= + if parser.respond_to?(:watch_for_set=) && @watch_for_set + parser.watch_for_set = @watch_for_set + end + } - log.info("status: #{ page.code }") if log + } - if page.respond_to? :watch_for_set - page.watch_for_set = @watch_for_set - end + # If the server sends back keep alive options, save them + if keep_alive_info = response['keep-alive'] + keep_alive_info.split(/,\s*/).each do |option| + k, v = option.split(/=/) + cache_obj[:keep_alive_options] ||= {} + cache_obj[:keep_alive_options][k.intern] = v + end + end - case page.code - when "200" - return page - when "301", "302" - log.info("follow redirect to: #{ response['Location'] }") if log - abs_uri = to_absolute_uri( - URI.parse( - URI.escape(URI.unescape(response['Location'].to_s))), page) - request = fetch_request(abs_uri) - return fetch_page(abs_uri, request, page) - else - raise ResponseCodeError.new(page.code), "Unhandled response", caller - end + (response.get_fields('Set-Cookie')||[]).each do |cookie| + Cookie::parse(uri, cookie, log) { |c| + log.debug("saved cookie: #{c}") if log + @cookie_jar.add(uri, c) } - } + end + + log.info("status: #{ page.code }") if log + + res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s] + + if follow_meta_refresh && page.respond_to?(:meta) && + (redirect = page.meta.first) + return redirect.click + end + + return page if res_klass <= Net::HTTPSuccess + + if res_klass == Net::HTTPNotModified + log.debug("Got cached page") if log + return visited_page(uri) + elsif res_klass <= Net::HTTPRedirection + return page unless follow_redirect? + log.info("follow redirect to: #{ response['Location'] }") if log + from_uri = page.uri + abs_uri = to_absolute_uri(response['Location'].to_s, page) + page = fetch_page(abs_uri, fetch_request(abs_uri), page) + @history.push(page, from_uri) + return page + elsif res_klass <= Net::HTTPUnauthorized + raise ResponseCodeError.new(page) unless @user || @password + raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host) + if response['www-authenticate'] =~ /Digest/i + @auth_hash[uri.host] = :digest + @digest = response['www-authenticate'] + else + @auth_hash[uri.host] = :basic + end + return fetch_page( uri, + fetch_request(uri, request.method.downcase.to_sym), + cur_page, + request_data + ) + end + + raise ResponseCodeError.new(page), "Unhandled response", caller end def self.build_query_string(parameters) @@ -416,12 +632,26 @@ class Mechanize end def add_to_history(page) - @history.push(page) - if @max_history and @history.size > @max_history - # keep only the last @max_history entries - @history = @history[@history.size - @max_history, @max_history] + @history.push(page, to_absolute_uri(page.uri)) + end + + # :stopdoc: + class Util + def self.html_unescape(s) + return s unless s + s.gsub(/&(\w+|#[0-9]+);/) { |match| + number = case match + when /&(\w+);/ + Hpricot::NamedCharacters[$1] + when /&#([0-9]+);/ + $1.to_i + end + + number ? ([number].pack('U') rescue match) : match + } end end + # :startdoc: end end # module WWW lib/mechanize.rb @@ -6,9 +6,8 @@ module WWW class Mechanize # This class is used to represent an HTTP Cookie. class Cookie < WEBrick::Cookie - def self.parse(uri, str) - cookies = [] - str.gsub(/(,([^;,]*=)|,$)/) { "\r\n#{$2}" }.split(/\r\n/).each { |c| + def self.parse(uri, str, log = nil) + return str.split(/,(?=[^;,]*=)|,$/).collect { |c| cookie_elem = c.split(/;/) first_elem = cookie_elem.shift first_elem.strip! @@ -24,25 +23,38 @@ module WWW when "domain" then cookie.domain = value.sub(/^\./, '') when "path" then cookie.path = value when 'expires' - cookie.expires = begin - Time::parse(value) + begin + cookie.expires = Time::parse(value) + rescue + if log + log.warn("Couldn't parse expires: #{value}") + end + end + when "max-age" then + begin + cookie.max_age = Integer(value) rescue - Time.now + log.warn("Couldn't parse max age '#{value}'") if log + cookie.max_age = nil end - when "max-age" then cookie.max_age = Integer(value) when "comment" then cookie.comment = value - when "version" then cookie.version = Integer(value) + when "version" then + begin + cookie.version = Integer(value) + rescue + log.warn("Couldn't parse version '#{value}'") if log + cookie.version = nil + end when "secure" then cookie.secure = true end } - cookie.path ||= uri.path + + cookie.path ||= uri.path.to_s.sub(/[^\/]*$/, '') cookie.secure ||= false cookie.domain ||= uri.host # Move this in to the cookie jar yield cookie if block_given? - cookies << cookie } - return cookies end def to_s @@ -61,12 +73,13 @@ module WWW # Add a cookie to the Jar. def add(uri, cookie) - return unless uri.host =~ /#{cookie.domain}$/ - unless @jar.has_key?(cookie.domain) - @jar[cookie.domain] = Hash.new + return unless uri.host =~ /#{cookie.domain}$/i + normal_domain = cookie.domain.downcase + unless @jar.has_key?(normal_domain) + @jar[normal_domain] = Hash.new end - @jar[cookie.domain][cookie.name] = cookie + @jar[normal_domain][cookie.name] = cookie cleanup() cookie end @@ -77,7 +90,7 @@ module WWW cookies = [] url.path = '/' if url.path.empty? @jar.each_key do |domain| - if url.host =~ /#{domain}$/ + if url.host =~ /#{domain}$/i @jar[domain].each_key do |name| if url.path =~ /^#{@jar[domain][name].path}/ if @jar[domain][name].expires.nil? lib/mechanize/cookie.rb @@ -20,10 +20,18 @@ module WWW # Any other response code is up to the user to handle. class ResponseCodeError < RuntimeError attr_reader :response_code + attr_reader :page - def initialize(response_code) - @response_code = response_code + def initialize(page) + @page = page + @response_code = page.code end + + def to_s + "#{response_code} => #{Net::HTTPResponse::CODE_TO_OBJ[response_code]}" + end + + def inspect; to_s; end end end end lib/mechanize/errors.rb @@ -26,15 +26,16 @@ module WWW attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes attr_reader :enctype + + alias :elements :fields def initialize(form_node, elements_node) @form_node, @elements_node = form_node, elements_node - @form_node.attributes ||= {} - @method = (@form_node.attributes['method'] || 'GET').upcase - @action = @form_node.attributes['action'] - @name = @form_node.attributes['name'] - @enctype = @form_node.attributes['enctype'] || 'application/x-www-form-urlencoded' + @method = (@form_node['method'] || 'GET').upcase + @action = Util::html_unescape(@form_node['action']) + @name = @form_node['name'] + @enctype = @form_node['enctype'] || 'application/x-www-form-urlencoded' @clicked_buttons = [] parse @@ -47,7 +48,6 @@ module WWW query = [] fields().each do |f| - next unless f.value query.push(*f.query_value) end @@ -105,6 +105,11 @@ module WWW end end + # Removes all fields with name +field_name+. + def delete_field!(field_name) + @fields.delete_if{ |f| f.name == field_name} + end + private def parse @fields = WWW::Mechanize::List.new @@ -115,41 +120,40 @@ module WWW # Find all input tags (@elements_node/'input').each do |node| - node.attributes ||= {} - type = (node.attributes['type'] || 'text').downcase - name = node.attributes['name'] - next if type != 'submit' && name.nil? + type = (node['type'] || 'text').downcase + name = node['name'] + next if name.nil? && !(type == 'submit' || type =='button') case type when 'text', 'password', 'hidden', 'int' - @fields << Field.new(node.attributes['name'], node.attributes['value'] || '') + @fields << Field.new(node['name'], node['value'] || '') when 'radio' - @radiobuttons << RadioButton.new(node.attributes['name'], node.attributes['value'], node.attributes.has_key?('checked'), self) + @radiobuttons << RadioButton.new(node['name'], node['value'], node.has_attribute?('checked'), self) when 'checkbox' - @checkboxes << CheckBox.new(node.attributes['name'], node.attributes['value'], node.attributes.has_key?('checked'), self) + @checkboxes << CheckBox.new(node['name'], node['value'], node.has_attribute?('checked'), self) when 'file' - @file_uploads << FileUpload.new(node.attributes['name'], nil) + @file_uploads << FileUpload.new(node['name'], nil) when 'submit' - @buttons << Button.new(node.attributes['name'], node.attributes['value']) + @buttons << Button.new(node['name'], node['value']) + when 'button' + @buttons << Button.new(node['name'], node['value']) when 'image' - @buttons << ImageButton.new(node.attributes['name'], node.attributes['value']) + @buttons << ImageButton.new(node['name'], node['value']) end end # Find all textarea tags (@elements_node/'textarea').each do |node| - next if node.attributes.nil? - next if node.attributes['name'].nil? - @fields << Field.new(node.attributes['name'], node.all_text) + next if node['name'].nil? + @fields << Field.new(node['name'], node.inner_text) end # Find all select tags (@elements_node/'select').each do |node| - next if node.attributes.nil? - next if node.attributes['name'].nil? - if node.attributes.has_key? 'multiple' - @fields << MultiSelectList.new(node.attributes['name'], node) + next if node['name'].nil? + if node.has_attribute? 'multiple' + @fields << MultiSelectList.new(node['name'], node) else - @fields << SelectList.new(node.attributes['name'], node) + @fields << SelectList.new(node['name'], node) end end end @@ -172,9 +176,10 @@ module WWW end def file_to_multipart(file) + file_name = file.file_name ? ::File.basename(file.file_name) : '' body = "Content-Disposition: form-data; name=\"" + "#{mime_value_quote(file.name)}\"; " + - "filename=\"#{mime_value_quote(file.file_name || '')}\"\r\n" + + "filename=\"#{mime_value_quote(file_name)}\"\r\n" + "Content-Transfer-Encoding: binary\r\n" if file.file_data.nil? and ! file.file_name.nil? @@ -212,17 +217,39 @@ module WWW # puts form['name'] class Form < GlobalForm attr_reader :node + attr_reader :page - def initialize(node) - @node = node - super(@node, @node) + def initialize(node, mech=nil, page=nil) + super(node, node) + @page = page + @mech = mech + end + + # Returns whether or not the form contains a field with +field_name+ + def has_field?(field_name) + ! fields.find { |f| f.name.eql? field_name }.nil? end - # Fetch the first field whose name is equal to field_name + alias :has_key? :has_field? + + def has_value?(value) + ! fields.find { |f| f.value.eql? value }.nil? + end + + def keys; fields.map { |f| f.name }; end + + def values; fields.map { |f| f.value }; end + + # Fetch the first field whose name is equal to +field_name+ def field(field_name) fields.find { |f| f.name.eql? field_name } end + # Add a field with +field_name+ and +value+ + def add_field!(field_name, value = nil) + fields << WWW::Mechanize::Field.new(field_name, value) + end + # This method sets multiple fields on the form. It takes a list of field # name, value pairs. If there is more than one field found with the # same name, this method will set the first one found. If you want to @@ -248,7 +275,8 @@ module WWW # Fetch the value set in the input field 'name' # puts form['name'] def [](field_name) - field(field_name).value + f = field(field_name) + f && f.value end # Set the value of the first input field with the name passed in @@ -256,7 +284,12 @@ module WWW # Set the value in the input field 'name' to "Aaron" # form['name'] = 'Aaron' def []=(field_name, value) - field(field_name).value = value + f = field(field_name) + if f.nil? + add_field!(field_name, value) + else + f.value = value + end end # Treat form fields like accessors. @@ -268,6 +301,11 @@ module WWW end super end + + # Submit this form with the button passed in + def submit(button=nil) + @mech.submit(self, button) + end end end end lib/mechanize/form.rb @@ -10,7 +10,12 @@ module WWW attr_accessor :name, :value def initialize(name, value) - @name, @value = name, value + @name = Util.html_unescape(name) + @value = if value.is_a? String + Util.html_unescape(value) + else + value + end end def query_value @@ -22,9 +27,8 @@ module WWW # class, set WWW::FileUpload#file_data= to the data of the file you want # to upload and WWW::FileUpload#mime_type= to the appropriate mime type # of the file. - # See the example in EXAMPLES[link://files/EXAMPLES.html] + # See the example in EXAMPLES[link://files/EXAMPLES_txt.html] class FileUpload < Field - attr_accessor :name # Field name attr_accessor :file_name # File name attr_accessor :mime_type # Mime Type (Optional) @@ -32,7 +36,7 @@ module WWW alias :file_data= :value= def initialize(name, file_name) - @file_name = file_name + @file_name = Util.html_unescape(file_name) @file_data = nil super(name, @file_data) end @@ -125,7 +129,7 @@ module WWW end def query_value - value.collect { |v| [name, v] } + value ? value.collect { |v| [name, v] } : '' end # Select no options @@ -213,9 +217,9 @@ module WWW alias :selected? :selected def initialize(node, select_list) - @text = node.all_text - @value = node.attributes['value'] - @selected = node.attributes.has_key?('selected') ? true : false + @text = node.inner_text + @value = Util.html_unescape(node['value']) + @selected = node.has_attribute? 'selected' @select_list = select_list # The select list this option belongs to end lib/mechanize/form_elements.rb @@ -40,6 +40,9 @@ module WWW } } end + if RUBY_VERSION > '1.8.4' + alias :inspect :pretty_inspect + end end class Link @@ -49,6 +52,9 @@ module WWW q.breakable; q.pp href } end + if RUBY_VERSION > '1.8.4' + alias :inspect :pretty_inspect + end end class Form lib/mechanize/inspect.rb @@ -54,12 +54,16 @@ module WWW alias :and :with def method_missing(meth_sym, *args) - return first.send(meth_sym) if args.empty? - arg = args.first - if arg.class == Regexp - WWW::Mechanize::List.new(find_all { |e| e.send(meth_sym) =~ arg }) + if length > 0 + return first.send(meth_sym) if args.empty? + arg = args.first + if arg.class == Regexp + WWW::Mechanize::List.new(find_all { |e| e.send(meth_sym) =~ arg }) + else + WWW::Mechanize::List.new(find_all { |e| e.send(meth_sym) == arg }) + end else - WWW::Mechanize::List.new(find_all { |e| e.send(meth_sym) == arg }) + '' end end end lib/mechanize/list.rb @@ -1,4 +1,4 @@ -# +# :enddoc: # = net/http.rb # # Copyright (C) 1999-2005 Yukihiro Matsumoto @@ -1826,7 +1826,7 @@ module Net # :nodoc: '416' => HTTPRequestedRangeNotSatisfiable, '417' => HTTPExpectationFailed, - '501' => HTTPInternalServerError, + '500' => HTTPInternalServerError, '501' => HTTPNotImplemented, '502' => HTTPBadGateway, '503' => HTTPServiceUnavailable, lib/mechanize/net-overrides/net/http.rb @@ -1,3 +1,4 @@ +# :enddoc: =begin = $RCSfile: https.rb,v $ -- SSL/TLS enhancement for Net::HTTP. lib/mechanize/net-overrides/net/https.rb @@ -1,4 +1,4 @@ -# +# :enddoc: # = net/protocol.rb # #-- lib/mechanize/net-overrides/net/protocol.rb @@ -1,5 +1,6 @@ require 'fileutils' require 'hpricot' +require 'forwardable' module WWW class Mechanize @@ -15,23 +16,27 @@ module WWW # agent.get('http://google.com/').class #=> WWW::Mechanize::Page # class Page < File - attr_reader :root, :title, :watch_for_set - attr_reader :frames, :iframes, :links, :forms, :meta, :watches + extend Forwardable - def initialize(uri=nil, response=nil, body=nil, code=nil) - super(uri, response, body, code) - @watch_for_set = {} + attr_reader :parser, :title, :watch_for_set + attr_reader :frames, :iframes, :links, :forms, :meta, :watches, :bases + attr_accessor :mech + + alias :root :parser - yield self if block_given? + def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil) + super(uri, response, body, code) + @watch_for_set ||= {} + @mech ||= mech raise Mechanize::ContentTypeError.new(response['content-type']) unless content_type() =~ /^text\/html/ - parse_html if body && response - end - - # Get the response header - def header - @response + + # construct parser and feed with HTML + if body && response + @parser ||= Hpricot.parse(body) + parse_html + end end # Get the content type @@ -40,84 +45,88 @@ module WWW end # Search through the page like HPricot - def search(*args) - @root.search(*args) - end + def_delegator :@parser, :search, :search + def_delegator :@parser, :/, :/ + def_delegator :@parser, :at, :at - def at(*args) - @root.at(*args) - end - - alias :/ :search - def watch_for_set=(obj) @watch_for_set = obj - parse_html if @body + parse_html if @body && @watch_for_set end + # Find a form with +name+. Form will be yeilded if a block is given. def form(name) - forms.name(name).first + f = forms.name(name).first + yield f if block_given? + f end private def parse_html - # construct parser and feed with HTML - @root = Hpricot.parse(@body) - @forms = WWW::Mechanize::List.new @links = WWW::Mechanize::List.new @meta = WWW::Mechanize::List.new @frames = WWW::Mechanize::List.new @iframes = WWW::Mechanize::List.new + @bases = WWW::Mechanize::List.new @watches = {} # Set the title - @title = if (@root/'title').text.length > 0 - (@root/'title').text + @title = if (@parser/'title').text.length > 0 + (@parser/'title').text + end + + # Find all 'base' tags + (@parser/'base').each do |node| + @bases << Base.new(node, @mech, self) end # Find all the form tags - (@root/'form').each do |html_form| - form = Form.new(html_form) + (@parser/'form').each do |html_form| + form = Form.new(html_form, @mech, self) form.action ||= @uri @forms << form end # Find all the 'a' tags - (@root/'a').each do |node| - @links << Link.new(node) + (@parser/'a').each do |node| + @links << Link.new(node, @mech, self) + end + + # Find all the 'area' tags + (@parser/'area').each do |node| + @links << Link.new(node, @mech, self) end # Find all 'meta' tags - (@root/'meta').each do |node| - next if node.attributes.nil? - next unless node.attributes.has_key? 'http-equiv' - next unless node.attributes.has_key? 'content' - equiv = node.attributes['http-equiv'] - content = node.attributes['content'] + (@parser/'meta').each do |node| + next unless node['http-equiv'] + next unless node['content'] + equiv = node['http-equiv'] + content = node['content'] if equiv != nil && equiv.downcase == 'refresh' - if content != nil && content =~ /^\d+\s*;\s*url\s*=\s*(\S+)/i - node.attributes['href'] = $1 - @meta << Meta.new(node) + if content != nil && content =~ /^\d+\s*;\s*url\s*=\s*'?([^\s']+)/i + node['href'] = $1 + @meta << Meta.new(node, @mech, self) end end end # Find all 'frame' tags - (@root/'frame').each do |node| - @frames << Frame.new(node) + (@parser/'frame').each do |node| + @frames << Frame.new(node, @mech, self) end # Find all 'iframe' tags - (@root/'iframe').each do |node| - @iframes << Frame.new(node) + (@parser/'iframe').each do |node| + @iframes << Frame.new(node, @mech, self) end # Find all watch tags unless @watch_for_set.nil? @watch_for_set.each do |key, klass| - (@root/key).each do |node| + (@parser/key).each do |node| @watches[key] ||= [] @watches[key] << (klass ? klass.new(node) : node) end lib/mechanize/page.rb @@ -13,21 +13,23 @@ module WWW attr_reader :href attr_reader :text attr_reader :attributes + attr_reader :page alias :to_s :text + alias :referer :page - def initialize(node) - node.attributes ||= {} + def initialize(node, mech, page) @node = node - @href = node.attributes['href'] - @text = node.all_text - @attributes = node.attributes + @href = node['href'] + @text = node.inner_text + @page = page + @mech = mech + @attributes = node # If there is no text, try to find an image and use it's alt text if (@text.nil? || @text.length == 0) && (node/'img').length > 0 @text = '' (node/'img').each do |e| - e.attributes ||= {} - @text << (e.attributes.has_key?('alt') ? e.attributes['alt'] : '') + @text << ( e['alt'] || '') end end @@ -36,6 +38,11 @@ module WWW def uri URI.parse(@href) end + + # Click on this link + def click + @mech.click self + end end # This class encapsulates a Meta tag. Mechanize treats meta tags just @@ -53,12 +60,18 @@ module WWW alias :src :href alias :name :text - def initialize(node) - node.attributes ||= {} + def initialize(node, mech, referer) + super(node, mech, referer) @node = node - @text = node.attributes['name'] - @href = node.attributes['src'] + @text = node['name'] + @href = node['src'] end end + + # This class encapsulates a Base tag. Mechanize treats base tags just like + # 'a' tags. Base objects will contain links, but most likely will have + # no text. + class Base < Link + end end end lib/mechanize/page_elements.rb @@ -17,20 +17,57 @@ module WWW # agent.get('http://example.com/foo.jpg').class #=> WWW::Mechanize::File # class File - attr_accessor :uri, :response, :body, :code + attr_accessor :uri, :response, :body, :code, :filename + alias :header :response alias :content :body def initialize(uri=nil, response=nil, body=nil, code=nil) - @uri, @response, @body, @code = uri, response, body, code + @uri, @body, @code = uri, body, code + @response = Headers.new + + # Copy the headers in to a hash to prevent memory leaks + if response + response.each { |k,v| + @response[k] = v + } + end + + @filename = 'index.html' + + # Set the filename + if disposition = @response['content-disposition'] + disposition.split(/;\s*/).each do |pair| + k,v = pair.split(/=/, 2) + @filename = v if k.downcase == 'filename' + end + else + if @uri + @filename = @uri.path.split(/\//).last || 'index.html' + @filename << ".html" unless @filename =~ /\./ + end + end + + yield self if block_given? end # Use this method to save the content of this object to filename - def save_as(filename) + def save_as(filename = nil) + if filename.nil? + filename = @filename + number = 1 + while(::File.exists?(filename)) + filename = "#{@filename}.#{number}" + number += 1 + end + end + ::File::open(filename, "wb") { |f| f.write body } end + + alias :save :save_as end # = Synopsis @@ -50,7 +87,7 @@ module WWW attr_reader :filename def initialize(uri=nil, response=nil, body=nil, code=nil) - @uri, @response, @body, @code = uri, response, body, code + super(uri, response, body, code) path = uri.path.empty? ? 'index.html' : uri.path.gsub(/^[\/]*/, '') path += 'index.html' if path =~ /\/$/ @@ -154,5 +191,14 @@ module WWW @parsers[content_type] = klass end end + + class Headers < Hash + def [](key) + super(key.downcase) + end + def []=(key, value) + super(key.downcase, value) + end + end end end lib/mechanize/pluggable_parsers.rb @@ -3,6 +3,7 @@ <body> <a href="alt_text.html"><img alt="alt text" src="hello"></a> <a href="no_alt_text.html"><img src="hello"></a> + <a href="nil_alt_text.html"><img alt src="hello"></a> <a href="no_image.html">no image</a> <a href="no_text.html"></a> </body> test/htdocs/alt_text.html @@ -3,6 +3,7 @@ <meta http_equiv="Refresh" content="0; url=http://www.incorrect.com"> <meta http-equiv="Rfresh" content="0; url=http://www.also-wrong.com"> <meta http-equiv="Refresh" content="0; url=http://www.drphil.com/"> + <meta http-equiv="Refresh" content="0; url='http://tenderlovemaking.com/'"> <META HTTP-EQUIV="REFRESH" CONTENT="0; URL=HTTP://WWW.UPCASE.COM/"> <TITLE>Testing the links</TITLE> </head> test/htdocs/find_link.html @@ -4,6 +4,7 @@ <select name="list"> <option value="1">Option 1</option> <option value="2">Option 2</option> + <option>Option No Value</option> <option value="3">Option 3</option> <option value="4">Option 4</option> <option value="5">Option 5</option> test/htdocs/form_select_none.html @@ -13,14 +13,23 @@ class BasicAuthTest < Test::Unit::TestCase end def test_auth_success - @agent.basic_auth('mech', 'password') - page = @agent.get("http://localhost:#{PORT}/htpasswd_auth") + @agent.basic_auth('user', 'pass') + page = @agent.get("http://localhost/basic_auth") assert_equal('You are authenticated', page.body) end + def test_auth_bad_user_pass + @agent.basic_auth('aaron', 'aaron') + begin + page = @agent.get("http://localhost/basic_auth") + rescue WWW::Mechanize::ResponseCodeError => e + assert_equal("401", e.response_code) + end + end + def test_auth_failure begin - page = @agent.get("http://localhost:#{PORT}/htpasswd_auth") + page = @agent.get("http://localhost/basic_auth") rescue WWW::Mechanize::ResponseCodeError => e assert_equal("401", e.response_code) end test/tc_authenticate.rb @@ -10,7 +10,7 @@ class TestCheckBoxes < Test::Unit::TestCase def setup @agent = WWW::Mechanize.new - @page = @agent.get("http://localhost:#{PORT}/tc_checkboxes.html") + @page = @agent.get('http://localhost/tc_checkboxes.html') end def test_select_one test/tc_checkboxes.rb @@ -7,8 +7,8 @@ require 'test_includes' module Enumerable def combine - masks = inject([[], 1]){|(ar, m), e| [ar<<m, m<<1]}[0] - all = masks.inject(0){|al, m| al|m} + masks = inject([[], 1]){|(ar, m), e| [ar << m, m << 1 ] }[0] + all = masks.inject(0){ |al, m| al|m } result = [] for i in 1..all do @@ -23,6 +23,14 @@ module Enumerable end class CookieClassTest < Test::Unit::TestCase + def silently + warn_level = $VERBOSE + $VERBOSE = false + res = yield + $VERBOSE = warn_level + res + end + def test_parse_dates url = URI.parse('http://localhost/') @@ -47,12 +55,30 @@ class CookieClassTest < Test::Unit::TestCase dates.each do |date| cookie = "PREF=1; expires=#{date}" - WWW::Mechanize::Cookie.parse(url, cookie) { |cookie| - assert_equal(true, cookie.expires < yesterday) - } + silently do + WWW::Mechanize::Cookie.parse(url, cookie) { |cookie| + assert_equal(true, cookie.expires < yesterday) + } + end end end + def test_parse_bad_version + bad_cookie = 'PRETANET=TGIAqbFXtt; Name=/PRETANET; Path=/; Version=1.2; Content-type=text/html; Domain=192.168.6.196; expires=Friday, 13-November-2026 23:01:46 GMT;' + url = URI.parse('http://localhost/') + WWW::Mechanize::Cookie.parse(url, bad_cookie) { |cookie| + assert_nil(cookie.version) + } + end + + def test_parse_bad_max_age + bad_cookie = 'PRETANET=TGIAqbFXtt; Name=/PRETANET; Path=/; Max-Age=1.2; Content-type=text/html; Domain=192.168.6.196; expires=Friday, 13-November-2026 23:01:46 GMT;' + url = URI.parse('http://localhost/') + WWW::Mechanize::Cookie.parse(url, bad_cookie) { |cookie| + assert_nil(cookie.max_age) + } + end + def test_parse_date_fail url = URI.parse('http://localhost/') @@ -60,11 +86,13 @@ class CookieClassTest < Test::Unit::TestCase "20/06/95 21:07", ] - dates.each do |date| - cookie = "PREF=1; expires=#{date}" - WWW::Mechanize::Cookie.parse(url, cookie) { |cookie| - assert_equal(true, cookie.expires > (Time.now - 86400)) - } + silently do + dates.each do |date| + cookie = "PREF=1; expires=#{date}" + WWW::Mechanize::Cookie.parse(url, cookie) { |cookie| + assert_equal(true, cookie.expires.nil?) + } + end end end @@ -104,9 +132,46 @@ class CookieClassTest < Test::Unit::TestCase end end + def test_parse_valid_cookie_empty_value + url = URI.parse('http://rubyforge.org/') + cookie_params = {} + cookie_params['expires'] = 'expires=Sun, 27-Sep-2037 00:00:00 GMT' + cookie_params['path'] = 'path=/' + cookie_params['domain'] = 'domain=.rubyforge.org' + cookie_params['httponly'] = 'HttpOnly' + cookie_value = '12345%7D=' + + expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT') + + cookie_params.keys.combine.each do |c| + cookie_text = "#{cookie_value}; " + c.each_with_index do |key, idx| + if idx == (c.length - 1) + cookie_text << "#{cookie_params[key]}" + else + cookie_text << "#{cookie_params[key]}; " + end + end + cookie = nil + WWW::Mechanize::Cookie.parse(url, cookie_text) { |p_cookie| cookie = p_cookie } + assert_not_nil(cookie) + assert_equal('12345%7D=', cookie.to_s) + assert_equal('', cookie.value) + assert_equal('/', cookie.path) + assert_equal('rubyforge.org', cookie.domain) + + # if expires was set, make sure we parsed it + if c.find { |k| k == 'expires' } + assert_equal(expires, cookie.expires) + else + assert_nil(cookie.expires) + end + end + end + # If no path was given, use the one from the URL def test_cookie_using_url_path - url = URI.parse('http://rubyforge.org/login') + url = URI.parse('http://rubyforge.org/login.php') cookie_params = {} cookie_params['expires'] = 'expires=Sun, 27-Sep-2037 00:00:00 GMT' cookie_params['path'] = 'path=/' @@ -131,7 +196,7 @@ class CookieClassTest < Test::Unit::TestCase assert_not_nil(cookie) assert_equal('12345%7D=ASDFWEE345%3DASda', cookie.to_s) assert_equal('rubyforge.org', cookie.domain) - assert_equal('/login', cookie.path) + assert_equal('/', cookie.path) # if expires was set, make sure we parsed it if c.find { |k| k == 'expires' } test/tc_cookie_class.rb @@ -15,6 +15,61 @@ class CookieJarTest < Test::Unit::TestCase } c end + + def test_domain_case + values = { :name => 'Foo', + :value => 'Bar', + :path => '/', + :expires => Time.now + (10 * 86400), + :domain => 'rubyforge.org' + } + url = URI.parse('http://rubyforge.org/') + + jar = WWW::Mechanize::CookieJar.new + assert_equal(0, jar.cookies(url).length) + + # Add one cookie with an expiration date in the future + cookie = cookie_from_hash(values) + jar.add(url, cookie) + assert_equal(1, jar.cookies(url).length) + + jar.add(url, cookie_from_hash( values.merge( :domain => 'RuByForge.Org', + :name => 'aaron' + ) ) ) + + assert_equal(2, jar.cookies(url).length) + + url2 = URI.parse('http://RuByFoRgE.oRg/') + assert_equal(2, jar.cookies(url2).length) + end + + def test_empty_value + values = { :name => 'Foo', + :value => '', + :path => '/', + :expires => Time.now + (10 * 86400), + :domain => 'rubyforge.org' + } + url = URI.parse('http://rubyforge.org/') + + jar = WWW::Mechanize::CookieJar.new + assert_equal(0, jar.cookies(url).length) + + # Add one cookie with an expiration date in the future + cookie = cookie_from_hash(values) + jar.add(url, cookie) + assert_equal(1, jar.cookies(url).length) + + jar.add(url, cookie_from_hash( values.merge( :domain => 'RuByForge.Org', + :name => 'aaron' + ) ) ) + + assert_equal(2, jar.cookies(url).length) + + url2 = URI.parse('http://RuByFoRgE.oRg/') + assert_equal(2, jar.cookies(url2).length) + end + def test_add_future_cookies values = { :name => 'Foo', :value => 'Bar', test/tc_cookie_jar.rb @@ -15,7 +15,7 @@ class CookiesMechTest < Test::Unit::TestCase def test_send_cookies page = @agent.get("http://localhost:#{PORT}/many_cookies") page = @agent.get("http://localhost:#{PORT}/send_cookies") - assert_equal(2, page.links.length) + assert_equal(3, page.links.length) assert_not_nil(page.links.find { |l| l.text == "name:Aaron" }) assert_not_nil(page.links.find { |l| l.text == "no_expires:nope" }) end @@ -57,7 +57,7 @@ class CookiesMechTest < Test::Unit::TestCase no_path_cookie = @agent.cookies.find { |k| k.name == "no_path" } assert_not_nil(no_path_cookie, "No path cookie is nil") assert_equal("no_path", no_path_cookie.value) - assert_equal("/many_cookies_as_string", no_path_cookie.path) + assert_equal("/", no_path_cookie.path) assert_equal(true, Time.now < no_path_cookie.expires) end @@ -88,7 +88,7 @@ class CookiesMechTest < Test::Unit::TestCase no_path_cookie = @agent.cookies.find { |k| k.name == "no_path" } assert_not_nil(no_path_cookie, "No path cookie is nil") assert_equal("no_path", no_path_cookie.value) - assert_equal("/many_cookies", no_path_cookie.path) + assert_equal("/", no_path_cookie.path) assert_equal(true, Time.now < no_path_cookie.expires) end test/tc_cookies.rb @@ -19,6 +19,14 @@ class MechErrorsTest < Test::Unit::TestCase } end + def test_non_exist + begin + page = @agent.get("http://localhost:#{PORT}/bad_form_test.html") + rescue RuntimeError => ex + assert_equal("404", ex.inspect) + end + end + def test_too_many_radio page = @agent.get("http://localhost:#{PORT}/form_test.html") form = page.forms.name('post_form1').first test/tc_errors.rb @@ -10,11 +10,11 @@ class FormNoInputNameTest < Test::Unit::TestCase def setup @agent = WWW::Mechanize.new + @page = @agent.get('http://localhost/form_no_input_name.html') end def test_no_input_name - page = @agent.get("http://localhost:#{PORT}/form_no_input_name.html") - form = page.forms.first + form = @page.forms.first assert_equal(0, form.fields.length) assert_equal(0, form.radiobuttons.length) assert_equal(0, form.checkboxes.length) test/tc_form_no_inputname.rb @@ -39,6 +39,26 @@ class FormsMechTest < Test::Unit::TestCase assert_not_nil(page.links.text('first:Patterson').first) end + # Test calling submit on the form object + def test_submit_on_form + page = @agent.get("http://localhost:#{PORT}/form_multival.html") + form = page.forms.name('post_form').first + + assert_not_nil(form) + assert_equal(2, form.fields.name('first').length) + + form.fields.name('first')[0].value = 'Aaron' + form.fields.name('first')[1].value = 'Patterson' + + page = form.submit + + assert_not_nil(page) + + assert_equal(2, page.links.length) + assert_not_nil(page.links.text('first:Aaron').first) + assert_not_nil(page.links.text('first:Patterson').first) + end + # Test submitting form with two fields of the same name def test_get_multival page = @agent.get("http://localhost:#{PORT}/form_multival.html") @@ -436,7 +456,7 @@ class FormsMechTest < Test::Unit::TestCase page = @agent.submit(get_form, get_form.buttons.first) # Check that the submitted fields exist - assert_equal(5, page.links.size, "Not enough links") + assert_equal(3, page.links.size, "Not enough links") assert_not_nil( page.links.find { |l| l.text == "likes ham:on" }, "likes ham check box missing" @@ -449,14 +469,6 @@ class FormsMechTest < Test::Unit::TestCase page.links.find { |l| l.text == "gender:male" }, "gender field missing" ) - assert_not_nil( - page.links.find { |l| l.text == "great day:yes" }, - "great day field missing" - ) - assert_not_nil( - page.links.find { |l| l.text == "one:two" }, - "one field missing" - ) end def test_field_addition @@ -477,15 +489,50 @@ class FormsMechTest < Test::Unit::TestCase assert_equal('Aaron', form.first) end - def test_fields_as_hash + def test_add_field page = @agent.get("http://localhost:#{PORT}/form_multival.html") form = page.forms.name('post_form').first assert_not_nil(form) - assert_equal(2, form.fields.name('first').length) + number_of_fields = form.fields.length + + f = form.add_field!('intarweb') + assert_not_nil(f) + assert_equal(number_of_fields + 1, form.fields.length) + end + + def test_delete_field + page = @agent.get("http://localhost:#{PORT}/form_multival.html") + form = page.forms.name('post_form').first + + assert_not_nil(form) + number_of_fields = form.fields.length + + form.delete_field!('first') + assert_nil(form['first']) + assert_equal(number_of_fields - 2, form.fields.length) + end + + def test_has_field + page = @agent.get("http://localhost:#{PORT}/form_multival.html") + form = page.forms.name('post_form').first + + assert_not_nil(form) + assert_equal(false, form.has_field?('intarweb')) + f = form.add_field!('intarweb') + assert_not_nil(f) + assert_equal(true, form.has_field?('intarweb')) + end + + def test_field_error + @page = @agent.get('http://localhost/empty_form.html') + form = @page.forms.first + assert_raise(NoMethodError) { + form.foo = 'asdfasdf' + } - form['first'] = 'Aaron' - assert_equal('Aaron', form['first']) - assert_equal('Aaron', form.fields.name('first').first.value) + assert_raise(NoMethodError) { + form.foo + } end end test/tc_forms.rb @@ -12,21 +12,30 @@ class LinksMechTest < Test::Unit::TestCase @agent = WWW::Mechanize.new end + def test_base + page = @agent.get("http://google.com/tc_base_link.html") + page = page.links.first.click + assert @agent.visited?("http://localhost/index.html") + end + def test_find_meta page = @agent.get("http://localhost:#{PORT}/find_link.html") - assert_equal(2, page.meta.length) - assert_equal("http://www.drphil.com/", page.meta[0].href.downcase) - assert_equal("http://www.upcase.com/", page.meta[1].href.downcase) + assert_equal(3, page.meta.length) + assert_equal(%w{ + http://www.drphil.com/ + http://www.upcase.com/ + http://tenderlovemaking.com/ }.sort, + page.meta.map { |x| x.href.downcase }.sort) end def test_find_link page = @agent.get("http://localhost:#{PORT}/find_link.html") - assert_equal(15, page.links.length) + assert_equal(18, page.links.length) end def test_alt_text page = @agent.get("http://localhost:#{PORT}/alt_text.html") - assert_equal(4, page.links.length) + assert_equal(5, page.links.length) assert_equal(1, page.meta.length) assert_equal('', page.meta.first.text) @@ -34,6 +43,7 @@ class LinksMechTest < Test::Unit::TestCase assert_equal('', page.links.href('no_alt_text.html').first.text) assert_equal('no image', page.links.href('no_image.html').first.text) assert_equal('', page.links.href('no_text.html').first.text) + assert_equal('', page.links.href('nil_alt_text.html').first.text) end def test_click_link @@ -46,4 +56,51 @@ class LinksMechTest < Test::Unit::TestCase assert_equal("http://localhost:#{PORT}/form_test.html", @agent.history.last.uri.to_s) end + + def test_click_method + page = @agent.get("http://localhost:#{PORT}/frame_test.html") + link = page.links.text("Form Test") + assert_not_nil(link) + assert_equal('Form Test', link.text) + page = link.click + assert_equal("http://localhost:#{PORT}/form_test.html", + @agent.history.last.uri.to_s) + end + + def test_find_bold_link + page = @agent.get("http://localhost:#{PORT}/tc_links.html") + link = page.links.text(/Bold Dude/) + assert_equal(1, link.length) + assert_equal('Bold Dude', link.first.text) + + link = page.links.text('Aaron James Patterson') + assert_equal(1, link.length) + assert_equal('Aaron James Patterson', link.first.text) + + link = page.links.text('Aaron Patterson') + assert_equal(1, link.length) + assert_equal('Aaron Patterson', link.first.text) + + link = page.links.text('Ruby Rocks!') + assert_equal(1, link.length) + assert_equal('Ruby Rocks!', link.first.text) + end + + def test_link_with_encoded_space + page = @agent.get("http://localhost:#{PORT}/tc_links.html") + link = page.links.text('encoded space').first + page = @agent.click link + end + + def test_link_with_space + page = @agent.get("http://localhost:#{PORT}/tc_links.html") + link = page.links.text('not encoded space').first + page = @agent.click link + end + + def test_link_with_unusual_characters + page = @agent.get("http://localhost:#{PORT}/tc_links.html") + link = page.links.text('unusual characters').first + assert_nothing_raised { @agent.click link } + end end test/tc_links.rb @@ -13,6 +13,26 @@ class TestMechMethods < Test::Unit::TestCase @agent = WWW::Mechanize.new end + def test_weird_url + assert_nothing_raised { + @agent.get('http://localhost/?action=bing&bang=boom=1|a=|b=|c=') + } + assert_nothing_raised { + @agent.get('http://localhost/?a=b&#038;b=c&#038;c=d') + } + assert_nothing_raised { + @agent.get("http://localhost/?a=#{[0xd6].pack('U')}") + } + end + + def test_kcode_url + $KCODE = 'u' + page = @agent.get("http://localhost/?a=#{[0xd6].pack('U')}") + assert_not_nil(page) + assert_equal('http://localhost/?a=%D6', page.uri.to_s) + $KCODE = 'NONE' + end + def test_history 0.upto(25) do |i| assert_equal(i, @agent.history.size) @@ -24,6 +44,8 @@ class TestMechMethods < Test::Unit::TestCase @agent.history.last.uri.to_s) assert_equal("http://localhost:#{PORT}/", @agent.history[-2].uri.to_s) + assert_equal("http://localhost:#{PORT}/", + @agent.history[-2].uri.to_s) assert_equal(true, @agent.visited?("http://localhost:#{PORT}/")) assert_equal(true, @agent.visited?("/form_test.html")) @@ -32,6 +54,24 @@ class TestMechMethods < Test::Unit::TestCase end + def test_visited + @agent.get("http://localhost/content_type_test?ct=application/pdf") + assert_equal(true, + @agent.visited?("http://localhost/content_type_test?ct=application/pdf")) + assert_equal(false, + @agent.visited?("http://localhost/content_type_test")) + assert_equal(false, + @agent.visited?("http://localhost/content_type_test?ct=text/html")) + end + + def test_visited_after_redirect + @agent.get("http://localhost/response_code?code=302") + assert_equal("http://localhost/index.html", + @agent.current_page.uri.to_s) + assert_equal(true, + @agent.visited?('http://localhost/response_code?code=302')) + end + def test_max_history @agent.max_history = 10 0.upto(10) do |i| @@ -45,6 +85,23 @@ class TestMechMethods < Test::Unit::TestCase end end + def test_max_history_order + @agent.max_history = 2 + assert_equal(0, @agent.history.length) + + @agent.get('http://localhost/form_test.html') + assert_equal(1, @agent.history.length) + + @agent.get('http://localhost/empty_form.html') + assert_equal(2, @agent.history.length) + + @agent.get('http://localhost/tc_checkboxes.html') + assert_equal(2, @agent.history.length) + assert_equal('http://localhost/empty_form.html', @agent.history[0].uri.to_s) + assert_equal('http://localhost/tc_checkboxes.html', + @agent.history[1].uri.to_s) + end + def test_back_button 0.upto(5) do |i| assert_equal(i, @agent.history.size) test/tc_mech.rb @@ -14,7 +14,7 @@ class TestNoAttributes < Test::Unit::TestCase def test_parse_no_attributes assert_nothing_raised do - page = @agent.get("http://localhost:#{PORT}/tc_no_attributes.html") + page = @agent.get('http://localhost/tc_no_attributes.html') end end end test/tc_no_attributes.rb @@ -63,7 +63,7 @@ class PluggableParserTest < Test::Unit::TestCase @agent.pluggable_parser.html = Filter page = @agent.get("http://localhost:#{PORT}/find_link.html") assert_kind_of(Filter, page) - assert_equal(16, page.links.length) + assert_equal(19, page.links.length) assert_not_nil(page.links.text('Net::DAAP::Client').first) assert_equal(1, page.links.text('Net::DAAP::Client').length) end @@ -74,7 +74,7 @@ class PluggableParserTest < Test::Unit::TestCase assert_kind_of(Class, @agent.pluggable_parser['text/html']) assert_equal(Filter, @agent.pluggable_parser['text/html']) assert_kind_of(Filter, page) - assert_equal(16, page.links.length) + assert_equal(19, page.links.length) assert_not_nil(page.links.text('Net::DAAP::Client').first) assert_equal(1, page.links.text('Net::DAAP::Client').length) end test/tc_pluggable_parser.rb @@ -13,6 +13,10 @@ class ResponseCodeMechTest < Test::Unit::TestCase end def test_redirect + @agent.get("http://localhost:#{PORT}/response_code?code=300") + assert_equal("http://localhost:#{PORT}/index.html", + @agent.current_page.uri.to_s) + @agent.get("http://localhost:#{PORT}/response_code?code=301") assert_equal("http://localhost:#{PORT}/index.html", @agent.current_page.uri.to_s) @@ -20,6 +24,22 @@ class ResponseCodeMechTest < Test::Unit::TestCase @agent.get("http://localhost:#{PORT}/response_code?code=302") assert_equal("http://localhost:#{PORT}/index.html", @agent.current_page.uri.to_s) + + @agent.get("http://localhost:#{PORT}/response_code?code=303") + assert_equal("http://localhost:#{PORT}/index.html", + @agent.current_page.uri.to_s) + + @agent.get("http://localhost:#{PORT}/response_code?code=307") + assert_equal("http://localhost:#{PORT}/index.html", + @agent.current_page.uri.to_s) + end + + def test_do_not_follow_redirect + @agent.redirect_ok = false + + @agent.get("http://localhost:#{PORT}/response_code?code=302") + assert_equal("http://localhost:#{PORT}/response_code?code=302", + @agent.current_page.uri.to_s) end def test_error test/tc_response_code.rb @@ -22,4 +22,35 @@ class TestSaveFile < Test::Unit::TestCase FileUtils.rm("test.html") assert_equal(length.to_i, file_length) end + + def test_save_file_default + page = WWW::Mechanize::File.new( + URI.parse('http://localhost/test.html'), + {}, + "hello" + ) + page.save + assert(File.exists?('test.html')) + page.save + assert(File.exists?('test.html.1')) + page.save + assert(File.exists?('test.html.2')) + FileUtils.rm("test.html") + FileUtils.rm("test.html.1") + FileUtils.rm("test.html.2") + end + + def test_save_file_default_with_dots + page = WWW::Mechanize::File.new( + URI.parse('http://localhost/../test.html'), + {}, + "hello" + ) + page.save + assert(File.exists?('test.html')) + page.save + assert(File.exists?('test.html.1')) + FileUtils.rm("test.html") + FileUtils.rm("test.html.1") + end end test/tc_save_file.rb @@ -10,99 +10,94 @@ class UploadMechTest < Test::Unit::TestCase def setup @agent = WWW::Mechanize.new + @page = @agent.get("http://localhost:#{PORT}/file_upload.html") end def test_form_enctype - page = @agent.get("http://localhost:#{PORT}/file_upload.html") - assert_equal('multipart/form-data', page.forms[0].enctype) + assert_equal('multipart/form-data', @page.forms[0].enctype) - form = page.forms.first - form.file_uploads.first.file_name = "README" + form = @page.forms.first + form.file_uploads.first.file_name = "#{BASE_DIR}/test_all.rb" form.file_uploads.first.mime_type = "text/plain" form.file_uploads.first.file_data = "Hello World\n\n" - page = @agent.submit(form) + @page = @agent.submit(form) assert_match( - "Content-Disposition: form-data; name=\"userfile1\"; filename=\"README\"", - page.body + "Content-Disposition: form-data; name=\"userfile1\"; filename=\"test_all.rb\"", + @page.body ) assert_match( "Content-Disposition: form-data; name=\"name\"", - page.body + @page.body ) - assert_match('Content-Type: text/plain', page.body) - assert_match('Hello World', page.body) - assert_match('foo[aaron]', page.body) + assert_match('Content-Type: text/plain', @page.body) + assert_match('Hello World', @page.body) + assert_match('foo[aaron]', @page.body) end def test_form_multipart - page = @agent.get("http://localhost:#{PORT}/file_upload.html") - assert_equal('multipart/form-data', page.forms[1].enctype) + assert_equal('multipart/form-data', @page.forms[1].enctype) - form = page.forms[1] - form.file_uploads.first.file_name = "README" + form = @page.forms[1] + form.file_uploads.first.file_name = "#{BASE_DIR}/test_all.rb" form.file_uploads.first.mime_type = "text/plain" form.file_uploads.first.file_data = "Hello World\n\n" - page = @agent.submit(form) + @page = @agent.submit(form) assert_match( - "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"README\"", - page.body + "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"test_all.rb\"", + @page.body ) end def test_form_read_file - page = @agent.get("http://localhost:#{PORT}/file_upload.html") - assert_equal('multipart/form-data', page.forms[1].enctype) + assert_equal('multipart/form-data', @page.forms[1].enctype) - form = page.forms[1] - form.file_uploads.first.file_name = "README" + form = @page.forms[1] + form.file_uploads.first.file_name = "#{BASE_DIR}/test_all.rb" - page = @agent.submit(form) + @page = @agent.submit(form) - contents = File.open("README", 'rb') { |f| f.read } + contents = File.open("#{BASE_DIR}/test_all.rb", 'rb') { |f| f.read } assert_match( - "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"README\"", - page.body + "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"test_all.rb\"", + @page.body ) - assert_match(contents, page.body) + assert_match(contents, @page.body) end def test_form_io_obj - page = @agent.get("http://localhost:#{PORT}/file_upload.html") - assert_equal('multipart/form-data', page.forms[1].enctype) + assert_equal('multipart/form-data', @page.forms[1].enctype) - form = page.forms[1] - form.file_uploads.first.file_name = "README" - form.file_uploads.first.file_data = File.open("README", 'rb') + form = @page.forms[1] + form.file_uploads.first.file_name = "#{BASE_DIR}/test_all.rb" + form.file_uploads.first.file_data = File.open("#{BASE_DIR}/test_all.rb", 'rb') - page = @agent.submit(form) + @page = @agent.submit(form) - contents = File.open("README", 'rb') { |f| f.read } + contents = File.open("#{BASE_DIR}/test_all.rb", 'rb') { |f| f.read } assert_match( - "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"README\"", - page.body + "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"test_all.rb\"", + @page.body ) - assert_match(contents, page.body) + assert_match(contents, @page.body) end def test_submit_no_file - page = @agent.get("http://localhost:#{PORT}/file_upload.html") - form = page.forms.first + form = @page.forms.first form.fields.name('name').value = 'Aaron' - page = @agent.submit(form) - assert_match('Aaron', page.body) + @page = @agent.submit(form) + assert_match('Aaron', @page.body) assert_match( "Content-Disposition: form-data; name=\"userfile1\"; filename=\"\"", - page.body + @page.body ) end def test_no_value - page = @agent.get("http://localhost:#{PORT}/file_upload.html") - form = page.form('value_test') + form = @page.form('value_test') assert_nil(form.file_uploads.first.value) assert_nil(form.file_uploads.first.file_name) end test/tc_upload.rb @@ -9,7 +9,7 @@ class Area attr_reader :name def initialize(node) - @name = node.attributes['name'] + @name = node['name'] end end test/tc_watches.rb @@ -1,5 +1,121 @@ +require 'net/http' +require 'test_servlets' +require 'webrick/httputils' + +BASE_DIR = File.dirname(__FILE__) + +class Net::HTTP + #def self.new(*args) + # obj = allocate + # return obj + #end + + alias :old_do_start :do_start + + def do_start + @started = true + end + + SERVLETS = { + '/gzip' => GzipServlet, + '/form_post' => FormTest, + '/basic_auth' => BasicAuthServlet, + '/form post' => FormTest, + '/response_code' => ResponseCodeTest, + '/bad_content_type' => BadContentTypeTest, + '/content_type_test' => ContentTypeTest, + '/referer' => RefererServlet, + '/file_upload' => FileUploadTest, + '/one_cookie' => OneCookieTest, + '/one_cookie_no_space' => OneCookieNoSpacesTest, + '/many_cookies' => ManyCookiesTest, + '/many_cookies_as_string' => ManyCookiesAsStringTest, + '/send_cookies' => SendCookiesTest, + '/if_modified_since' => ModifiedSinceServlet, + '/http_headers' => HeaderServlet, + } + + PAGE_CACHE = {} + + alias :old_request :request + + def request(request, *data, &block) + url = URI.parse(request.path) + path = URI.unescape(url.path) + + path = '/index.html' if path == '/' + + res = Response.new + request.query = WEBrick::HTTPUtils.parse_query(url.query) + request.cookies = WEBrick::Cookie.parse(request['Cookie']) + if SERVLETS[path] + if request.method == "POST" + if request['Content-Type'] =~ /^multipart\/form-data/ + request.body = data.first + else + request.query = WEBrick::HTTPUtils.parse_query(data.first) + end + end + SERVLETS[path].new({}).send("do_#{request.method}", request, res) + else + filename = "htdocs#{path.gsub(/[^\/\\.\w_\s]/, '_')}" + unless PAGE_CACHE[filename] + File.open("#{BASE_DIR}/#{filename}", 'rb') { |file| + PAGE_CACHE[filename] = file.read + } + end + res.body = PAGE_CACHE[filename] + end + + res['Content-Type'] ||= 'text/html' + res['Content-Length'] ||= res.body.length.to_s + res.code ||= "200" + + res.cookies.each do |cookie| + res.add_field('Set-Cookie', cookie.to_s) + end + yield res if block_given? + res + end +end + +class Net::HTTPRequest + attr_accessor :query, :body, :cookies, :user +end + +class Response + include Net::HTTPHeader + + attr_reader :code + attr_accessor :body, :query, :cookies + + def code=(c) + @code = c.to_s + end + + alias :status :code + alias :status= :code= + + def initialize + @header = {} + @body = '' + @code = nil + @query = nil + @cookies = [] + end + + def read_body + yield body + end +end + + module TestMethods PORT = 2000 PROXYPORT = 2001 SSLPORT = 2002 + + def html_response + { 'content-type' => 'text/html' } + end end test/test_includes.rb CHANGELOG EXAMPLES GUIDE LICENSE NOTES README lib/mechanize/hpricot.rb lib/mechanize/mech_version.rb test/README test/proxy.rb test/server.rb test/servlets.rb test/ts_mech.rb 7af819fa236093fe989f4a98272f970c2e61a1fb aaronp aaronp@f1cf478b-080f-0410-abad-959bfeec9ea8 http://github.com/github/mechanize/commit/63d4e1539fdd02e128b4ace7d346806d355deaa2 63d4e1539fdd02e128b4ace7d346806d355deaa2 2007-12-04T20:17:00-08:00 2007-12-04T20:17:00-08:00 merging REL-0.9.0 -> trunk 308:HEAD git-svn-id: svn+ssh://rubyforge.org/var/svn/mechanize/trunk@451 f1cf478b-080f-0410-abad-959bfeec9ea8 8a49a5128e4a07f214d804ab73cc601da262fcd7 aaronp aaronp@f1cf478b-080f-0410-abad-959bfeec9ea8