Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

1574 lines (1114 sloc) 42.759 kb
# coding: utf-8
require 'mechanize/test_case'
class TestMechanizeHttpAgent < Mechanize::TestCase
def setup
super
@agent = @mech.agent
@uri = URI.parse 'http://example/'
@req = Net::HTTP::Get.new '/'
@res = Net::HTTPOK.allocate
@res.instance_variable_set :@code, 200
@res.instance_variable_set :@header, {}
@headers = if RUBY_VERSION > '1.9' then
%w[accept user-agent]
else
%w[accept]
end
end
def auth_realm uri, scheme, type
base_uri = uri + '/'
realm = Mechanize::HTTP::AuthRealm.new scheme, base_uri, 'r'
@agent.authenticate_methods[base_uri][type] << realm
realm
end
def test_auto_io
Tempfile.open 'input' do |input_io|
input_io.binmode
input_io.write '12345'
input_io.rewind
out_io = @agent.auto_io __name__, 1024, input_io
assert_equal '12345', out_io.string
assert_equal Encoding::BINARY, out_io.string.encoding if
Object.const_defined? :Encoding
end
end
def test_auto_io_chunk
Tempfile.open 'input' do |input_io|
chunks = []
input_io.binmode
input_io.write '12345'
input_io.rewind
@agent.auto_io __name__, 1, input_io do |chunk|
chunks << chunk
end
assert_equal %w[1 2 3 4 5], chunks
end
end
def test_auto_io_tempfile
@agent.max_file_buffer = 3
Tempfile.open 'input' do |input_io|
input_io.binmode
input_io.write '12345'
input_io.rewind
out_io = @agent.auto_io __name__, 1, input_io
result = out_io.read
assert_equal '12345', result
assert_equal Encoding::BINARY, result.encoding if
Object.const_defined? :Encoding
end
end
def test_auto_io_yield
Tempfile.open 'input' do |input_io|
input_io.binmode
input_io.write '12345'
input_io.rewind
out_io = @agent.auto_io __name__, 1024, input_io do |chunk|
"x#{chunk}"
end
assert_equal 'x12345', out_io.string
end
end
def test_certificate_equals
cert_path = File.expand_path '../data/server.crt', __FILE__
cert = OpenSSL::X509::Certificate.new File.read cert_path
@agent.certificate = cert
assert_equal cert.to_pem, @agent.certificate.to_pem
end
def test_certificate_equals_file
cert_path = File.expand_path '../data/server.crt', __FILE__
cert = OpenSSL::X509::Certificate.new File.read cert_path
@agent.certificate = cert_path
assert_equal cert.to_pem, @agent.certificate.to_pem
end
def test_connection_for_file
uri = URI.parse 'file:///nonexistent'
conn = @agent.connection_for uri
assert_equal Mechanize::FileConnection.new, conn
end
def test_connection_for_http
conn = @agent.connection_for @uri
assert_equal @agent.http, conn
end
def test_disable_keep_alive
@agent.disable_keep_alive @req
refute @req['connection']
end
def test_disable_keep_alive_no
@agent.keep_alive = false
@agent.disable_keep_alive @req
assert_equal 'close', @req['connection']
end
def test_enable_gzip
@agent.enable_gzip @req
assert_equal 'gzip,deflate,identity', @req['accept-encoding']
end
def test_enable_gzip_no
@agent.gzip_enabled = false
@agent.enable_gzip @req
assert_equal 'identity', @req['accept-encoding']
end
def test_fetch_file_nonexistent
in_tmpdir do
nonexistent = File.join Dir.pwd, 'nonexistent'
uri = URI.parse "file://#{nonexistent}"
e = assert_raises Mechanize::ResponseCodeError do
@agent.fetch uri
end
assert_match "404 => Net::HTTPNotFound for #{uri}", e.message
end
end
def test_fetch_file_plus
Tempfile.open '++plus++' do |io|
content = 'plusses +++'
io.write content
io.rewind
uri = URI.parse "file://#{Mechanize::Util.uri_escape io.path}"
page = @agent.fetch uri
assert_equal content, page.body
assert_kind_of Mechanize::File, page
end
end
def test_fetch_file_space
foo = File.expand_path("../htdocs/dir with spaces/foo.html", __FILE__)
uri = URI.parse "file://#{Mechanize::Util.uri_escape foo}"
page = @agent.fetch uri
assert_equal File.read(foo), page.body
assert_kind_of Mechanize::Page, page
end
def test_fetch_head_gzip
uri = @uri + '/gzip?file=index.html'
page = @agent.fetch uri, :head
assert_kind_of Mechanize::Page, page
end
def test_fetch_hooks
@agent.pre_connect_hooks << proc do |agent, request|
assert_equal '/index.html', request.path
assert_equal @agent, agent
end
@agent.post_connect_hooks << proc do |agent, uri, response, body|
assert_equal @agent, agent
assert_equal URI('http://example/index.html'), uri
assert_equal '200', response.code
assert_kind_of String, body
end
@agent.fetch URI 'http://example/index.html'
end
def test_fetch_ignore_bad_chunking
@agent.ignore_bad_chunking = true
file = @agent.fetch 'http://example/bad_chunking'
assert_equal '0123456789', file.content
end
def test_fetch_post_connect_hook
response = nil
@agent.post_connect_hooks << lambda { |_, _, res, _| response = res }
@agent.fetch 'http://localhost/'
assert response
end
def test_fetch_redirect_header
page = @agent.fetch('http://example/redirect', :get,
'X-Location' => '/http_headers',
'Range' => 'bytes=0-99999')
assert_match 'range|bytes=0-999', page.body
end
def test_fetch_server_error
e = assert_raises Mechanize::ResponseCodeError do
@mech.get 'http://localhost/response_code?code=500'
end
assert_equal '500', e.response_code
end
def test_get_meta_refresh_header_follow_self
@agent.follow_meta_refresh = true
@agent.follow_meta_refresh_self = true
page = Mechanize::Page.new(@uri, nil, '', 200, @mech)
@res.instance_variable_set :@header, 'refresh' => ['0']
refresh = @agent.get_meta_refresh @res, @uri, page
assert_equal [0.0, URI('http://example/')], refresh
end
def test_get_meta_refresh_header_no_follow
page = Mechanize::Page.new(@uri, nil, '', 200, @mech)
@res.instance_variable_set :@header, 'refresh' => ['0']
refresh = @agent.get_meta_refresh @res, @uri, page
assert_nil refresh
end
def test_get_meta_refresh_header_no_follow_self
@agent.follow_meta_refresh = true
page = Mechanize::Page.new(@uri, nil, '', 200, @mech)
@res.instance_variable_set :@header, 'refresh' => ['0']
refresh = @agent.get_meta_refresh @res, @uri, page
assert_nil refresh
end
def test_get_meta_refresh_meta_follow_self
@agent.follow_meta_refresh = true
@agent.follow_meta_refresh_self = true
body = <<-BODY
<title></title>
<meta http-equiv="refresh" content="0">
BODY
page = Mechanize::Page.new(@uri, nil, body, 200, @mech)
refresh = @agent.get_meta_refresh @res, @uri, page
assert_equal [0, nil], refresh
end
def test_get_meta_refresh_meta_no_follow
body = <<-BODY
<title></title>
<meta http-equiv="refresh" content="0">
BODY
page = Mechanize::Page.new(@uri, nil, body, 200, @mech)
refresh = @agent.get_meta_refresh @res, @uri, page
assert_nil refresh
end
def test_get_meta_refresh_meta_no_follow_self
@agent.follow_meta_refresh = true
body = <<-BODY
<title></title>
<meta http-equiv="refresh" content="0">
BODY
page = Mechanize::Page.new(@uri, nil, body, 200, @mech)
refresh = @agent.get_meta_refresh @res, @uri, page
assert_nil refresh
end
def test_get_robots
robotstxt = @agent.get_robots 'http://localhost/robots.txt'
refute_equal '', robotstxt
robotstxt = @agent.get_robots 'http://localhost/response_code?code=404'
assert_equal '', robotstxt
end
def test_hook_content_encoding_response
@mech.content_encoding_hooks << lambda{|agent, uri, response, response_body_io|
response['content-encoding'] = 'gzip' if response['content-encoding'] == 'agzip'}
@res.instance_variable_set :@header, 'content-encoding' => %w[agzip]
body_io = StringIO.new 'part'
@agent.hook_content_encoding @res, @uri, body_io
assert_equal 'gzip', @res['content-encoding']
end
def test_http_request_file
uri = URI.parse 'file:///nonexistent'
request = @agent.http_request uri, :get
assert_kind_of Mechanize::FileRequest, request
assert_equal '/nonexistent', request.path
end
def test_http_request_get
request = @agent.http_request @uri, :get
assert_kind_of Net::HTTP::Get, request
assert_equal '/', request.path
end
def test_http_request_post
request = @agent.http_request @uri, :post
assert_kind_of Net::HTTP::Post, request
assert_equal '/', request.path
end
def test_idle_timeout_equals
@agent.idle_timeout = 1
assert_equal 1, @agent.http.idle_timeout
end
def test_inflate
body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01\xB8"
result = @agent.inflate body_io
assert_equal 'part', result.read
end
def test_post_connect
@agent.post_connect_hooks << proc { |agent, uri, response, body|
assert_equal @agent, agent
assert_equal @res, response
assert_equal 'body', body
throw :called
}
io = StringIO.new 'body'
assert_throws :called do
@agent.post_connect @uri, @res, io
end
assert_equal 0, io.pos
end
def test_pre_connect
@agent.pre_connect_hooks << proc { |agent, request|
assert_equal @agent, agent
assert_equal @req, request
throw :called
}
assert_throws :called do
@agent.pre_connect @req
end
end
def test_request_add_headers
@agent.request_add_headers @req, 'Content-Length' => 300
assert_equal '300', @req['content-length']
end
def test_request_add_headers_etag
@agent.request_add_headers @req, :etag => '300'
assert_equal '300', @req['etag']
end
def test_request_add_headers_if_modified_since
@agent.request_add_headers @req, :if_modified_since => 'some_date'
assert_equal 'some_date', @req['if-modified-since']
end
def test_request_add_headers_none
@agent.request_add_headers @req
assert_equal @headers, @req.to_hash.keys.sort
end
def test_request_add_headers_request_headers
@agent.request_headers['X-Foo'] = 'bar'
@agent.request_add_headers @req
assert_equal @headers + %w[x-foo], @req.to_hash.keys.sort
end
def test_request_add_headers_symbol
e = assert_raises ArgumentError do
@agent.request_add_headers @req, :content_length => 300
end
assert_equal 'unknown header symbol content_length', e.message
end
def test_request_auth_basic
@agent.add_auth @uri, 'user', 'password'
auth_realm @uri, 'Basic', :basic
@agent.request_auth @req, @uri
assert_match %r%^Basic %, @req['Authorization']
end
def test_request_auth_digest
@agent.add_auth @uri, 'user', 'password'
realm = auth_realm @uri, 'Digest', :digest
@agent.digest_challenges[realm] = 'Digest realm=r, qop="auth"'
@agent.request_auth @req, @uri
assert_match %r%^Digest %, @req['Authorization']
assert_match %r%qop=auth%, @req['Authorization']
end
def test_request_auth_iis_digest
@agent.add_auth @uri, 'user', 'password'
realm = auth_realm @uri, 'Digest', :digest
@agent.digest_challenges[realm] = 'Digest realm=r, qop="auth"'
@agent.request_auth @req, @uri
assert_match %r%^Digest %, @req['Authorization']
assert_match %r%qop=auth%, @req['Authorization']
end
def test_request_auth_none
@agent.request_auth @req, @uri
assert_nil @req['Authorization']
end
def test_request_cookies
uri = URI.parse 'http://host.example.com'
Mechanize::Cookie.parse uri, 'hello=world domain=.example.com' do |cookie|
@agent.cookie_jar.add uri, cookie
end
@agent.request_cookies @req, uri
assert_equal 'hello=world domain=.example.com', @req['Cookie']
end
def test_request_cookies_many
uri = URI.parse 'http://host.example.com'
cookie_str = 'a=b domain=.example.com, c=d domain=.example.com'
Mechanize::Cookie.parse uri, cookie_str do |cookie|
@agent.cookie_jar.add uri, cookie
end
@agent.request_cookies @req, uri
expected = cookie_str.sub ', ', '; '
assert_equal expected, @req['Cookie']
end
def test_request_cookies_none
@agent.request_cookies @req, @uri
assert_nil @req['Cookie']
end
def test_request_cookies_wrong_domain
uri = URI.parse 'http://host.example.com'
Mechanize::Cookie.parse uri, 'hello=world domain=.example.com' do |cookie|
@agent.cookie_jar.add uri, cookie
end
@agent.request_cookies @req, @uri
assert_nil @req['Cookie']
end
def test_request_host
@agent.request_host @req, @uri
assert_equal 'example', @req['host']
end
def test_request_host_nonstandard
@uri.port = 81
@agent.request_host @req, @uri
assert_equal 'example:81', @req['host']
end
def test_request_language_charset
@agent.request_language_charset @req
assert_equal 'en-us,en;q=0.5', @req['accept-language']
assert_equal 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', @req['accept-charset']
end
def test_request_referer
referer = URI.parse 'http://old.example'
@agent.request_referer @req, @uri, referer
assert_equal 'http://old.example', @req['referer']
end
def test_request_referer_https
uri = URI.parse 'https://example'
referer = URI.parse 'https://old.example'
@agent.request_referer @req, uri, referer
assert_equal 'https://old.example', @req['referer']
end
def test_request_referer_https_downgrade
referer = URI.parse 'https://old.example'
@agent.request_referer @req, @uri, referer
assert_nil @req['referer']
end
def test_request_referer_https_downgrade_case
uri = URI.parse 'http://example'
referer = URI.parse 'httpS://old.example'
@agent.request_referer @req, uri, referer
assert_nil @req['referer']
end
def test_request_referer_https_upgrade
uri = URI.parse 'https://example'
referer = URI.parse 'http://old.example'
@agent.request_referer @req, uri, referer
assert_equal 'http://old.example', @req['referer']
end
def test_request_referer_none
@agent.request_referer @req, @uri, nil
assert_nil @req['referer']
end
def test_request_referer_strip
uri = URI.parse 'http://example.com/index.html'
host_path = "old.example/page.html?q=x"
referer = "http://#{host_path}"
[
"",
"@",
"user1@",
":@",
"user1:@",
":password1@",
"user1:password1@",
].each { |userinfo|
['', '#frag'].each { |frag|
url = URI.parse "http://#{userinfo}#{host_path}#{frag}"
@agent.request_referer @req, uri, url
assert_equal referer, @req['referer'], url
}
}
end
def test_request_user_agent
@agent.request_user_agent @req
assert_match %r%^Mechanize/#{Mechanize::VERSION}%, @req['user-agent']
ruby_version = if RUBY_PATCHLEVEL >= 0 then
"#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}"
else
"#{RUBY_VERSION}dev#{RUBY_REVISION}"
end
assert_match %r%Ruby/#{ruby_version}%, @req['user-agent']
end
def test_resolve_bad_uri
e = assert_raises ArgumentError do
@agent.resolve 'google'
end
assert_equal 'absolute URL needed (not google)', e.message
end
def test_resolve_utf8
uri = 'http://example?q=ü'
resolved = @agent.resolve uri
assert_equal '/?q=%C3%BC', resolved.request_uri
end
def test_resolve_parameters_body
input_params = { :q => 'hello' }
uri, params = @agent.resolve_parameters @uri, :post, input_params
assert_equal 'http://example/', uri.to_s
assert_equal input_params, params
end
def test_resolve_parameters_query
uri, params = @agent.resolve_parameters @uri, :get, :q => 'hello'
assert_equal 'http://example/?q=hello', uri.to_s
assert_nil params
end
def test_resolve_parameters_query_append
input_params = { :q => 'hello' }
@uri.query = 'a=b'
uri, params = @agent.resolve_parameters @uri, :get, input_params
assert_equal 'http://example/?a=b&q=hello', uri.to_s
assert_nil params
end
def test_response_authenticate
@agent.add_auth @uri, 'user', 'password'
@res.instance_variable_set :@header, 'www-authenticate' => ['Basic realm=r']
@agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
base_uri = @uri + '/'
realm = Mechanize::HTTP::AuthRealm.new 'Basic', base_uri, 'r'
assert_equal [realm], @agent.authenticate_methods[base_uri][:basic]
end
def test_response_authenticate_digest
@agent.add_auth @uri, 'user', 'password'
@res.instance_variable_set(:@header,
'www-authenticate' => ['Digest realm=r'])
@agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
base_uri = @uri + '/'
realm = Mechanize::HTTP::AuthRealm.new 'Digest', base_uri, 'r'
assert_equal [realm], @agent.authenticate_methods[base_uri][:digest]
challenge = Mechanize::HTTP::AuthChallenge.new('Digest',
{ 'realm' => 'r' },
'Digest realm=r')
assert_equal challenge, @agent.digest_challenges[realm]
end
def test_response_authenticate_digest_iis
@agent.add_auth @uri, 'user', 'password'
@res.instance_variable_set(:@header,
'www-authenticate' => ['Digest realm=r'],
'server' => ['Microsoft-IIS'])
@agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
base_uri = @uri + '/'
realm = Mechanize::HTTP::AuthRealm.new 'Digest', base_uri, 'r'
assert_equal [realm], @agent.authenticate_methods[base_uri][:iis_digest]
end
def test_response_authenticate_multiple
@agent.add_auth @uri, 'user', 'password'
@res.instance_variable_set(:@header,
'www-authenticate' =>
['Basic realm=r, Digest realm=r'])
@agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
base_uri = @uri + '/'
realm = Mechanize::HTTP::AuthRealm.new 'Digest', base_uri, 'r'
assert_equal [realm], @agent.authenticate_methods[base_uri][:digest]
assert_empty @agent.authenticate_methods[base_uri][:basic]
end
def test_response_authenticate_no_credentials
@res.instance_variable_set :@header, 'www-authenticate' => ['Basic realm=r']
e = assert_raises Mechanize::UnauthorizedError do
@agent.response_authenticate @res, fake_page, @uri, @req, {}, nil, nil
end
assert_match 'no credentials', e.message
assert_match 'available realms: r', e.message
end
def test_response_authenticate_no_www_authenticate
@agent.add_auth @uri, 'user', 'password'
denied_uri = URI('http://example/denied')
denied = page denied_uri, 'text/html', '', 401
e = assert_raises Mechanize::UnauthorizedError do
@agent.response_authenticate @res, denied, @uri, @req, {}, nil, nil
end
assert_equal "401 => Net::HTTPUnauthorized for #{denied_uri} -- " \
"WWW-Authenticate header missing in response",
e.message
end
def test_response_authenticate_ntlm
@uri += '/ntlm'
@agent.add_auth @uri, 'user', 'password'
@res.instance_variable_set(:@header,
'www-authenticate' => ['Negotiate, NTLM'])
page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
assert_equal 'ok', page.body # lame test
end
def test_response_authenticate_unknown
@agent.add_auth @uri, 'user', 'password'
page = Mechanize::File.new nil, nil, nil, 401
@res.instance_variable_set(:@header,
'www-authenticate' => ['Unknown realm=r'])
assert_raises Mechanize::UnauthorizedError do
@agent.response_authenticate @res, page, @uri, @req, nil, nil, nil
end
end
def test_response_content_encoding_7_bit
@res.instance_variable_set :@header, 'content-encoding' => %w[7bit]
body = @agent.response_content_encoding @res, StringIO.new('part')
assert_equal 'part', body.read
end
def test_response_content_encoding_deflate
@res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01\xB8"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
assert body_io.closed?
end
def test_response_content_encoding_deflate_chunked
@res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01\xB8"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
end
def test_response_content_encoding_deflate_corrupt
@res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01" # missing 1 byte
e = assert_raises Mechanize::Error do
@agent.response_content_encoding @res, body_io
end
assert_match %r%error handling content-encoding deflate:%, e.message
assert_match %r%Zlib%, e.message
assert body_io.closed?
end
def test_response_content_encoding_deflate_empty
@res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
body = @agent.response_content_encoding @res, StringIO.new
assert_equal '', body.read
end
# IIS/6.0 ASP.NET/2.0.50727 does not wrap deflate with zlib, WTF?
def test_response_content_encoding_deflate_no_zlib
@res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
body = @agent.response_content_encoding @res, StringIO.new("+H,*\001\000")
assert_equal 'part', body.read
end
def test_response_content_encoding_gzip
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000\000"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
assert body_io.closed?
end
def test_response_content_encoding_gzip_chunked
def @res.content_length() nil end
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000\000"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
end
def test_response_content_encoding_gzip_corrupt
log = StringIO.new
logger = Logger.new log
@agent.context.log = logger
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001"
e = assert_raises Mechanize::Error do
@agent.response_content_encoding @res, body_io
end
assert_match %r%error handling content-encoding gzip:%, e.message
assert_match %r%Zlib%, e.message
assert_match %r%unable to gunzip response: unexpected end of file%,
log.string
assert_match %r%unable to inflate response: buffer error%,
log.string
assert body_io.closed?
end
def test_response_content_encoding_gzip_checksum_corrupt_crc
log = StringIO.new
logger = Logger.new log
@agent.context.log = logger
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017J\004\000\000\000"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
assert body_io.closed?
assert_match %r%invalid compressed data -- crc error%, log.string
end
def test_response_content_encoding_gzip_checksum_corrupt_length
log = StringIO.new
logger = Logger.new log
@agent.context.log = logger
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\005\000\000\000"
body = @agent.response_content_encoding @res, body_io
assert body_io.closed?
assert_match %r%invalid compressed data -- length error%, log.string
end
def test_response_content_encoding_gzip_checksum_truncated
log = StringIO.new
logger = Logger.new log
@agent.context.log = logger
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000"
body = @agent.response_content_encoding @res, body_io
assert body_io.closed?
assert_match %r%unable to gunzip response: footer is not found%, log.string
end
def test_response_content_encoding_gzip_empty
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body = @agent.response_content_encoding @res, StringIO.new
assert_equal '', body.read
end
def test_response_content_encoding_gzip_encoding_bad
@res.instance_variable_set(:@header,
'content-encoding' => %w[gzip],
'content-type' => 'text/html; charset=UTF-8')
# "test\xB2"
body_io = StringIO.new \
"\037\213\b\000*+\314N\000\003+I-.\331\004\000x\016\003\376\005\000\000\000"
body = @agent.response_content_encoding @res, body_io
expected = "test\xB2"
expected.force_encoding Encoding::BINARY if have_encoding?
content = body.read
assert_equal expected, content
assert_equal Encoding::BINARY, content.encoding if have_encoding?
end
def test_response_content_encoding_gzip_no_footer
@res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
assert body_io.closed?
end
def test_response_content_encoding_none
@res.instance_variable_set :@header, 'content-encoding' => %w[none]
body = @agent.response_content_encoding @res, StringIO.new('part')
assert_equal 'part', body.read
end
def test_response_content_encoding_tempfile_7_bit
body_io = tempfile 'part'
@res.instance_variable_set :@header, 'content-encoding' => %w[7bit]
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
refute body_io.closed?
ensure
begin
body_io.close! unless body_io.closed?
rescue IOError
# HACK for ruby 1.8
end
end
def test_response_content_encoding_tempfile_gzip
body_io = tempfile "x\x9C+H,*\x01\x00\x04?\x01\xB8"
@res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
assert body_io.closed?
ensure
body_io.close! unless body_io.closed?
end
def test_response_content_encoding_unknown
@res.instance_variable_set :@header, 'content-encoding' => %w[unknown]
body = StringIO.new 'part'
e = assert_raises Mechanize::Error do
@agent.response_content_encoding @res, body
end
assert_equal 'unsupported content-encoding: unknown', e.message
end
def test_response_content_encoding_x_gzip
@res.instance_variable_set :@header, 'content-encoding' => %w[x-gzip]
body_io = StringIO.new \
"\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000\000"
body = @agent.response_content_encoding @res, body_io
assert_equal 'part', body.read
end
def test_response_cookies
uri = URI.parse 'http://host.example.com'
cookie_str = 'a=b domain=.example.com'
@res.instance_variable_set(:@header,
'set-cookie' => [cookie_str],
'content-type' => %w[text/html])
page = Mechanize::Page.new uri, @res, '', 200, @mech
@agent.response_cookies @res, uri, page
assert_equal ['a=b domain=.example.com'],
@agent.cookie_jar.cookies(uri).map { |c| c.to_s }
end
def test_response_cookies_many
uri = URI.parse 'http://host.example.com'
cookie1 = 'a=b domain=.example.com'
cookie2 = 'c=d domain=.example.com'
@res.instance_variable_set(:@header,
'set-cookie' => [cookie1, cookie2],
'content-type' => %w[text/html])
page = Mechanize::Page.new uri, @res, '', 200, @mech
@agent.response_cookies @res, uri, page
assert_equal ['a=b domain=.example.com', 'c=d domain=.example.com'],
@agent.cookie_jar.cookies(uri).map { |c| c.to_s }
end
def test_response_cookies_meta
uri = URI.parse 'http://host.example.com'
cookie_str = 'a=b domain=.example.com'
body = <<-BODY
<head>
<meta http-equiv="Set-Cookie" content="#{cookie_str}">
</head>"
BODY
@res.instance_variable_set(:@header,
'content-type' => %w[text/html])
page = Mechanize::Page.new uri, @res, body, 200, @mech
@agent.response_cookies @res, uri, page
assert_equal ['a=b domain=.example.com'],
@agent.cookie_jar.cookies(uri).map { |c| c.to_s }
end
def test_response_follow_meta_refresh
uri = URI.parse 'http://example/#id+1'
body = <<-BODY
<title></title>
<meta http-equiv="refresh" content="0">
BODY
page = Mechanize::Page.new(uri, nil, body, 200, @mech)
@agent.follow_meta_refresh = true
@agent.follow_meta_refresh_self = true
page = @agent.response_follow_meta_refresh @res, uri, page, 0
assert_equal uri, page.uri
end
def test_response_follow_meta_refresh_limit
uri = URI.parse 'http://example/#id+1'
body = <<-BODY
<title></title>
<meta http-equiv="refresh" content="0">
BODY
page = Mechanize::Page.new(uri, nil, body, 200, @mech)
@agent.follow_meta_refresh = true
@agent.follow_meta_refresh_self = true
assert_raises Mechanize::RedirectLimitReachedError do
@agent.response_follow_meta_refresh(@res, uri, page,
@agent.redirection_limit)
end
end
def test_response_parse
body = '<title>hi</title>'
@res.instance_variable_set :@header, 'content-type' => %w[text/html]
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal @mech, page.mech
end
def test_response_parse_content_type_case
body = '<title>hi</title>'
@res.instance_variable_set(:@header, 'content-type' => %w[text/HTML])
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal 'text/HTML', page.content_type
end
def test_response_parse_content_type_encoding
body = '<title>hi</title>'
@res.instance_variable_set(:@header,
'content-type' =>
%w[text/html;charset=ISO-8859-1])
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal @mech, page.mech
assert_equal 'ISO-8859-1', page.encoding
assert_equal 'ISO-8859-1', page.parser.encoding
end
def test_response_parse_content_type_encoding_broken_iso_8859_1
body = '<title>hi</title>'
@res.instance_variable_set(:@header,
'content-type' =>
%w[text/html; charset=ISO_8859-1])
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal 'ISO_8859-1', page.encoding
end
def test_response_parse_content_type_encoding_broken_utf_8
body = '<title>hi</title>'
@res.instance_variable_set(:@header,
'content-type' =>
%w[text/html; charset=UTF8])
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal 'UTF8', page.encoding
assert_equal 'UTF8', page.parser.encoding
end
def test_response_parse_content_type_encoding_garbage
body = '<title>hi</title>'
@res.instance_variable_set(:@header,
'content-type' =>
%w[text/html; charset=garbage_charset])
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal @mech, page.mech
end
def test_response_parse_content_type_encoding_semicolon
body = '<title>hi</title>'
@res.instance_variable_set(:@header,
'content-type' =>
%w[text/html;charset=UTF-8;])
page = @agent.response_parse @res, body, @uri
assert_instance_of Mechanize::Page, page
assert_equal 'UTF-8', page.encoding
end
def test_response_read
def @res.read_body() yield 'part' end
def @res.content_length() 4 end
io = @agent.response_read @res, @req, @uri
body = io.read
assert_equal 'part', body
assert_equal Encoding::BINARY, body.encoding if body.respond_to? :encoding
end
def test_response_read_chunked_no_trailer
@res['Transfer-Encoding'] = 'chunked'
def @res.content_length() end
def @res.read_body
yield 'a' * 10
raise EOFError
end
e = assert_raises Mechanize::ChunkedTerminationError do
@agent.response_read @res, @req, @uri
end
assert_equal 'aaaaaaaaaa', e.body_io.read
end
def test_response_read_content_length_head
req = Net::HTTP::Head.new '/'
def @res.content_length() end
def @res.read_body() end
io = @agent.response_read @res, req, @uri
assert_equal '', io.read
end
def test_response_read_content_length_mismatch
def @res.content_length() 5 end
def @res.read_body() yield 'part' end
e = assert_raises EOFError do
@agent.response_read @res, @req, @uri
end
assert_equal 'Content-Length (5) does not match response body length (4)',
e.message
end
def test_response_read_content_length_redirect
res = Net::HTTPFound.allocate
def res.content_length() 5 end
def res.code() 302 end
def res.read_body() yield 'part' end
res.instance_variable_set :@header, {}
io = @agent.response_read res, @req, @uri
assert_equal 'part', io.read
end
def test_response_read_error
def @res.read_body()
yield 'part'
raise Net::HTTP::Persistent::Error
end
e = assert_raises Mechanize::ResponseReadError do
@agent.response_read @res, @req, @uri
end
assert_equal @res, e.response
assert_equal 'part', e.body_io.read
assert_kind_of Net::HTTP::Persistent::Error, e.error
end
def test_response_read_file
Tempfile.open 'pi.txt' do |tempfile|
tempfile.write "π\n"
tempfile.flush
tempfile.rewind
uri = URI.parse "file://#{tempfile.path}"
req = Mechanize::FileRequest.new uri
res = Mechanize::FileResponse.new tempfile.path
io = @agent.response_read res, req, uri
expected = "π\n"
expected.force_encoding Encoding::BINARY if expected.respond_to? :encoding
body = io.read
assert_equal expected, body
assert_equal Encoding::BINARY, body.encoding if body.respond_to? :encoding
end
end
def test_response_read_large
@agent.max_file_buffer = 10240
def @res.read_body() yield 'a' * 10241 end
def @res.content_length() 10241 end
io = @agent.response_read @res, @req, @uri
assert_kind_of Tempfile, io
assert_equal 10241, io.stat.size
end
def test_response_read_large_chunked
@agent.max_file_buffer = 10240
def @res.read_body
11.times do yield 'a' * 1024 end
end
def @res.content_length() end
io = @agent.response_read @res, @req, @uri
assert_kind_of Tempfile, io
assert_equal 11264, io.stat.size
end
def test_response_read_no_body
req = Net::HTTP::Options.new '/'
def @res.content_length() end
def @res.read_body() end
io = @agent.response_read @res, req, @uri
assert_equal '', io.read
end
def test_response_read_unknown_code
res = Net::HTTPUnknownResponse.allocate
res.instance_variable_set :@code, 9999
res.instance_variable_set :@header, {}
def res.read_body() yield 'part' end
e = assert_raises Mechanize::ResponseCodeError do
@agent.response_read res, @req, @uri
end
assert_equal res, e.page
end
def test_response_redirect
@agent.redirect_ok = true
referer = page 'http://example/referer'
page = fake_page
page = @agent.response_redirect({ 'Location' => '/index.html' }, :get,
page, 0, {}, referer)
assert_equal URI('http://fake.example/index.html'), page.uri
assert_equal 'http://example/referer', requests.first['Referer']
end
def test_response_redirect_header
@agent.redirect_ok = true
referer = page 'http://example/referer'
headers = {
'Range' => 'bytes=0-9999',
'Content-Type' => 'application/x-www-form-urlencoded',
'Content-Length' => '9999',
}
page = fake_page
page = @agent.response_redirect({ 'Location' => '/http_headers' }, :get,
page, 0, headers, referer)
assert_equal URI('http://fake.example/http_headers'), page.uri
assert_match 'range|bytes=0-9999', page.body
refute_match 'content-type|application/x-www-form-urlencoded', page.body
refute_match 'content-length|9999', page.body
end
def test_response_redirect_malformed
@agent.redirect_ok = true
referer = page 'http://example/referer'
page = fake_page
page = @agent.response_redirect({ 'Location' => '/index.html?q=あ' }, :get,
page, 0, {}, referer)
assert_equal URI('http://fake.example/index.html?q=%E3%81%82'), page.uri
assert_equal 'http://example/referer', requests.first['Referer']
end
def test_response_redirect_limit
@agent.redirect_ok = true
referer = page 'http://example/referer'
assert_raises Mechanize::RedirectLimitReachedError do
@agent.response_redirect({ 'Location' => '/index.html' }, :get,
fake_page, @agent.redirection_limit, {}, referer)
end
end
def test_response_redirect_not_ok
@agent.redirect_ok = false
page = fake_page
page = @agent.response_redirect({ 'Location' => '/other' }, :get, page, 0,
{}, page)
assert_equal URI('http://fake.example'), page.uri
end
def test_response_redirect_permanent
@agent.redirect_ok = :permanent
response = Net::HTTPMovedPermanently.allocate
response.instance_variable_set :@header, { 'location' => %w[/index.html] }
page = fake_page
page = @agent.response_redirect response, :get, page, 0, {}, page
assert_equal URI('http://fake.example/index.html'), page.uri
end
def test_response_redirect_permanent_temporary
@agent.redirect_ok = :permanent
response = Net::HTTPMovedTemporarily.allocate
response.instance_variable_set :@header, { 'location' => %w[/index.html] }
page = fake_page
page = @agent.response_redirect response, :get, page, 0, {}, page
assert_equal URI('http://fake.example/'), page.uri
end
def test_retry_change_request_equals
refute @agent.http.retry_change_requests
@agent.retry_change_requests = true
assert @agent.http.retry_change_requests
end
def test_robots_allowed_eh
allowed = URI 'http://localhost/index.html'
disallowed = URI 'http://localhost/norobots.html'
assert @agent.robots_allowed? allowed
refute @agent.robots_allowed? disallowed
refute @agent.robots_disallowed? allowed
assert @agent.robots_disallowed? disallowed
end
def test_robots_allowed_eh_noindex
@agent.robots = true
noindex = URI 'http://localhost/noindex.html'
assert @agent.robots_allowed? noindex
assert_raises Mechanize::RobotsDisallowedError do
@agent.fetch noindex
end
end
def test_set_proxy
@agent.set_proxy 'www.example.com', 9001, 'joe', 'lol'
assert_equal @agent.proxy_uri.host, 'www.example.com'
assert_equal @agent.proxy_uri.port, 9001
assert_equal @agent.proxy_uri.user, 'joe'
assert_equal @agent.proxy_uri.password, 'lol'
end
def test_set_proxy_port_string
@agent.set_proxy 'www.example.com', '9001', 'joe', 'lol'
assert_equal @agent.proxy_uri.host, 'www.example.com'
assert_equal @agent.proxy_uri.port, 9001
assert_equal @agent.proxy_uri.user, 'joe'
assert_equal @agent.proxy_uri.password, 'lol'
end
def test_set_proxy_service_name
@agent.set_proxy 'www.example.com', 'http', 'joe', 'lol'
assert_equal @agent.proxy_uri.host, 'www.example.com'
assert_equal @agent.proxy_uri.port, 80
assert_equal @agent.proxy_uri.user, 'joe'
assert_equal @agent.proxy_uri.password, 'lol'
end
def test_set_proxy_service_name_bad
e = assert_raises ArgumentError do
@agent.set_proxy 'www.example.com', 'nonexistent service', 'joe', 'lol'
end
assert_equal 'invalid value for port: "nonexistent service"', e.message
end
def test_ssl
in_tmpdir do
store = OpenSSL::X509::Store.new
@agent.ca_file = '.'
@agent.cert_store = store
@agent.certificate = ssl_certificate
@agent.private_key = ssl_private_key
@agent.ssl_version = 'SSLv3' if RUBY_VERSION > '1.9'
@agent.verify_callback = proc { |ok, context| }
http = @agent.http
assert_equal '.', http.ca_file
assert_equal store, http.cert_store
assert_equal ssl_certificate, http.certificate
assert_equal ssl_private_key, http.private_key
assert_equal 'SSLv3', http.ssl_version if
RUBY_VERSION > '1.9'
assert_equal OpenSSL::SSL::VERIFY_PEER, http.verify_mode
assert http.verify_callback
end
end
def test_use_tempfile_eh
refute @agent.use_tempfile? nil
@agent.max_file_buffer = 1
refute @agent.use_tempfile? 0
assert @agent.use_tempfile? 1
@agent.max_file_buffer = nil
refute @agent.use_tempfile? 1
end
def test_verify_none_equals
@agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
http = @agent.http
assert_equal OpenSSL::SSL::VERIFY_NONE, http.verify_mode
end
end
Jump to Line
Something went wrong with that request. Please try again.