Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Added support for proxy authentication #81

Open
wants to merge 2 commits into from

1 participant

This page is out of date. Refresh to see the latest.
Showing with 34 additions and 8 deletions.
  1. +4 −0 lib/anemone/core.rb
  2. +30 −8 lib/anemone/http.rb
View
4 lib/anemone/core.rb
@@ -54,6 +54,10 @@ class Core
:proxy_host => nil,
# proxy server port number
:proxy_port => false,
+ # proxy server username
+ :proxy_user => nil,
+ # proxy server password
+ :proxy_pass => nil,
# HTTP read timeout in seconds
:read_timeout => nil
}
View
38 lib/anemone/http.rb
@@ -33,13 +33,21 @@ def fetch_pages(url, referer = nil, depth = nil)
url = URI(url) unless url.is_a?(URI)
pages = []
get(url, referer) do |response, code, location, redirect_to, response_time|
- pages << Page.new(location, :body => response.body.dup,
- :code => code,
- :headers => response.to_hash,
- :referer => referer,
- :depth => depth,
- :redirect_to => redirect_to,
- :response_time => response_time)
+ body = response.body.dup
+ if response.to_hash.fetch('content-encoding', [])[0] == 'gzip'
+ gzip = Zlib::GzipReader.new(StringIO.new(body))
+ body = gzip.read
+ end
+ pages << Page.new(
+ location,
+ :body => body,
+ :code => code,
+ :headers => response.to_hash,
+ :referer => referer,
+ :depth => depth,
+ :redirect_to => redirect_to,
+ :response_time => response_time
+ )
end
return pages
@@ -89,6 +97,20 @@ def proxy_port
end
#
+ # The proxy username
+ #
+ def proxy_user
+ @opts[:proxy_user]
+ end
+
+ #
+ # The proxy password
+ #
+ def proxy_pass
+ @opts[:proxy_pass]
+ end
+
+ #
# HTTP read timeout in seconds
#
def read_timeout
@@ -160,7 +182,7 @@ def connection(url)
end
def refresh_connection(url)
- http = Net::HTTP.new(url.host, url.port, proxy_host, proxy_port)
+ http = Net::HTTP.new(url.host, url.port, proxy_host, proxy_port, proxy_user, proxy_pass)
http.read_timeout = read_timeout if !!read_timeout
Something went wrong with that request. Please try again.