-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
93 additions
and
89 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
module ProxyFetcher | ||
class Manager | ||
PROXY_PROVIDER_URL = 'http://proxylist.hidemyass.com/'.freeze | ||
|
||
class << self | ||
def config | ||
@config ||= ProxyFetcher::Configuration.new | ||
end | ||
end | ||
|
||
attr_reader :proxies | ||
|
||
# refresh: true - load proxy list from the remote server on initialization | ||
# refresh: false - just initialize the class, proxy list will be empty ([]) | ||
def initialize(refresh: true) | ||
if refresh | ||
refresh_list! | ||
else | ||
@proxies = [] | ||
end | ||
end | ||
|
||
# Update current proxy list from the provider | ||
def refresh_list! | ||
doc = Nokogiri::HTML(load_html(PROXY_PROVIDER_URL)) | ||
rows = doc.xpath('//table[@id="listable"]/tbody/tr') | ||
|
||
@proxies = rows.map { |row| Proxy.new(row) } | ||
end | ||
|
||
alias fetch! refresh_list! | ||
|
||
# Pop just first proxy (and back it to the end of the proxy list) | ||
def get | ||
return if @proxies.empty? | ||
|
||
first_proxy = @proxies.shift | ||
@proxies << first_proxy | ||
|
||
first_proxy | ||
end | ||
|
||
alias pop get | ||
|
||
# Pop first valid proxy (and back it to the end of the proxy list) | ||
# Invalid proxies will be removed from the list | ||
def get! | ||
index = @proxies.find_index(&:connectable?) | ||
return if index.nil? | ||
|
||
proxy = @proxies.delete_at(index) | ||
tail = @proxies[index..-1] | ||
|
||
@proxies = tail << proxy | ||
|
||
proxy | ||
end | ||
|
||
alias pop! get! | ||
|
||
# Clean current proxy list from dead proxies (doesn't respond by timeout) | ||
def cleanup! | ||
proxies.keep_if(&:connectable?) | ||
end | ||
|
||
alias validate! cleanup! | ||
|
||
# Just schema + host + port | ||
def raw_proxies | ||
proxies.map(&:url) | ||
end | ||
|
||
# No need to put all the attr_readers | ||
def inspect | ||
to_s | ||
end | ||
|
||
private | ||
|
||
# Get HTML from the requested URL | ||
def load_html(url) | ||
uri = URI.parse(url) | ||
http = Net::HTTP.new(uri.host, uri.port) | ||
response = http.get(uri.request_uri) | ||
response.body | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters