Add Ruby 3 to Travis

.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.4
   Exclude:
     - 'bin/*'
   DisplayCopNames: true
@@ -16,11 +16,15 @@ Style/StringLiteralsInInterpolation:
 
 Layout/MultilineMethodCallIndentation:
   EnforcedStyle: indented
-Layout/TrailingBlankLines:
+Layout/TrailingEmptyLines:
   Enabled: true
 Layout/DotPosition:
   EnforcedStyle: leading
 
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - spec/**/*
+
 Metrics/LineLength:
   Exclude:
     - spec/**/*

.travis.yml

@@ -4,7 +4,7 @@ cache: bundler
 before_install:
   - "find /home/travis/.rvm/rubies -wholename '*default/bundler-*.gemspec' -delete"
   - rvm @global do gem uninstall bundler -a -x -I || true
-  - gem install bundler -v '~> 1.10'
+  - gem install bundler
 
 bundler_args: --without yard guard benchmarks
 env: JRUBY_OPTS="$JRUBY_OPTS --debug"
@@ -18,6 +18,7 @@ rvm:
   - 2.5
   - 2.6
   - 2.7
+  - 3.0
   - ruby-head
   - jruby-9.2.8.0
   - truffleruby-head

Gemfile

@@ -6,9 +6,11 @@ gemspec
 
 gem "nokogiri", "~> 1.8"
 gem "oga", "~> 3.2"
-gem "rubocop", "~> 0.80"
+gem "rubocop", "~> 1.0"
 
 group :test do
   gem "coveralls", require: false
+  # Until I find a way to introduce other MITM proxy
+  gem "webrick", "1.4.2"
   gem "evil-proxy", "~> 0.2"
 end

README.md

@@ -295,7 +295,7 @@ Default configuration looks as follows:
 
 ```ruby
 ProxyFetcher.configure do |config|
-  config.logger = Logger.new(STDOUT)
+  config.logger = Logger.new($stdout)
   config.user_agent = ProxyFetcher::Configuration::DEFAULT_USER_AGENT
   config.pool_size = 10
   config.client_timeout = 3

gemfiles/nokogiri.gemfile

@@ -8,6 +8,6 @@ gem "nokogiri", "~> 1.8"
 
 group :test do
   gem "coveralls", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.9"
 end

gemfiles/oga.gemfile

@@ -8,6 +8,6 @@ gem "oga", "~> 3.0"
 
 group :test do
   gem "coveralls", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.9"
 end

lib/proxy_fetcher.rb

@@ -4,47 +4,47 @@
 require "http"
 require "logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/version"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
-require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
-require File.dirname(__FILE__) + "/proxy_fetcher/manager"
-require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/exceptions"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration/providers_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/proxy"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/manager"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/http_client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_list_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/request"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/document"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/node"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/abstract_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/nokogiri_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_socks"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_us"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_http"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks4"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks5"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/base"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_socks"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_ssl"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_us"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/http_tunnel"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/mtpro"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxypedia"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_http"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks4"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks5"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/xroxy"
   end
 
   @__config_access_lock__ = Mutex.new

lib/proxy_fetcher/client/request.rb

@@ -41,15 +41,15 @@ class Request
       # @return [String]
       #   response body (requested resource content)
       #
-      def self.execute(args)
-        new(args).execute
+      def self.execute(**args)
+        new(**args).execute
       end
 
       # Initialize new HTTP request
       #
       # @return [Request]
       #
-      def initialize(args)
+      def initialize(**args)
         raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)

lib/proxy_fetcher/configuration.rb

@@ -114,7 +114,7 @@ def initialize
 
     # Sets default configuration options
     def reset!
-      @logger = Logger.new(STDOUT)
+      @logger = Logger.new($stdout)
       @user_agent = DEFAULT_USER_AGENT
       @pool_size = 10
       @client_timeout = 3

lib/proxy_fetcher/document/node.rb

@@ -83,7 +83,7 @@ def html
       def clear(text)
         return "" if text.nil? || text.empty?
 
-        text.strip.gsub(/[\t]/i, "")
+        text.strip.gsub(/\t/i, "")
       end
     end
   end

lib/proxy_fetcher/utils/http_client.rb

@@ -41,9 +41,8 @@ class HTTPClient
     # @return [String]
     #   resource content
     #
-    def self.fetch(*args)
-      url = args.shift
-      new(url, **args.first).fetch
+    def self.fetch(*args, **kwargs, &block)
+      new(*args, **kwargs, &block).fetch
     end
 
     # Initialize HTTP client instance
@@ -52,24 +51,28 @@ def self.fetch(*args)
     #
     def initialize(url, method: :get, params: {}, headers: {})
       @url = url.to_s
-      @method = method
+      @method = method.to_sym
       @params = params
       @headers = headers
 
-      @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
-      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      unless HTTP::Request::METHODS.include?(@method)
+        raise ArgumentError, "'#{@method}' is a wrong HTTP method name"
+      end
 
-      @ssl_ctx = OpenSSL::SSL::SSLContext.new
-      @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      @http = build_http_engine
+      @ssl_ctx = build_ssl_context
     end
 
     # Fetches resource content by sending HTTP request to it.
     #
     # @return [String]
     #   response body
     #
-    def fetch
-      response = process_http_request
+    def fetch(**options)
+      response = perform_http_request
+      return response if options.fetch(:raw, false)
+
       response.body.to_s
     rescue StandardError => e
       ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
@@ -78,13 +81,20 @@ def fetch
 
     protected
 
-    def process_http_request(http_method: method, http_params: params)
-      unless HTTP::Request::METHODS.include?(http_method)
-        raise ArgumentError, "'#{http_method}' is a wrong HTTP method name!"
+    def build_ssl_context
+      OpenSSL::SSL::SSLContext.new.tap do |context|
+        context.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
+    end
+
+    def build_http_engine
+      HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
+    end
 
+    def perform_http_request(http_method: method, http_params: params)
       http.public_send(
-        http_method.to_sym, url,
+        http_method,
+        url,
         form: http_params,
         ssl_context: ssl_ctx
       )

proxy_fetcher.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |gem|
   gem.version = ProxyFetcher.gem_version
   gem.summary = "Ruby gem for dealing with proxy lists from different providers"
   gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ")
-    This gem can help your Ruby application to make HTTP(S) requests 
-    using proxies by fetching and validating proxy lists from 
+    This gem can help your Ruby application to make HTTP(S) requests
+    using proxies by fetching and validating proxy lists from
     the different providers.
   TEXT
   gem.authors = ["Nikita Bulai"]

spec/proxy_fetcher/client/client_spec.rb

@@ -3,6 +3,11 @@
 require "spec_helper"
 require "json"
 
+begin
+  require "webrick"
+rescue LoadError
+  # nop
+end
 require "evil-proxy"
 require "evil-proxy/async"