v0.3.3

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    spidy (0.2.9)
+    spidy (0.3.3)
       activesupport
       mechanize
       pry
@@ -11,12 +11,24 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.2)
+    activesupport (6.0.3.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    capybara (3.33.0)
+      addressable
+      mini_mime (>= 0.1.3)
+      nokogiri (~> 1.8)
+      rack (>= 1.6.0)
+      rack-test (>= 0.6.3)
+      regexp_parser (~> 1.5)
+      xpath (~> 3.2)
+    capybara_discoball (0.1.0)
+      capybara (>= 2.7, < 4)
     coderay (1.1.2)
     concurrent-ruby (1.1.7)
     connection_pool (2.2.3)
@@ -41,9 +53,12 @@ GEM
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
+    mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     minitest (5.14.2)
     mixlib-shellout (2.4.4)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     net-http-digest_auth (1.4.1)
     net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
@@ -53,7 +68,14 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    public_suffix (4.0.6)
+    rack (2.2.3)
+    rack-protection (2.0.8.1)
+      rack
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
     rake (10.5.0)
+    regexp_parser (1.8.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -74,27 +96,38 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.2)
+    ruby2_keywords (0.0.2)
+    sinatra (2.0.8.1)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.8.1)
+      tilt (~> 2.0)
     socksify (1.7.1)
     thread_safe (0.3.6)
+    tilt (2.0.10)
     tor (0.1.4)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
     webrobots (0.1.2)
+    xpath (3.2.0)
+      nokogiri (~> 1.8)
     zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 2.0)
+  capybara_discoball
   ffaker
   pry
   rake (~> 10.0)
   rspec (~> 3.0)
   rspec-command
+  sinatra
   spidy!
 
 BUNDLED WITH

lib/spidy/connector.rb

@@ -15,7 +15,7 @@ module Spidy::Connector
   #
   # default user agent
   #
-  DEFAULT_USER_AGENT = [
+  USER_AGENT = [
     'Mozilla/5.0',
     '(Macintosh; Intel Mac OS X 10_12_6)',
     'AppleWebKit/537.36',
@@ -36,7 +36,7 @@ module StaticAccessor
     extend ActiveSupport::Concern
     class_methods do
       def call(url, wait_time: 5, logger: Spidy::Connector::DEFAULT_LOGGER, user_agent: Spidy::Connector::USER_AGENT, &block)
-        new(wait_time: wait_time, user_agent: user_agent, logger: logger).call(url, &block)
+        ::Spidy::Connector::RetryableCaller.new(new(user_agent: user_agent), wait_time: wait_time, logger: logger).call(url, &block)
       end
     end
   end
@@ -57,7 +57,7 @@ def initialize(object: nil, error: nil, response_code: nil)
   #
   # retry
   #
-  class Retryable
+  class RetryableCaller
     attr_reader :origin_connector
 
     def initialize(connector, logger:, wait_time:)
@@ -68,6 +68,7 @@ def initialize(connector, logger:, wait_time:)
     end
 
     def call(url, &block)
+      block ||= ->(result) { break result }
       connect(url, &block)
     end
 
@@ -120,12 +121,12 @@ def try_connection!
   end
 
   def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
+    user_agent ||= USER_AGENT
     logger ||= DEFAULT_LOGGER
-    user_agent ||= DEFAULT_USER_AGENT
     wait_time ||= DEFAULT_WAIT_TIME
 
     connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
-    Retryable.new(connector, wait_time: wait_time, logger: logger)
+    RetryableCaller.new(connector, wait_time: wait_time, logger: logger)
   end
 
   #

lib/spidy/connector/json.rb

@@ -8,7 +8,7 @@ class Spidy::Connector::Json
 
   attr_reader :logger
 
-  def initialize(user_agent: nil)
+  def initialize(user_agent:)
     @user_agent = user_agent
   end
 

lib/spidy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spidy
-  VERSION = '0.3.2'
+  VERSION = '0.3.3'
 end

spec/connector_spec.rb

@@ -1,6 +1,42 @@
 # frozen_string_literal: true
 
 require 'spec_helper'
+require 'capybara_discoball'
+require 'sinatra'
+
+class ConnectorMock < Sinatra::Base
+  get '/test.html' do
+    <<-HTML
+      <html>
+        <head>
+          <title>TEST</title>
+        </head>
+        <body>
+          test
+        </body>
+      </html>
+    HTML
+  end
+
+  get '/test.json' do
+    content_type :json
+    {
+      title: 'test'
+    }.to_json
+  end
+
+  get '/test.xml' do
+    <<-XML
+      <TEST>
+        <A>TEST</A>
+      </TEST>
+    XML
+  end
+end
+
+Capybara::Discoball.spin(ConnectorMock) do |server|
+  ConnectorMock::BASE_URL = server.url
+end
 
 RSpec.describe Spidy::Connector do
   specify do
@@ -15,4 +51,44 @@
   specify do
     expect(Spidy::Connector.get(:direct).origin_connector).to be_kind_of(Spidy::Connector::Direct)
   end
+
+  describe 'static accessor' do
+
+    specify :html do
+      expect {
+        Spidy::Connector::Html.call("#{ConnectorMock::BASE_URL}/test.html")
+      }.not_to raise_error
+    end
+
+    specify :json do
+      expect {
+        Spidy::Connector::Json.call("#{ConnectorMock::BASE_URL}/test.json")
+      }.not_to raise_error
+    end
+
+    specify :xml do
+      expect {
+        Spidy::Connector::Xml.call("#{ConnectorMock::BASE_URL}/test.xml")
+      }.not_to raise_error
+    end
+
+  end
+
+  specify :html do
+    expect {
+      Spidy::Connector.get(:html).call("#{ConnectorMock::BASE_URL}/test.html")
+    }.not_to raise_error
+  end
+
+  specify :json do
+    expect {
+      Spidy::Connector.get(:json).call("#{ConnectorMock::BASE_URL}/test.json")
+    }.not_to raise_error
+  end
+
+  specify :xml do
+    expect {
+      Spidy::Connector.get(:xml).call("#{ConnectorMock::BASE_URL}/test.xml")
+    }.not_to raise_error
+  end
 end

spec/master_detail_html_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe 'Master detail page' do
-  spidy = Spidy.open("#{__dir__}/example/master_detail.rb")
+  spidy = Spidy.open("#{__dir__}/../example/master_detail.rb")
 
   specify 'each' do
     last_url = nil

spec/shell_spec.rb

@@ -7,12 +7,12 @@
   include RSpecCommand
 
   describe '#each' do
-    command "spidy each #{__dir__}/example.rb"
+    command "spidy each #{__dir__}/../example/master_detail.rb"
     its(:exitstatus) { is_expected.to eq 0 }
   end
 
   describe 'call' do
-    command { "echo 'http://localhost/?id=91' | spidy call #{__dir__}/example/master_detail.rb" }
+    command { "echo 'http://localhost/?id=91' | spidy call #{__dir__}/../example/master_detail.rb" }
     its(:exitstatus) { is_expected.to eq 0 }
     its(:stdout) { is_expected.to include('{"title":"title_91","body":"body_91"}') }
   end

spidy.gemspec

@@ -30,6 +30,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'ffaker'
   spec.add_development_dependency 'rspec-command'
+  spec.add_development_dependency 'capybara_discoball'
+  spec.add_development_dependency 'sinatra'
 
   spec.add_runtime_dependency 'tor'
   spec.add_runtime_dependency 'activesupport'