Initial commit

.gitignore

@@ -0,0 +1,3 @@
+.byebug_history
+Gemfile.*
+pkg

.ruby-version

@@ -0,0 +1 @@
+2.6.3

Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+ruby File.read(".ruby-version").chomp
+
+gemspec

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Dmitry Vorotilin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,15 @@
+# Ferrum - fearless Ruby headless Chrome driver (as simple as Puppeteer, though even simpler).
+
+Navigate to `example.com` and save a screenshot:
+
+```ruby
+browser = Ferrum::Browser.new
+browser.goto("https://example.com")
+browser.screenshot(path: "example.png")
+browser.quit
+```
+
+## Links
+https://medium.com/@aslushnikov/automating-clicks-in-chromium-a50e7f01d3fb
+https://developer.mozilla.org/en-US/docs/Web/API/Intersection_Observer_API
+https://github.com/machinio/cuprite/commit/9b1041dd6cd954e0b40b17bc74824e7a3a3ff3f4

Rakefile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new("test")
+
+task default: :test

bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "irb"
+require "irb/completion"
+require "ferrum"
+
+IRB.start

ferrum.gemspec

@@ -0,0 +1,33 @@
+lib = File.expand_path("lib", __dir__)
+$:.unshift(lib) unless $:.include?(lib)
+
+require "ferrum/version"
+
+Gem::Specification.new do |s|
+  s.name          = "ferrum"
+  s.version       = Ferrum::VERSION
+  s.platform      = Gem::Platform::RUBY
+  s.authors       = ["Dmitry Vorotilin"]
+  s.email         = ["d.vorotilin@gmail.com"]
+  s.homepage      = "https://github.com/route/ferrum"
+  s.summary       = "Ruby headless Chrome driver"
+  s.description   = "Ferrum allows you to control headless Chrome browser"
+  s.license       = "MIT"
+  s.require_paths = ["lib"]
+  s.files         = Dir["{lib}/**/*"] + %w[LICENSE README.md]
+
+  s.required_ruby_version = ">= 2.3.0"
+
+  s.add_runtime_dependency "websocket-driver", ">= 0.6", "< 0.8"
+  s.add_runtime_dependency "cliver",           "~> 0.3"
+  s.add_runtime_dependency "addressable",      "~> 2.6"
+
+  s.add_development_dependency "rake",         "~> 12.3"
+  s.add_development_dependency "rspec",        "~> 3.8"
+  s.add_development_dependency "sinatra",      "~> 2.0"
+  s.add_development_dependency "puma",         "~> 4.1"
+  s.add_development_dependency "byebug",       "~> 10.0"
+  s.add_development_dependency "image_size",   "~> 2.0"
+  s.add_development_dependency "pdf-reader",   "~> 2.2"
+  s.add_development_dependency "chunky_png",   "~> 1.3"
+end

lib/ferrum.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+Thread.abort_on_exception = true
+Thread.report_on_exception = true if Thread.respond_to?(:report_on_exception=)
+
+module Ferrum
+  require "ferrum/browser"
+  require "ferrum/node"
+  require "ferrum/errors"
+  require "ferrum/cookie"
+
+  class << self
+    def windows?
+      RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
+    end
+
+    def mac?
+      RbConfig::CONFIG["host_os"] =~ /darwin/
+    end
+
+    def mri?
+      defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
+    end
+  end
+end

lib/ferrum/browser.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "base64"
+require "forwardable"
+require "ferrum/page"
+require "ferrum/targets"
+require "ferrum/browser/api"
+require "ferrum/browser/process"
+require "ferrum/browser/client"
+
+module Ferrum
+  class Browser
+    TIMEOUT = 5
+    WINDOW_SIZE = [1024, 768].freeze
+    BASE_URL_SCHEMA = %w[http https].freeze
+
+    include API
+    extend Forwardable
+
+    attr_reader :headers, :window_size
+
+    delegate on: :@client
+    delegate %i(window_handle window_handles switch_to_window open_new_window
+                close_window within_window page) => :targets
+    delegate %i(goto status body at_css at_xpath css xpath text property attributes attribute select_file
+                value visible? disabled? network_traffic clear_network_traffic
+                path response_headers refresh click right_click double_click
+                hover set click_coordinates select trigger scroll_to send_keys
+                evaluate evaluate_on evaluate_async execute frame_url
+                frame_title switch_to_frame current_url title go_back
+                go_forward find_modal accept_confirm dismiss_confirm
+                accept_prompt dismiss_prompt reset_modals authorize
+                proxy_authorize) => :page
+
+    attr_reader :process, :logger, :js_errors, :slowmo, :base_url,
+                :url_blacklist, :url_whitelist, :options
+    attr_writer :timeout
+
+    def initialize(options = nil)
+      options ||= {}
+
+      @client = nil
+      @window_size = options.fetch(:window_size, WINDOW_SIZE)
+      @original_window_size = @window_size
+
+      @options = Hash(options.merge(window_size: @window_size))
+      @logger, @timeout = @options.values_at(:logger, :timeout)
+      @js_errors = @options.fetch(:js_errors, false)
+      @slowmo = @options[:slowmo]
+
+      if @options.key?(:base_url)
+        self.base_url = @options[:base_url]
+      end
+
+      self.url_blacklist = @options[:url_blacklist]
+      self.url_whitelist = @options[:url_whitelist]
+
+      if ENV["FERRUM_DEBUG"] && !@logger
+        STDOUT.sync = true
+        @logger = STDOUT
+        @options[:logger] = @logger
+      end
+
+      @options.freeze
+
+      start
+    end
+
+    def base_url=(value)
+      parsed = Addressable::URI.parse(value)
+      unless BASE_URL_SCHEMA.include?(parsed.normalized_scheme)
+        raise "Set `base_url` should be absolute and include schema: #{BASE_URL_SCHEMA}"
+      end
+
+      @base_url = parsed
+    end
+
+    def extensions
+      @extensions ||= Array(@options[:extensions]).map { |p| File.read(p) }
+    end
+
+    def timeout
+      @timeout || TIMEOUT
+    end
+
+    def command(*args)
+      id = @client.command(*args)
+      @client.wait(id: id)
+    rescue DeadBrowser
+      restart
+      raise
+    end
+
+    def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
+      options = Hash.new
+      options[:userAgent] = user_agent if user_agent
+      options[:acceptLanguage] = accept_language if accept_language
+      options[:platform] if platform
+
+      page.command("Network.setUserAgentOverride", **options) if !options.empty?
+    end
+
+    def clear_memory_cache
+      page.command("Network.clearBrowserCache")
+    end
+
+    def reset
+      @headers = {}
+      @zoom_factor = nil
+      @window_size = @original_window_size
+      targets.reset
+    end
+
+    def restart
+      quit
+      start
+    end
+
+    def quit
+      @client.close
+      @process.stop
+      @client = @process = @targets = nil
+    end
+
+    def targets
+      @targets ||= Targets.new(self)
+    end
+
+    def resize(**options)
+      @window_size = [options[:width], options[:height]]
+      page.resize(**options)
+    end
+
+    def crash
+      command("Browser.crash")
+    end
+
+    private
+
+    def start
+      @headers = {}
+      @process = Process.start(@options)
+      @client = Client.new(self, @process.ws_url, 0, false)
+    end
+  end
+end

lib/ferrum/browser/api.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "ferrum/browser/api/cookie"
+require "ferrum/browser/api/header"
+require "ferrum/browser/api/screenshot"
+require "ferrum/browser/api/intercept"
+
+module Ferrum
+  class Browser
+    module API
+      include Cookie, Header, Screenshot, Intercept
+    end
+  end
+end

lib/ferrum/browser/api/cookie.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module API
+      module Cookie
+        def cookies
+          cookies = page.command("Network.getAllCookies")["cookies"]
+          cookies.map { |c| [c["name"], ::Ferrum::Cookie.new(c)] }.to_h
+        end
+
+        def set_cookie(name: nil, value: nil, cookie: nil, **options)
+          cookie = options.dup
+          cookie[:name]   ||= name
+          cookie[:value]  ||= value
+          cookie[:domain] ||= default_domain
+
+          expires = cookie.delete(:expires).to_i
+          cookie[:expires] = expires if expires > 0
+
+          page.command("Network.setCookie", **cookie)
+        end
+
+        # Supports :url, :domain and :path options
+        def remove_cookie(name:, **options)
+          raise "Specify :domain or :url option" if !options[:domain] && !options[:url] && !default_domain
+
+          options = options.merge(name: name)
+          options[:domain] ||= default_domain
+
+          page.command("Network.deleteCookies", **options)
+        end
+
+        def clear_cookies
+          page.command("Network.clearBrowserCookies")
+        end
+
+        private
+
+        def default_domain
+          URI.parse(base_url).host if base_url
+        end
+      end
+    end
+  end
+end

lib/ferrum/browser/api/header.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module API
+      module Header
+        def headers=(headers)
+          @headers = {}
+          add_headers(headers)
+        end
+
+        def add_headers(headers, permanent: true)
+          if headers["Referer"]
+            page.referrer = headers["Referer"]
+            headers.delete("Referer") unless permanent
+          end
+
+          @headers.merge!(headers)
+          user_agent = @headers["User-Agent"]
+          accept_language = @headers["Accept-Language"]
+
+          set_overrides(user_agent: user_agent, accept_language: accept_language)
+          page.command("Network.setExtraHTTPHeaders", headers: @headers)
+        end
+
+        def add_header(header, permanent: true)
+          add_headers(header, permanent: permanent)
+        end
+      end
+    end
+  end
+end