add webservice client

changes.md

@@ -7,7 +7,7 @@
   - [redis bitfield](https://stackoverflow.com/questions/47100606/optimal-way-to-store-array-of-integers-in-redis-database)
 - Add support for [zadd](http://redis.io/topics/data-types-intro) so one could determine single call versus multiple calls on a line, letting us determine the most executed code in production.
 
-### Coverband 4.X
+### Coverband Future...
 
 Will be the fully modern release that drops maintenance legacy support in favor of increased performance, ease of use, and maintainability.
 
@@ -60,7 +60,14 @@ Will be the fully modern release that drops maintenance legacy support in favor
   - drops S3 support
   - drops static report support
   - drops gem support
-- ?
+  - only loaded web reporter files when required
+- improved load order allowing more time for ENV vars (better dotenv, figaro, rails secrets support)
+- improved resque patching pattern
+- improved default ignores
+- additional adapters
+  - supports web-service adapter for http coverage collection
+  - support log/file adapter
+- reduce logs / errors / alerts on bad startup configurations
 
 # Released
 

coverband.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "coveralls"
   spec.add_development_dependency "minitest-profile"
+  spec.add_development_dependency "webmock"
 
   # TODO: Remove when other production adapters exist
   # because the default configuration of redis store, we really do require

lib/coverband.rb

@@ -16,6 +16,7 @@
 require "coverband/utils/file_hasher"
 require "coverband/collectors/coverage"
 require "coverband/collectors/view_tracker"
+require "coverband/collectors/view_tracker_service"
 require "coverband/reporters/base"
 require "coverband/reporters/console_report"
 require "coverband/integrations/background"
@@ -37,7 +38,7 @@ module Coverband
   def self.configure(file = nil)
     configuration_file = file || ENV["COVERBAND_CONFIG"]
     if configuration_file.nil?
-      configuration_file = File.exist?(SERVICE_CONFIG) ? SERVICE_CONFIG : CONFIG_FILE
+      configuration_file = coverband_service? ? SERVICE_CONFIG : CONFIG_FILE
     end
 
     configuration
@@ -52,6 +53,10 @@ def self.configure(file = nil)
     coverage_instance.reset_instance
   end
 
+  def self.coverband_service?
+    !!File.exist?(SERVICE_CONFIG)
+  end
+
   def self.configured?
     @@configured
   end

lib/coverband/adapters/web_service_store.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Coverband
+  module Adapters
+    ###
+    # WebServiceStore store a merged coverage file to local disk
+    # TODO: add webmock tests
+    ###
+    class WebServiceStore < Base
+      attr_reader :coverband_url, :process_type, :runtime_env, :hostname, :pid
+
+      def initialize(coverband_url, opts = {})
+        super()
+        require "socket"
+        require "securerandom"
+        @coverband_url = coverband_url
+        @process_type = opts.fetch(:process_type) { $PROGRAM_NAME&.split("/")&.last || Coverband.configuration.process_type }
+        @hostname = opts.fetch(:hostname) { ENV["DYNO"] || Socket.gethostname.force_encoding("utf-8").encode }
+        @hostname = @hostname.delete("'", "").delete("’", "")
+        @runtime_env = opts.fetch(:runtime_env) { Coverband.configuration.coverband_env }
+        @failed_coverage_reports = []
+      end
+
+      def logger
+        Coverband.configuration.logger
+      end
+
+      def clear!
+        # done via service UI
+        raise "not supported via service"
+      end
+
+      def clear_file!(filename)
+        # done via service UI
+        raise "not supported via service"
+      end
+
+      # NOTE: Should support nil to mean not supported
+      # the size feature doesn't really makde sense for the service
+      def size
+        0
+      end
+
+      ###
+      # Fetch coverband coverage via the API
+      # This would allow one to expore from the service and move back to the open source
+      # without having to reset coverage
+      ###
+      def coverage(local_type = nil, opts = {})
+        return if Coverband.configuration.service_disabled_dev_test_env?
+
+        local_type ||= opts.key?(:override_type) ? opts[:override_type] : type
+        env_filter = opts.key?(:env_filter) ? opts[:env_filter] : "production"
+        uri = URI("#{coverband_url}/api/coverage?type=#{local_type}&env_filter=#{env_filter}")
+        req = Net::HTTP::Get.new(uri, "Content-Type" => "application/json", "Coverband-Token" => Coverband.configuration.api_key)
+        res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
+          http.request(req)
+        end
+        coverage_data = JSON.parse(res.body)
+        coverage_data
+      rescue => e
+        logger&.error "Coverband: Error while retrieving coverage #{e}" if Coverband.configuration.verbose || Coverband.configuration.service_dev_mode
+      end
+
+      def save_report(report)
+        return if report.empty?
+
+        # We set here vs initialize to avoid setting on the primary process vs child processes
+        @pid ||= ::Process.pid
+
+        # TODO: do we need dup
+        # TODO: we don't need upstream timestamps, server will track first_seen
+        Thread.new do
+          data = expand_report(report.dup)
+          full_package = {
+            collection_type: "coverage_delta",
+            collection_data: {
+              tags: {
+                process_type: process_type,
+                app_loading: type == Coverband::EAGER_TYPE,
+                runtime_env: runtime_env,
+                pid: pid,
+                hostname: hostname
+              },
+              file_coverage: data
+            }
+          }
+
+          save_coverage(full_package)
+          retry_failed_reports
+        end&.join
+      end
+
+      def raw_store
+        raise "not supported via service"
+      end
+
+      private
+
+      def retry_failed_reports
+        retries = []
+        @failed_coverage_reports.any? do
+          begin
+            report_body = @failed_coverage_reports.pop
+            send_report_body(report_body)
+          rescue
+            retries << report_body
+          end
+        end
+        retries.each do |report_body|
+          add_retry_message(report_body)
+        end
+      end
+
+      def add_retry_message(report_body)
+        if @failed_coverage_reports.length > 5
+          logger&.info "Coverband: The errored reporting queue has reached 5. Subsequent reports will not be transmitted"
+        else
+          @failed_coverage_reports << report_body
+        end
+      end
+
+      def save_coverage(data)
+        if Coverband.configuration.api_key.nil?
+          puts "Coverband: Error: no Coverband API key was found!"
+          return
+        end
+
+        coverage_body = {remote_uuid: SecureRandom.uuid, data: data}.to_json
+        send_report_body(coverage_body)
+      rescue => e
+        add_retry_message(coverage_body)
+        logger&.info "Coverband: Error while saving coverage #{e}" if Coverband.configuration.verbose || Coverband.configuration.service_dev_mode
+      end
+
+      def send_report_body(coverage_body)
+        uri = URI("#{coverband_url}/api/collector")
+        req = ::Net::HTTP::Post.new(uri, "Content-Type" => "application/json", "Coverband-Token" => Coverband.configuration.api_key)
+        req.body = coverage_body
+        logger&.info "Coverband: saving (#{uri}) #{req.body}" if Coverband.configuration.verbose
+        res = ::Net::HTTP.start(
+          uri.hostname,
+          uri.port,
+          open_timeout: Coverband.configuration.coverband_timeout,
+          read_timeout: Coverband.configuration.coverband_timeout,
+          ssl_timeout: Coverband.configuration.coverband_timeout,
+          use_ssl: uri.scheme == "https"
+        ) do |http|
+          http.request(req)
+        end
+        if res.code.to_i >= 500
+          add_retry_message(coverage_body)
+        end
+      end
+    end
+  end
+end

lib/coverband/collectors/view_tracker_service.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Coverband
+  module Collectors
+    ###
+    # This class extends view tracker to support web service reporting
+    ###
+    class ViewTrackerService < ViewTracker
+      def report_views_tracked
+        reported_time = Time.now.to_i
+        if views_to_record.any?
+          relative_views = views_to_record.map! do |view|
+            roots.each do |root|
+              view = view.gsub(/#{root}/, "")
+            end
+            view
+          end
+          save_tracked_views(views: relative_views, reported_time: reported_time)
+        end
+        self.views_to_record = []
+      rescue => e
+        # we don't want to raise errors if Coverband can't reach the service
+        logger&.error "Coverband: view_tracker failed to store, error #{e.class.name}" if Coverband.configuration.verbose || Coverband.configuration.service_dev_mode
+      end
+
+      def self.supported_version?
+        defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::STRING.split(".").first.to_i >= 4
+      end
+
+      private
+
+      def logger
+        Coverband.configuration.logger
+      end
+
+      def save_tracked_views(views:, reported_time:)
+        uri = URI("#{Coverband.configuration.service_url}/api/collector")
+        req = Net::HTTP::Post.new(uri, "Content-Type" => "application/json", "Coverband-Token" => Coverband.configuration.api_key)
+        data = {
+          collection_type: "view_tracker_delta",
+          collection_data: {
+            tags: {
+              runtime_env: Coverband.configuration.coverband_env
+            },
+            collection_time: reported_time,
+            tracked_views: views
+          }
+        }
+        # puts "sending #{data}"
+        req.body = {remote_uuid: SecureRandom.uuid, data: data}.to_json
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
+          http.request(req)
+        end
+      rescue => e
+        logger&.error "Coverband: Error while saving coverage #{e}" if Coverband.configuration.verbose || Coverband.configuration.service_dev_mode
+      end
+    end
+  end
+end