branch rebased

lib/polipus.rb

@@ -204,20 +204,13 @@ def takeover
             # Execute on_before_save blocks
             @on_before_save.each { |e| e.call(page) }
 
-<<<<<<< HEAD
-            page.storable? && @storage.add(page)
-
-            @logger.debug { "[worker ##{worker_number}] Fetched page: [#{page.url}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]" }
-            @logger.info  { "[worker ##{worker_number}] Page (#{page.url}) downloaded" }
-=======
             if page.storable?
-              @storage.add page
+              @storage.add(page)
               execute_plugin 'on_page_stored'
             end
-
-            @logger.debug {"[worker ##{worker_number}] Fetched page: [#{page.url.to_s}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]"}
-            @logger.info  {"[worker ##{worker_number}] Page (#{page.url.to_s}) downloaded"}
->>>>>>> new plugin strategy
+
+            @logger.debug { "[worker ##{worker_number}] Fetched page: [#{page.url}] Referrer: [#{page.referer}] Depth: [#{page.depth}] Code: [#{page.code}] Response Time: [#{page.response_time}]" }
+            @logger.info  { "[worker ##{worker_number}] Page (#{page.url}) downloaded" }
 
             incr_pages
 
@@ -464,24 +457,21 @@ def overflow_items_controller
           else
             @logger.info { 'Lock not acquired' }
           end
-<<<<<<< HEAD
 
           sleep @options[:queue_overflow_manager_check_time]
-=======
+
         end
       end
 
       # It invokes a plugin method if any
-      def execute_plugin method
+      def execute_plugin(method)
         method = method.to_sym
-        Polipus::Plugin.plugins.each do |k,plugin_instance|
+        Polipus::Plugin.plugins.each do |k, plugin_instance|
           if plugin_instance.class.plugin_data[method]
-            @logger.info("Running plugin method #{method} on #{k}")
-            self.instance_exec(plugin_instance, &plugin_instance.class.plugin_data[method])
+            @logger.info { "Running plugin method #{method} on #{k}" }
+            instance_exec(plugin_instance, &plugin_instance.class.plugin_data[method])
           end
->>>>>>> new plugin strategy
         end
-
       end
     end
 

lib/polipus/plugin.rb

@@ -1,6 +1,5 @@
 module Polipus
   module Plugin
-
     @@plugins = {}
     def self.register(plugin, options = {})
       o = plugin.new(options)
@@ -12,7 +11,6 @@ def self.plugins
     end
 
     module Base
-
       attr_reader :plugin_options
       def initialize(options = {})
         @plugin_options = options
@@ -25,7 +23,6 @@ def self.included(mod)
       end
 
       module ClassMethods
-
         @@plugin_data = {}
 
         def plugin_data
@@ -64,8 +61,6 @@ def on_crawl_end(&block)
           @@plugin_data[__callee__] = block
         end
       end
-
     end
-
   end
 end

lib/polipus/plugins/cleaner.rb

@@ -1,24 +1,24 @@
 module Polipus
   module Plugin
     class Cleaner
+      include Polipus::Plugin::Base
 
       on_initialize do |plugin_instance|
         if plugin_instance.options[:reset]
-          @logger.info {"Cleaning all: url_tracker, storage, queue"}
+          @logger.info { 'Cleaning all: url_tracker, storage, queue' }
           url_tracker.clear
           storage.clear
           queue_factory.clear
           @options[:queue_overflow_adapter].clear if @options[:queue_overflow_adapter]
         else
-          @logger.info {"Cleaner plugin is disable, add :reset => true to the plugin if you really know what you are doing"}
+          @logger.info { 'Cleaner plugin is disable, add :reset => true to the plugin if you really know what you are doing' }
         end
       end
 
       def plugin_registered
         puts "Plugin #{self.class.name} registered with options: #{plugin_options}"
         @options[:reset] ||= false
       end
-
     end
   end
-end
+end

lib/polipus/plugins/sample.rb

@@ -1,20 +1,18 @@
 module Polipus
   module Plugin
     class Sample
-
       include Polipus::Plugin::Base
 
-      on_initialize do |plugin_instance|
+      on_initialize do |_plugin_instance|
         puts 'on_initialize called'
         puts '-------------------------------------'
-        @options.each { |k,v| @logger.info { "Polipus configuration: #{k.to_s} => #{v}" } }
+        @options.each { |k, v| @logger.info { "Polipus configuration: #{k} => #{v}" } }
         puts '-------------------------------------'
       end
 
       def plugin_registered
         puts "Plugin registered with options: #{plugin_options}"
       end
-
     end
   end
 end

lib/polipus/plugins/sleeper.rb

@@ -1,12 +1,11 @@
 module Polipus
   module Plugin
     class Sleeper
-
       include Polipus::Plugin::Base
 
       on_initialize do |plugin_instance|
         @options[:workers] = 1
-        @logger.info {"#{self.class.name}: options: #{plugin_instance.plugin_options}"}
+        @logger.info { "#{self.class.name}: options: #{plugin_instance.plugin_options}" }
       end
 
       on_message_processed do |plugin_instance|
@@ -16,7 +15,6 @@ class Sleeper
       def plugin_registered
         puts "Plugin #{self.class.name} registered with options: #{plugin_options}"
       end
-
     end
   end
 end