Sidekiq web (#297)

* Initial sidekiq web extention commit

* Add yardopts file

* A simple web interface for unique digests

* Adds a filter function for the web extension

* Makes search form nicer

* Remove unnecessary require

* Add readme entry about the web extension

* Styles

* Remove unused files

* Fix broken specs

* Fix styles

* Allow to complete in more fractions

.reek.yml

@@ -10,6 +10,7 @@ detectors:
   DuplicateMethodCall:
     exclude:
     - Sidekiq#self.use_options
+    - SidekiqUniqueJobs::Web#self.registered
   IrresponsibleModule:
     enabled: false
   LongParameterList:
@@ -30,6 +31,9 @@ detectors:
     - SidekiqUniqueJobs::Server::Middleware#call
     - SidekiqUniqueJobs::UniqueArgs#filtered_args
     - SidekiqUniqueJobs::Util#del
+    - SidekiqUniqueJobs::Digests#batch_delete
+    - SidekiqUniqueJobs::Digests#delete_by_pattern
+    - SidekiqUniqueJobs::Web#self.registered
   UncommunicativeVariableName:
     exclude:
     - Hash#slice
@@ -48,6 +52,7 @@ detectors:
     - Hash#slice!
     - SidekiqUniqueJobs::OnConflict::Reject#deadset_kill?
     - SidekiqUniqueJobs::SidekiqWorkerMethods#worker_method_defined?
+    - SidekiqUniqueJobs::Web::Helpers#redirect_to
   MissingSafeMethod:
     exclude:
     - Array
@@ -58,12 +63,14 @@ detectors:
     - SidekiqUniqueJobs::OnConflict::Reject#push_to_deadset
     - SidekiqUniqueJobs::Logging#debug_item
     - SidekiqUniqueJobs::Util#batch_delete
+    - SidekiqUniqueJobs::Digests#batch_delete
   NestedIterators:
     exclude:
     - SidekiqUniqueJobs::Locksmith#create_lock
     - SidekiqUniqueJobs::Middleware#configure_client_middleware
     - SidekiqUniqueJobs::Middleware#configure_server_middleware
     - SidekiqUniqueJobs::Util#batch_delete
+    - SidekiqUniqueJobs::Digests#batch_delete
   TooManyInstanceVariables:
     exclude:
     - SidekiqUniqueJobs::Locksmith

.yardopts

@@ -0,0 +1,7 @@
+  --no-private
+  --exclude lib/sidekiq_unique_jobs/testing.rb
+  --exclude lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb
+  --exclude lib/sidekiq_unique_jobs/middleware.rb
+  --exclude lib/sidekiq_unique_jobs/core_ext.rb
+  --exclude lib/sidekiq_unique_jobs/cli.rb
+

CHANGELOG.md

@@ -11,6 +11,7 @@
 - Removed concurrency for now (a0cff5bc42edbe7190d6ede7e7f845074d2d7af6)
 - Improved integration testing for locks
 - Totally delete the hash that was growing out of proportion
+- Adds a sidekiq web extension for viewing and deleting unique digests
 
 ## v5.1.0
 

README.md

@@ -25,6 +25,7 @@
    * [After Unlock Callback](#after-unlock-callback)
    * [Logging](#logging)
 * [Debugging](#debugging)
+   * [Sidekiq Web](#sidekiq-web)
    * [Console](#console)
       * [List Unique Keys](#list-unique-keys)
       * [Remove Unique Keys](#remove-unique-keys)
@@ -342,21 +343,28 @@ end
 
 There are two ways to display and remove keys regarding uniqueness. The console way and the command line way.
 
-### Console
+### Sidekiq Web
 
-Start the console with the following command `bundle exec uniquejobs console`.
+To use the web extension you need to require it in your routes.
 
-#### List Unique Keys
+```ruby
+# app/config/routes.rb
+require 'sidekiq-unique-jobs/web'
+mount Sidekiq::Web, at: '/sidekiq'
+```
+
+There is no need to `require 'sidekiq/web'` since `sidekiq_unique_jobs/web` 
+already does this.
 
-`keys '*', 100`
 
-#### Remove Unique Keys
+#### Show Unique Digests
 
-`del '*', 100, false` the dry_run and count parameters are both required. This is to have some type of protection against clearing out all uniqueness.
+![Unique Digests](assets/unique_digests_1.png)
 
-### Command Line
+#### Show keys for digest
 
-`bundle exec uniquejobs` displays help on how to use the unique jobs command line.
+![Unique Digests](assets/unique_digests_2
+.png)
 
 ## Communication
 

lib/sidekiq_unique_jobs.rb

@@ -12,6 +12,7 @@
 require 'sidekiq_unique_jobs/connection'
 require 'sidekiq_unique_jobs/exceptions'
 require 'sidekiq_unique_jobs/util'
+require 'sidekiq_unique_jobs/digests'
 require 'sidekiq_unique_jobs/cli'
 require 'sidekiq_unique_jobs/core_ext'
 require 'sidekiq_unique_jobs/timeout'

lib/sidekiq_unique_jobs/constants.rb

@@ -14,6 +14,7 @@ module SidekiqUniqueJobs
   UNIQUE_ARGS_KEY           ||= 'unique_args'
   UNIQUE_DIGEST_KEY         ||= 'unique_digest'
   UNIQUE_KEY                ||= 'unique'
+  UNIQUE_SET                ||= 'unique:keys'
   LOCK_KEY                  ||= 'lock'
   ON_CONFLICT_KEY           ||= 'on_conflict'
   UNIQUE_ON_ALL_QUEUES_KEY  ||= 'unique_on_all_queues' # TODO: Remove in v6.1

lib/sidekiq_unique_jobs/digests.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Utility module to help manage unique digests in redis.
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  module Digests
+    DEFAULT_COUNT = 1_000
+    SCAN_PATTERN  = '*'
+    CHUNK_SIZE    = 100
+
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
+    extend self # rubocop:disable Style/ModuleFunction
+
+    # Return unique digests matching pattern
+    #
+    # @param [String] pattern a pattern to match with
+    # @param [Integer] count the maximum number to match
+    # @return [Array<String>] with unique digests
+    def all(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      redis { |conn| conn.sscan_each(UNIQUE_SET, match: pattern, count: count).to_a }
+    end
+
+    # Get a total count of unique digests
+    #
+    # @return [Integer] number of digests
+    def count
+      redis { |conn| conn.scard(UNIQUE_SET) }
+    end
+
+    # Deletes unique digest either by a digest or pattern
+    #
+    # @param [String] digest the full digest to delete
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
+    # @raise [ArgumentError] when both pattern and digest are nil
+    # @return [Array<String>] with unique digests
+    def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
+      return delete_by_pattern(pattern, count: count) if pattern
+      return delete_by_digest(digest) if digest
+
+      raise ArgumentError, 'either digest or pattern need to be provided'
+    end
+
+    private
+
+    # Deletes unique digests by pattern
+    #
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
+    # @return [Array<String>] with unique digests
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = all(pattern: pattern, count: count)
+        batch_delete(digests)
+        digests.size
+      end
+
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    # Get a total count of unique digests
+    #
+    # @param [String] digest a key pattern to match with
+    def delete_by_digest(digest)
+      result, elapsed = timed do
+        Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, digest])
+        count
+      end
+
+      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    def batch_delete(digests) # rubocop:disable Metrics/MethodLength
+      redis do |conn|
+        digests.each_slice(CHUNK_SIZE) do |chunk|
+          conn.pipelined do
+            chunk.each do |digest|
+              conn.del digest
+              conn.srem(UNIQUE_SET, digest)
+              conn.del("#{digest}:EXISTS")
+              conn.del("#{digest}:GRABBED")
+              conn.del("#{digest}:VERSION")
+              conn.del("#{digest}:AVAILABLE")
+              conn.del("#{digest}:RUN:EXISTS")
+              conn.del("#{digest}:RUN:GRABBED")
+              conn.del("#{digest}:RUN:VERSION")
+              conn.del("#{digest}:RUN:AVAILABLE")
+            end
+          end
+        end
+      end
+    end
+
+    def timed
+      start = current_time
+      result = yield
+      elapsed = (current_time - start).round(2)
+      [result, elapsed]
+    end
+
+    def current_time
+      Time.now
+    end
+  end
+end

lib/sidekiq_unique_jobs/exceptions.rb

@@ -15,7 +15,7 @@ def initialize(item)
   # @author Mikael Henriksson <mikael@zoolutions.se>
   class ScriptError < StandardError
     # @param [Symbol] file_name the name of the lua script
-    # @param [Redis::CommandError] ex exception to handle
+    # @param [Redis::CommandError] source_exception exception to handle
     def initialize(file_name:, source_exception:)
       super("Problem compiling #{file_name}. Message: #{source_exception.message}")
     end

lib/sidekiq_unique_jobs/locksmith.rb

@@ -29,7 +29,7 @@ def create
       Scripts.call(
         :create,
         redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, unique_digest],
+        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
         argv: [jid, expiration, API_VERSION, concurrency],
       )
     end
@@ -59,7 +59,7 @@ def delete!
       Scripts.call(
         :delete,
         redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, unique_digest],
+        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
       )
     end
 
@@ -105,7 +105,7 @@ def signal(token = nil)
       Scripts.call(
         :signal,
         redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, unique_digest],
+        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
         argv: [token, expiration],
       )
     end

lib/sidekiq_unique_jobs/util.rb

@@ -6,10 +6,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
   module Util
-    COUNT             = 'COUNT'
     DEFAULT_COUNT     = 1_000
-    EXPIRE_BATCH_SIZE = 100
-    SCAN_METHOD       = 'SCAN'
     SCAN_PATTERN      = '*'
 
     include SidekiqUniqueJobs::Logging
@@ -20,7 +17,7 @@ module Util
     #
     # @param [String] pattern a pattern to scan for in redis
     # @param [Integer] count the maximum number of keys to delete
-    # @return [Array] an array with active unique keys
+    # @return [Array<String>] an array with active unique keys
     def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
       return redis(&:keys) if pattern.nil?
       redis { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }

lib/sidekiq_unique_jobs/web.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+begin
+  require 'sidekiq/web'
+rescue LoadError # rubocop:disable Lint/HandleExceptions
+  # client-only usage
+end
+
+require_relative 'web/helpers'
+
+module SidekiqUniqueJobs
+  # Utility module to help manage unique keys in redis.
+  # Useful for deleting keys that for whatever reason wasn't deleted
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  module Web
+    def self.registered(app) # rubocop:disable Metrics/MethodLength
+      app.helpers do
+        include Web::Helpers
+      end
+
+      app.get '/unique_digests' do
+        @total_size     = Digests.count
+        @filter         = params[:filter] || '*'
+        @filter         = '*' if @filter == ''
+        @count          = (params[:count] || 100).to_i
+        @unique_digests = Digests.all(pattern: @filter, count: @count)
+
+        erb(unique_template(:unique_digests))
+      end
+
+      app.get '/unique_digests/:digest' do
+        @digest = params[:digest]
+        @unique_keys = Util.keys("#{@digest}*", 1000)
+
+        erb(unique_template(:unique_digest))
+      end
+
+      app.get '/unique_digests/:digest/delete' do
+        Digests.del(digest: params[:digest])
+        redirect_to :unique_digests
+      end
+    end
+  end
+end
+
+if defined?(Sidekiq::Web)
+  Sidekiq::Web.register SidekiqUniqueJobs::Web
+  Sidekiq::Web.tabs['Unique Digests'] = 'unique_digests'
+  # Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), 'locales')
+end

lib/sidekiq_unique_jobs/web/helpers.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Web
+    module Helpers
+      VIEW_PATH = File.expand_path('../web/views', __dir__)
+
+      def filtering(pattern, count)
+        SidekiqUniqueJobs::Util.keys(pattern, count)
+      end
+
+      def unique_template(name)
+        File.open(File.join(VIEW_PATH, "#{name}.erb")).read
+      end
+
+      def redirect_to(subpath)
+        if respond_to?(:to)
+          # Sinatra-based web UI
+          redirect to(subpath)
+        else
+          # Non-Sinatra based web UI (Sidekiq 4.2+)
+          redirect "#{root_path}#{subpath}"
+        end
+      end
+
+      def safe_relative_time(time)
+        time = if time.is_a?(Numeric)
+                 Time.at(time)
+               else
+                 Time.parse(time)
+               end
+
+        relative_time(time)
+      end
+    end
+  end
+end