Present the entire changelog in its own view (#569)

* Adds functionality to clear a sorted set

* Add view for changelog entries

* Fix deleting a single lock

* Move sidekiq.rb to initializers

* Tweak to minimal implementation

* Add a worker without lock limit

* Mandatory rubocop commit

lib/sidekiq_unique_jobs/changelog.rb

@@ -7,6 +7,13 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   class Changelog < Redis::SortedSet
+    #
+    # @return [Integer] the number of matches to return by default
+    DEFAULT_COUNT = 1_000
+    #
+    # @return [String] the default pattern to use for matching
+    SCAN_PATTERN  = "*"
+
     def initialize
       super(CHANGELOGS)
     end
@@ -34,10 +41,10 @@ def add(message:, digest:, job_id:, script:)
     #
     # @return [Array<Hash>] an array of entries
     #
-    def entries(pattern: "*", count: nil)
+    def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
       options = {}
       options[:match] = pattern
-      options[:count] = count if count
+      options[:count] = count
 
       redis do |conn|
         conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
@@ -53,7 +60,7 @@ def entries(pattern: "*", count: nil)
     #
     # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
     #
-    def page(cursor, pattern: "*", page_size: 100)
+    def page(cursor: 0, pattern: "*", page_size: 100)
       redis do |conn|
         total_size, result = conn.multi do
           conn.zcard(key)

lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -23,6 +23,23 @@ def entries(with_scores: true)
         entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
       end
 
+      #
+      # Adds a value to the sorted set
+      #
+      # @param [Array<Float, String>, String] the values to add
+      #
+      # @return [Boolean, Integer] <description>
+      #
+      def add(values)
+        redis do |conn|
+          if values.is_a?(Array)
+            conn.zadd(key, values)
+          else
+            conn.zadd(key, now_f, values)
+          end
+        end
+      end
+
       #
       # Return the zrak of the member
       #
@@ -45,6 +62,16 @@ def score(member)
         redis { |conn| conn.zscore(key, member) }
       end
 
+      #
+      # Clears the sorted set from all entries
+      #
+      #
+      # @return [Integer] number of entries removed
+      #
+      def clear
+        redis { |conn| conn.zremrangebyrank(key, 0, count) }
+      end
+
       #
       # Returns the count for this sorted set
       #

lib/sidekiq_unique_jobs/web.rb

@@ -13,6 +13,23 @@ def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
         include Web::Helpers
       end
 
+      app.get "/changelogs" do
+        @filter         = params[:filter] || "*"
+        @filter         = "*" if @filter == ""
+        @count          = (params[:count] || 100).to_i
+        @current_cursor = params[:cursor]
+        @prev_cursor    = params[:prev_cursor]
+        @pagination     = { pattern: @filter, cursor: @current_cursor, page_size: @count }
+        @total_size, @next_cursor, @changelogs = changelog.page(**@pagination)
+
+        erb(unique_template(:changelogs))
+      end
+
+      app.get "/changelogs/delete_all" do
+        changelog.clear
+        redirect_to :changelogs
+      end
+
       app.get "/locks" do
         @filter         = params[:filter] || "*"
         @filter         = "*" if @filter == ""
@@ -58,7 +75,8 @@ def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
   require "sidekiq/web" unless defined?(Sidekiq::Web)
 
   Sidekiq::Web.register(SidekiqUniqueJobs::Web)
-  Sidekiq::Web.tabs["Locks"] = "locks"
+  Sidekiq::Web.tabs["Locks"]      = "locks"
+  Sidekiq::Web.tabs["Changelogs"] = "changelogs"
   Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
 rescue NameError, LoadError => ex
   SidekiqUniqueJobs.logger.error(ex)

lib/sidekiq_unique_jobs/web/helpers.rb

@@ -10,7 +10,7 @@ module Web
     module Helpers
       #
       # @return [String] the path to gem specific views
-      VIEW_PATH    = File.expand_path("../web/views", __dir__)
+      VIEW_PATH    = File.expand_path("../web/views", __dir__).freeze
       #
       # @return [Array<String>] safe params
       SAFE_CPARAMS = %w[cursor prev_cursor].freeze
@@ -25,7 +25,18 @@ module Helpers
       # @return [String] the file contents of the template
       #
       def unique_template(name)
-        File.open(File.join(VIEW_PATH, "#{name}.erb")).read
+        File.open(unique_filename(name)).read
+      end
+
+      #
+      # Construct template file name
+      #
+      # @param [Symbol] name the name of the template
+      #
+      # @return [String] the full name of the file
+      #
+      def unique_filename(name)
+        File.join(VIEW_PATH, "#{name}.erb")
       end
 
       #
@@ -38,6 +49,16 @@ def digests
         @digests ||= SidekiqUniqueJobs::Digests.new
       end
 
+      #
+      # The collection of changelog entries
+      #
+      #
+      # @return [SidekiqUniqueJobs::Digests] the sorted set with digests
+      #
+      def changelog
+        @changelog ||= SidekiqUniqueJobs::Changelog.new
+      end
+
       #
       # Creates url safe parameters
       #

lib/sidekiq_unique_jobs/web/views/changelogs.erb

@@ -0,0 +1,53 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>
+    <%= t('Changelog Entries') %>
+    </h3>
+  </div>
+  <form action="<%= root_path %>changelogs" class="form form-inline" method="get">
+    <%= csrf_tag %>
+    <input name="filter" class="form-control" type="text" value="<%= @filter %>" />
+    <button class="btn btn-default" type="submit">
+    <%= t('Filter') %>
+    </button>
+  </form>
+  <% if @changelogs.any? && @total_size > @count.to_i %>
+  <div class="col-sm-4">
+    <%= erb unique_template(:_paging), locals: { url: "#{root_path}changelogs" } %>
+  </div>
+  <% end %>
+</header>
+<% if @changelogs.any? %>
+<div class="table_container">
+  <form action="<%= root_path %>changelogs/delete_all" method="get">
+    <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+  <br/>
+  <table class="table table-striped table-bordered table-hover">
+    <thead>
+      <tr>
+        <th><%= t('Time') %></th>
+        <th><%= t('Digest') %></th>
+        <th><%= t('Script') %></th>
+        <th><%= t('JID') %></th>
+        <th><%= t('Prev JID') %></th>
+        <th><%= t('Message') %></th>
+      </tr>
+    </thead>
+    <tbody>
+    <% @changelogs.each do |changelog| %>
+      <tr>
+        <td><%= safe_relative_time(changelog["time"]) %></td>
+        <td><%= changelog["digest"] %></td>
+        <td><%= changelog["script"] %></td>
+        <td><%= changelog["prev_jid"] %></td>
+        <td><%= changelog["message"] %></th>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+  <form action="<%= root_path %>changelogs/delete_all" method="get">
+    <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+</div>
+<% end %>

lib/sidekiq_unique_jobs/web/views/locks.erb

@@ -32,7 +32,7 @@
     <% @locks.each do |lock| %>
     <tr>
       <td>
-        <form action="<%= root_path %>locks/<%= lock.key %>" method="get">
+        <form action="<%= root_path %>locks/<%= lock.key %>/delete" method="get">
           <%= csrf_tag %>
           <input name="lock" value="<%= h lock.key %>" type="hidden" />
           <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />

myapp/Gemfile

@@ -4,7 +4,6 @@ source "https://rubygems.org"
 
 ruby "2.7.2"
 
-gem "apartment-sidekiq"
 gem "bigdecimal"
 gem "coverband"
 gem "devise"
@@ -15,10 +14,7 @@ gem "puma"
 gem "rack-protection"
 gem "rails", ">= 6.0"
 gem "redis"
-gem "sidekiq", "~> 6.1"
-gem "sidekiq-cron"
-gem "sidekiq-global_id"
-gem "sidekiq-status"
+gem "sidekiq", "6.1.2"
 gem "sidekiq-unique-jobs", path: ".."
 gem "sinatra"
 gem "slim-rails"

myapp/app/workers/status_worker.rb

@@ -2,7 +2,6 @@
 
 class StatusWorker
   include Sidekiq::Worker
-  include Sidekiq::Status::Worker
 
   sidekiq_options lock: :until_executed
 

myapp/app/workers/until_executed_job.rb

@@ -6,7 +6,6 @@ class UntilExecutedJob
   sidekiq_options lock: :until_executed,
                   lock_info: true,
                   lock_timeout: 0,
-                  lock_ttl: 0,
                   lock_limit: 5
 
   def perform

myapp/app/workers/until_executed_worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class UntilExecutedWorker
+  include Sidekiq::Worker
+
+  sidekiq_options lock: :until_executed,
+                  lock_info: true,
+                  lock_timeout: 0
+
+  def perform
+    logger.info("cowboy")
+    sleep(1) # hardcore processing
+    logger.info("beebop")
+  end
+end

myapp/config/sidekiq.rb → myapp/config/initializers/sidekiq.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "sidekiq"
+require "sidekiq-unique-jobs"
+
 Redis.exists_returns_integer = false
 
 REDIS = Redis.new(url: ENV["REDIS_URL"])
@@ -9,44 +12,52 @@
   retry: true,
 }
 
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Client
+  end
+end
+
 Sidekiq.configure_client do |config|
   config.redis = { url: ENV["REDIS_URL"], driver: :hiredis }
 
   config.client_middleware do |chain|
-    chain.add Sidekiq::GlobalId::ClientMiddleware
-    chain.add Apartment::Sidekiq::Middleware::Client
     chain.add SidekiqUniqueJobs::Middleware::Client
-    chain.add Sidekiq::Status.ClientMiddleware, expiration: 30.minutes
   end
 end
 
 Sidekiq.configure_server do |config|
   config.redis = { url: ENV["REDIS_URL"], driver: :hiredis }
 
   config.server_middleware do |chain|
-    chain.add Sidekiq::Status.ServerMiddleware, expiration: 30.minutes
-    chain.add Sidekiq::GlobalId::ServerMiddleware
-    chain.add Apartment::Sidekiq::Middleware::Server
     chain.add SidekiqUniqueJobs::Middleware::Server
   end
 
+  config.client_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Client
+  end
+
   config.error_handlers << ->(ex, ctx_hash) { p ex, ctx_hash }
-  config.death_handlers << lambda do |job, _ex|
+  config.death_handlers << lambda do |job, ex|
     digest = job["lock_digest"]
+    p ex
+    p digest
+    p job
     SidekiqUniqueJobs::Digests.new.delete_by_digest(digest) if digest
   end
 end
 
 Sidekiq.logger       = Sidekiq::Logger.new($stdout)
-Sidekiq.logger.level = :info
+Sidekiq.logger.level = :debug
 Sidekiq.log_format = :json if Sidekiq.respond_to?(:log_format)
 SidekiqUniqueJobs.configure do |config|
-  config.debug_lua       = false
+  config.debug_lua       = true
+  config.enabled         = true
   config.lock_info       = true
   config.logger          = Sidekiq.logger
   config.max_history     = 10_000
   config.reaper          = :lua
-  config.reaper_count    = 10_000
+  config.reaper_count    = 1_000
   config.reaper_interval = 10
   config.reaper_timeout  = 5
 end

spec/sidekiq_unique_jobs/changelog_spec.rb

@@ -24,6 +24,33 @@
     end
   end
 
+  describe "#clear" do
+    subject(:clear) { entity.clear }
+
+    context "with entries" do
+      before do
+        entity.add(
+          message: "Added from test",
+          job_id: job_id,
+          digest: digest,
+          script: __FILE__.to_s,
+        )
+      end
+
+      it "clears out all entries" do
+        expect { clear }.to change { entity.entries.size }.by(-1)
+        expect(clear).to be == 1
+      end
+    end
+
+    context "without entries" do
+      it "returns 0 (zero)" do
+        expect { clear }.not_to change { entity.entries.size }
+        expect(clear).to be == 0
+      end
+    end
+  end
+
   describe "#exist?" do
     subject(:exist?) { entity.exist? }
 
@@ -159,7 +186,7 @@
   end
 
   describe "#page" do
-    subject(:page) { entity.page(cursor, pattern: pattern, page_size: page_size) }
+    subject(:page) { entity.page(cursor: cursor, pattern: pattern, page_size: page_size) }
 
     let(:cursor)    { 0 }
     let(:pattern)   { "*" }