Replace litejob with solid_queue

Because litejob doesn't support retry_on, and solid_queue will be the default active job adapter in Rails 8, so use solid_queue instead.

.gitignore

@@ -10,6 +10,8 @@
 # Ignore the default SQLite database.
 /db/*.sqlite3
 /db/*.sqlite3-*
+/db/**/*.sqlite3
+/db/**/*.sqlite3-*
 
 # Ignore all logfiles and tempfiles.
 /log/*

Gemfile

@@ -22,6 +22,15 @@ gem "jsbundling-rails", "~> 1.1.2"
 # Use Puma as the app server
 gem "puma", "~> 6.4.0"
 
+# Default database
+gem "sqlite3", "~> 1.7.0"
+
+# Background job processing
+gem "solid_queue", github: "basecamp/solid_queue", branch: "main"
+
+# Default stack for cache and pub/sub
+gem "litestack", "~> 0.4.2"
+
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 gem "jbuilder", "~> 2.11.5"
 
@@ -55,9 +64,6 @@ gem "listen", "~> 3.8.0"
 # For daemonize library sync process
 gem "daemons", "~> 1.4.0"
 
-# Default stack for database, cache, background job and pub/sub
-gem "litestack", "~> 0.4.2"
-
 # Optional support for sidekiq as background job
 gem "sidekiq", "~> 7.1.2"
 

Gemfile.lock

@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/basecamp/solid_queue.git
+  revision: d57b8e280dd27b1ffda0e203e0fa2afce0b2bb12
+  branch: main
+  specs:
+    solid_queue (0.2.0)
+      rails (~> 7.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -335,7 +343,7 @@ GEM
     simplecov-lcov (0.8.0)
     simplecov_json_formatter (0.1.4)
     smart_properties (1.17.0)
-    sqlite3 (1.6.9)
+    sqlite3 (1.7.0)
       mini_portile2 (~> 2.8.0)
     sshkit (1.21.5)
       net-scp (>= 1.1.2)
@@ -414,6 +422,8 @@ DEPENDENCIES
   sidekiq (~> 7.1.2)
   simplecov (~> 0.22.0)
   simplecov-lcov (~> 0.8.0)
+  solid_queue!
+  sqlite3 (~> 1.7.0)
   standard (~> 1.25.0)
   standard-rails
   stimulus-rails (~> 1.2.1)

app/jobs/attach_cover_image_from_discogs_job.rb

@@ -6,6 +6,8 @@ class AttachCoverImageFromDiscogsJob < ApplicationJob
   queue_as :default
 
   def perform(imageable, discogs_client = Integrations::Discogs.new)
+    return if imageable.has_cover_image?
+
     image_resource = discogs_client.cover_image(imageable)
     return unless image_resource.present?
 

app/jobs/media_sync_job.rb

@@ -1,7 +1,9 @@
 class MediaSyncJob < ApplicationJob
-  queue_as :default
+  queue_as :critical
 
   def perform(type = :all, file_paths = [])
+    return if Media.syncing?
+
     if type == :all
       Media.sync_all
     else

app/models/concerns/imageable_concern.rb

@@ -6,11 +6,13 @@ module ImageableConcern
   included do
     has_one_attached :cover_image do |attachable|
       attachable.variant :small, resize_to_fill: [200, 200]
-      attachable.variant :medium, resize_to_fill: [300, 300], preprocessed: true
+      attachable.variant :medium, resize_to_fill: [300, 300]
       attachable.variant :large, resize_to_fill: [400, 400]
     end
 
     validate :content_type_of_cover_image
+
+    after_commit :transform_cover_image, if: :has_cover_image?
   end
 
   def has_cover_image?
@@ -26,4 +28,8 @@ def content_type_of_cover_image
       errors.add(:cover_image, :invalid_content_type)
     end
   end
+
+  def transform_cover_image
+    cover_image.variant(:medium).processed
+  end
 end

app/models/media.rb

@@ -105,13 +105,17 @@ def clean_up(file_hashes = [])
     def fetch_external_metadata
       return unless Setting.discogs_token.present?
 
+      jobs = []
+
       Artist.lack_metadata.find_each do |artist|
-        AttachCoverImageFromDiscogsJob.perform_later(artist)
+        jobs << AttachCoverImageFromDiscogsJob.new(artist)
       end
 
       Album.lack_metadata.find_each do |album|
-        AttachCoverImageFromDiscogsJob.perform_later(album)
+        jobs << AttachCoverImageFromDiscogsJob.new(album)
       end
+
+      ActiveJob.perform_all_later(jobs)
     end
   end
 end

app/models/media_file.rb

@@ -43,7 +43,8 @@ def extract_image_from(tag)
       image = tag.images.first
       return unless image.present?
 
-      image_format = Mime::Type.lookup(image[:mime_type]).symbol
+      mime_type = normalize_image_mime_type(image[:mime_type])
+      image_format = Mime::Type.lookup(mime_type).symbol
       return unless image_format.present?
 
       {
@@ -75,5 +76,14 @@ def get_tag_info(file_path)
         end
       end
     end
+
+    def normalize_image_mime_type(mime_type)
+      case mime_type
+      when "image/jpg"
+        "image/jpeg"
+      else
+        mime_type
+      end
+    end
   end
 end

config/application.rb

@@ -36,6 +36,12 @@ class Application < Rails::Application
 
     config.exceptions_app = routes
 
+    config.active_storage.resolve_model_to_route = :rails_storage_proxy
+    config.active_storage.analyzers.delete ActiveStorage::Analyzer::ImageAnalyzer::Vips
+
+    config.solid_queue.silence_polling = true
+    config.solid_queue.preserve_finished_jobs = false
+
     # Configuration for the application, engines, and railties goes here.
     #
     # These settings can be overridden in specific environments using the files

config/database.yml

@@ -1,5 +1,5 @@
 sqlite_default: &sqlite_default
-  adapter: litedb
+  adapter: sqlite3
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   timeout: 5000
 

config/environments/development.rb

@@ -71,7 +71,7 @@
 
   config.action_controller.action_on_unpermitted_parameters = :raise
 
-  config.active_job.queue_adapter = :litejob
+  config.active_job.queue_adapter = :solid_queue
 
   config.after_initialize do
     Bullet.enable = true

config/environments/production.rb

@@ -68,7 +68,7 @@
   config.cache_store = BlackCandy::Config.redis_cache_url.present? ? [:redis_cache_store, {url: BlackCandy::Config.redis_cache_url}] : [:litecache, {path: "storage/production_cache.sqlite3"}]
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  config.active_job.queue_adapter = BlackCandy::Config.redis_sidekiq_url.present? ? :sidekiq : :litejob
+  config.active_job.queue_adapter = BlackCandy::Config.redis_sidekiq_url.present? ? :sidekiq : :solid_queue
 
   # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
   # the I18n.default_locale when a translation cannot be found).

config/puma.rb

@@ -23,15 +23,17 @@
 # Specifies the `pidfile` that Puma will use.
 pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" }
 
+# Allow puma to be restarted by `rails restart` command.
+plugin :tmp_restart
+
+plugin :solid_queue
+
 # Use the `preload_app!` method when specifying a `workers` number.
 # This directive tells Puma to first boot the application and load code
 # before forking the application. This takes advantage of Copy On Write
 # process behavior so workers use less memory.
 preload_app!
 
-# Allow puma to be restarted by `rails restart` command.
-plugin :tmp_restart
-
 on_booted do
   MediaListener.start if Setting.enable_media_listener? && !MediaListener.running?
 end

config/solid_queue.yml

@@ -0,0 +1,19 @@
+default: &default
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+      concurrency_maintenance_interval: 30
+  workers:
+    - queues: [critical, high, default, low]
+      threads: 5
+      processes: 1
+      polling_interval: 0.1
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default

db/migrate/20240122024820_create_solid_queue_tables.solid_queue.rb

@@ -0,0 +1,101 @@
+# This migration comes from solid_queue (originally 20231211200639)
+class CreateSolidQueueTables < ActiveRecord::Migration[7.0]
+  def change
+    create_table :solid_queue_jobs do |t|
+      t.string :queue_name, null: false
+      t.string :class_name, null: false, index: true
+      t.text :arguments
+      t.integer :priority, default: 0, null: false
+      t.string :active_job_id, index: true
+      t.datetime :scheduled_at
+      t.datetime :finished_at, index: true
+      t.string :concurrency_key
+
+      t.timestamps
+
+      t.index [:queue_name, :finished_at], name: "index_solid_queue_jobs_for_filtering"
+      t.index [:scheduled_at, :finished_at], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    create_table :solid_queue_scheduled_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.datetime :scheduled_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [:scheduled_at, :priority, :job_id], name: "index_solid_queue_dispatch_all"
+    end
+
+    create_table :solid_queue_ready_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [:priority, :job_id], name: "index_solid_queue_poll_all"
+      t.index [:queue_name, :priority, :job_id], name: "index_solid_queue_poll_by_queue"
+    end
+
+    create_table :solid_queue_claimed_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.bigint :process_id
+      t.datetime :created_at, null: false
+
+      t.index [:process_id, :job_id]
+    end
+
+    create_table :solid_queue_blocked_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.string :concurrency_key, null: false
+      t.datetime :expires_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [:expires_at, :concurrency_key], name: "index_solid_queue_blocked_executions_for_maintenance"
+    end
+
+    create_table :solid_queue_failed_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.text :error
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_pauses do |t|
+      t.string :queue_name, null: false, index: {unique: true}
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_processes do |t|
+      t.string :kind, null: false
+      t.datetime :last_heartbeat_at, null: false, index: true
+      t.bigint :supervisor_id, index: true
+
+      t.integer :pid, null: false
+      t.string :hostname
+      t.text :metadata
+
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_semaphores do |t|
+      t.string :key, null: false, index: {unique: true}
+      t.integer :value, default: 1, null: false
+      t.datetime :expires_at, null: false, index: true
+
+      t.timestamps
+
+      t.index [:key, :value], name: "index_solid_queue_semaphores_on_key_and_value"
+    end
+
+    add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end