Merge branch 'mr_good_job_its_got_to_be_good' into adventist_dev

Gemfile

@@ -37,6 +37,7 @@ gem 'factory_bot_rails', group: %i[test]
 gem 'fcrepo_wrapper', '~> 0.4', group: %i[development test]
 gem 'flipflop', '~> 2.6.0' # waiting for hyrax 4 upgrade
 gem 'flutie'
+gem 'good_job'
 gem 'hyrax', '~> 3.5.0'
 gem 'hyrax-doi', github: 'samvera-labs/hyrax-doi', branch: 'main'
 gem 'hyrax-iiif_av', github: 'samvera-labs/hyrax-iiif_av', branch: 'main'

Gemfile.lock

@@ -13,6 +13,7 @@ GIT
   branch: upstream_main
   specs:
     hyku_knapsack (0.0.1)
+      good_job
       rails (>= 5.2.0)
 
 GIT
@@ -448,6 +449,8 @@ GEM
     equivalent-xml (0.6.0)
       nokogiri (>= 1.4.3)
     erubi (1.12.0)
+    et-orbi (1.2.7)
+      tzinfo
     ethon (0.16.0)
       ffi (>= 1.15.0)
     excon (0.71.1)
@@ -473,11 +476,23 @@ GEM
     flutie (2.2.0)
     font-awesome-rails (4.7.0.8)
       railties (>= 3.2, < 8.0)
+    fugit (1.8.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     gender_detector (0.1.2)
       unicode_utils (>= 1.3.0)
     geocoder (1.8.1)
     globalid (1.1.0)
       activesupport (>= 5.0)
+    good_job (2.99.0)
+      activejob (>= 5.2.0)
+      activerecord (>= 5.2.0)
+      concurrent-ruby (>= 1.0.2)
+      fugit (>= 1.1)
+      railties (>= 5.2.0)
+      thor (>= 0.14.1)
+      webrick (>= 1.3)
+      zeitwerk (>= 2.0)
     google-apis-core (0.11.0)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.16.2, < 2.a)
@@ -904,6 +919,7 @@ GEM
       nokogiri (~> 1.6)
       rails (>= 5.0, < 7.1)
       rdf
+    raabro (1.4.0)
     racc (1.7.1)
     rack (2.2.8)
     rack-oauth2 (1.21.3)
@@ -1318,6 +1334,7 @@ DEPENDENCIES
   fcrepo_wrapper (~> 0.4)
   flipflop (~> 2.6.0)
   flutie
+  good_job
   hyku_knapsack!
   hyrax (~> 3.5.0)
   hyrax-doi!

bin/worker

@@ -9,4 +9,10 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec sidekiq"
+queue = ENV.fetch('HYRAX_ACTIVE_JOB_QUEUE', 'sidekiq')
+case queue
+when 'sidekiq'
+  exec "echo $DATABASE_URL && bundle exec sidekiq"
+when 'good_job'
+  exec "echo $DATABASE_URL && bundle exec good_job start"
+end

config/environments/development.rb

@@ -4,7 +4,12 @@
   # In the development environment your application's code is reloaded on
   # every request. This slows down response time but is perfect for development
   # since you don't have to restart the web server when you make code changes.
-  config.cache_classes = !!Sidekiq.server?
+  if ENV.fetch('HYRAX_ACTIVE_JOB_QUEUE', 'sidekiq') == 'sidekiq'
+    config.cache_classes = !!Sidekiq.server?
+  else
+    config.cache_classes = false
+  end
+
 
   # Do not eager load code on boot.
   config.eager_load = false

config/initializers/good_job.rb

@@ -0,0 +1,44 @@
+
+# frozen_string_literal: true
+
+if ENV.fetch('HYRAX_ACTIVE_JOB_QUEUE', 'sidekiq') == 'good_job'
+  Rails.application.configure do
+    # Configure options individually...
+    config.good_job.preserve_job_records = true
+    config.good_job.retry_on_unhandled_error = false
+    config.good_job.on_thread_error = ->(exception) { Raven.capture_exception(exception) }
+    config.good_job.execution_mode = :external
+    # config.good_job.queues = '*'
+    config.good_job.shutdown_timeout = 60 # seconds
+    config.good_job.poll_interval = 5
+    # config.good_job.enable_cron = true
+    # config.good_job.cron = { example: { cron: '0 * * * *', class: 'ExampleJob'  } }
+  end
+
+  # Wrapping this in an after_initialize block to ensure that all constants are loaded
+  Rails.application.config.after_initialize do
+    # baseline of 0, higher is sooner
+
+    # Commented out the following two jobs because they were
+    # specfically used for the sdapi ingests.
+    # see sdapi_ingest_script directory and
+    # ref: https://github.com/scientist-softserv/adventist-dl/issues/468
+    # CollectionMembershipJob.priority = 70
+    # UpdateCollectionMembershipJob.priority = 60
+    Bulkrax::ScheduleRelationshipsJob.priority = 50
+    CreateDerivativesJob.priority = 40
+    CharacterizeJob.priority = 30
+    Hyrax::GrantEditToMembersJob.priority = 10
+    ImportUrlJob.priority = 10
+    IngestJob.priority = 10
+    ApplicationJob.priority = 0
+    AttachFilesToWorkJob.priority = -1
+    Bulkrax::ImportWorkJob.priority = -5
+    Bulkrax::ImportFileSetJob.priority = -15
+    Bulkrax::CreateRelationshipsJob.priority = -20
+    Bulkrax::ImporterJob.priority = -20
+    IiifPrint::Jobs::CreateRelationshipsJob.priority = -20
+    ContentDepositEventJob.priority = -50
+    ContentUpdateEventJob.priority = -50
+  end
+end

config/initializers/sidekiq.rb

@@ -1,10 +1,13 @@
-config = YAML.load(ERB.new(IO.read(Rails.root + 'config' + 'redis.yml')).result)[Rails.env].with_indifferent_access
-redis_config = config.merge(thread_safe: true)
+if ENV.fetch('HYRAX_ACTIVE_JOB_QUEUE', 'sidekiq') == 'sidekiq'
 
-Sidekiq.configure_server do |s|
-  s.redis = redis_config
-end
+  config = YAML.load(ERB.new(IO.read(Rails.root + 'config' + 'redis.yml')).result)[Rails.env].with_indifferent_access
+  redis_config = config.merge(thread_safe: true)
+
+  Sidekiq.configure_server do |s|
+    s.redis = redis_config
+  end
 
-Sidekiq.configure_client do |s|
-  s.redis = redis_config
+  Sidekiq.configure_client do |s|
+    s.redis = redis_config
+  end
 end

config/routes.rb

@@ -2,7 +2,9 @@
 
 # OVERRIDE Hyrax 2.9.0 to add featured collection routes
 
-require 'sidekiq/web'
+if ENV.fetch('HYRAX_ACTIVE_JOB_QUEUE', 'sidekiq')
+  require 'sidekiq/web'
+end
 
 Rails.application.routes.draw do # rubocop:disable Metrics/BlockLength
   resources :identity_providers
@@ -14,7 +16,13 @@
   mount Riiif::Engine => 'images', as: :riiif if Hyrax.config.iiif_image_server?
 
   authenticate :user, ->(u) { u.is_superadmin || u.is_admin } do
-    mount Sidekiq::Web => '/jobs'
+    queue = ENV.fetch('HYRAX_ACTIVE_JOB_QUEUE', 'sidekiq')
+    case queue
+      when 'sideki'
+        mount Sidekiq::Web => '/jobs'
+    when 'good_job'
+      mount GoodJob::Engine => '/jobs'
+    end
   end
 
   if ActiveModel::Type::Boolean.new.cast(ENV.fetch('HYKU_MULTITENANT', false))

db/migrate/20230406183814_create_good_jobs.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+class CreateGoodJobs < ActiveRecord::Migration[5.2]
+  def change
+    enable_extension 'pgcrypto'
+
+    create_table :good_jobs, id: :uuid do |t|
+      t.text :queue_name
+      t.integer :priority
+      t.jsonb :serialized_params
+      t.timestamp :scheduled_at
+      t.timestamp :performed_at
+      t.timestamp :finished_at
+      t.text :error
+
+      t.timestamps
+
+      t.uuid :active_job_id
+      t.text :concurrency_key
+      t.text :cron_key
+      t.uuid :retried_good_job_id
+      t.timestamp :cron_at
+    end
+
+    create_table :good_job_processes, id: :uuid do |t|
+      t.timestamps
+      t.jsonb :state
+    end
+
+    add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at"
+    add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at
+    add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at
+    add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished
+    add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at
+    add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true
+    add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id
+    add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at
+  end
+end