Merge pull request #1079 from transitland/sidekiq-queue-adjustments

Sidekiq queue adjustments
transitland · May 3, 2017 · 9c7232f · 9c7232f
2 parents ff1ba80 + c0977a9
commit 9c7232f
Show file tree

Hide file tree

Showing 11 changed files with 38 additions and 23 deletions.
diff --git a/app/workers/changeset_apply_worker.rb b/app/workers/changeset_apply_worker.rb
@@ -1,9 +1,10 @@
 class ChangesetApplyWorker
   include Sidekiq::Worker
-  sidekiq_options unique: :until_and_while_executing,
-                  unique_job_expiration: 60 * 60, # 1 hour
-                  log_duplicate_payload: true,
-                  retry: false
+  sidekiq_options queue: :high,
+                  retry: false,
+                  unique: :until_and_while_executing,
+                  unique_job_expiration: 60 * 60 * 22, # 22 hours
+                  log_duplicate_payload: true
 
   def perform(changeset_id, cachekey)
     log "ChangesetApplyWorker: #{changeset_id}"

diff --git a/app/workers/conflate_stops_with_osm_worker.rb b/app/workers/conflate_stops_with_osm_worker.rb
@@ -1,5 +1,7 @@
 class ConflateStopsWithOsmWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :default,
+                  retry: false
 
   def perform(stop_ids = [])
     Stop.where(id: stop_ids).find_in_batches do |batch_of_stops|

diff --git a/app/workers/feed_activation_worker.rb b/app/workers/feed_activation_worker.rb
@@ -1,5 +1,7 @@
 class FeedActivationWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :default,
+                  retry: false
 
   def perform(feed_onestop_id, feed_version_sha1, import_level)
     log "FeedActivationWorker #{feed_onestop_id}: activating #{feed_version_sha1} at import_level #{import_level}"

diff --git a/app/workers/feed_deactivation_worker.rb b/app/workers/feed_deactivation_worker.rb
@@ -1,5 +1,7 @@
 class FeedDeactivationWorker
   include Sidekiq::Worker
+  sidekiq_options queue: :default,
+                  retry: false
 
   def perform(feed_onestop_id, feed_version_sha1)
     log "FeedDeactivationWorker #{feed_onestop_id}: deactivating #{feed_version_sha1}"

diff --git a/app/workers/feed_eater_schedule_worker.rb b/app/workers/feed_eater_schedule_worker.rb
@@ -1,7 +1,7 @@
 class FeedEaterScheduleWorker
   include Sidekiq::Worker
-
-  sidekiq_options queue: :feed_eater_schedule, retry: false
+  sidekiq_options queue: :feed_eater_schedule,
+                  retry: false
 
   def perform(feed_onestop_id, feed_version_sha1, feed_schedule_import_id, trip_ids, agency_map, route_map, stop_map, rsp_map)
     log "FeedEaterScheduleWorker #{feed_onestop_id}: Importing #{trip_ids.size} trips"

diff --git a/app/workers/feed_eater_worker.rb b/app/workers/feed_eater_worker.rb
@@ -1,11 +1,10 @@
 class FeedEaterWorker
   include Sidekiq::Worker
-
-  sidekiq_options unique: :until_and_while_executing,
-                  unique_job_expiration: 60 * 60, # 1 hour
-                  log_duplicate_payload: true,
-                  queue: :feed_eater,
-                  retry: false
+  sidekiq_options queue: :feed_eater,
+                  retry: false,
+                  unique: :until_and_while_executing,
+                  unique_job_expiration: 60 * 60 * 22, # 22 hours
+                  log_duplicate_payload: true
 
   def perform(feed_onestop_id, feed_version_sha1=nil, import_level=0)
     feed = Feed.find_by!(onestop_id: feed_onestop_id)

diff --git a/app/workers/feed_fetcher_worker.rb b/app/workers/feed_fetcher_worker.rb
@@ -1,8 +1,10 @@
 class FeedFetcherWorker
   include Sidekiq::Worker
-
-  sidekiq_options unique: :until_and_while_executing,
-                  unique_job_expiration: 22 * 60 * 60 # 22 hours
+  sidekiq_options queue: :default,
+                  retry: false,
+                  unique: :until_and_while_executing,
+                  unique_job_expiration: 22 * 60 * 60, # 22 hours
+                  log_duplicate_payload: true
 
   def perform(feed_onestop_id)
     begin

diff --git a/app/workers/feed_info_worker.rb b/app/workers/feed_info_worker.rb
@@ -2,7 +2,8 @@
 
 class FeedInfoWorker
   include Sidekiq::Worker
-  sidekiq_options :retry => false
+  sidekiq_options queue: :high,
+                  retry: false
 
   def perform(url, cachekey)
     @url = url

diff --git a/app/workers/gtfs_statistics_worker.rb b/app/workers/gtfs_statistics_worker.rb
@@ -1,8 +1,10 @@
 class GTFSStatisticsWorker
   include Sidekiq::Worker
-
-  sidekiq_options unique: :until_and_while_executing,
-                  unique_job_expiration: 22 * 60 * 60 # 22 hours
+  sidekiq_options queue: :low,
+                  retry: false,
+                  unique: :until_and_while_executing,
+                  unique_job_expiration: 22 * 60 * 60, # 22 hours
+                  log_duplicate_payload: true
 
   def perform(feed_version_sha1)
     feed_version = FeedVersion.find_by!(sha1: feed_version_sha1)

diff --git a/app/workers/gtfs_validation_worker.rb b/app/workers/gtfs_validation_worker.rb
@@ -1,8 +1,10 @@
 class GTFSValidationWorker
   include Sidekiq::Worker
-
-  sidekiq_options unique: :until_and_while_executing,
-                  unique_job_expiration: 22 * 60 * 60 # 22 hours
+  sidekiq_options queue: :low,
+                  retry: false,
+                  unique: :until_and_while_executing,
+                  unique_job_expiration: 22 * 60 * 60, # 22 hours
+                  log_duplicate_payload: true
 
   def perform(feed_version_sha1)
     feed_version = FeedVersion.find_by!(sha1: feed_version_sha1)

diff --git a/config/sidekiq.yml b/config/sidekiq.yml
@@ -9,6 +9,8 @@ production:
 :process_limits:
   feed_eater_schedule: 1
 :queues:
+  - high
+  - default
   - feed_eater
   - feed_eater_schedule
-  - default
+  - low