diff --git a/Gemfile b/Gemfile index d450d59..d368327 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ ruby '3.2.0' # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main" gem 'rails', '~> 7.0.4' +gem 'async' gem 'bcrypt_pbkdf' gem 'ed25519' diff --git a/Gemfile.lock b/Gemfile.lock index c2182d2..ff991f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -71,6 +71,11 @@ GEM airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) + async (2.6.4) + console (~> 1.10) + fiber-annotation + io-event (~> 1.1) + timers (~> 4.1) bcrypt_pbkdf (1.1.0) bindex (0.8.1) bootsnap (1.16.0) @@ -103,6 +108,9 @@ GEM xpath (~> 3.2) coderay (1.1.3) concurrent-ruby (1.2.2) + console (1.23.2) + fiber-annotation + fiber-local coveralls_reborn (0.27.0) simplecov (~> 0.22.0) term-ansicolor (~> 1.7) @@ -118,6 +126,8 @@ GEM dry-cli (1.0.0) ed25519 (1.3.0) erubi (1.12.0) + fiber-annotation (0.2.0) + fiber-local (1.0.0) globalid (1.1.0) activesupport (>= 5.0) i18n (1.12.0) @@ -126,6 +136,7 @@ GEM actionpack (>= 6.0.0) railties (>= 6.0.0) io-console (0.6.0) + io-event (1.3.2) irb (1.6.3) reline (>= 0.3.0) jbuilder (2.11.5) @@ -295,6 +306,7 @@ GEM tins (~> 1.0) thor (1.2.1) timeout (0.3.2) + timers (4.3.5) tins (1.32.1) sync turbo-rails (1.4.0) @@ -330,6 +342,7 @@ PLATFORMS x86_64-linux DEPENDENCIES + async bcrypt_pbkdf bootsnap capistrano (~> 3.10) diff --git a/app/services/card_image_loading_service.rb b/app/services/card_image_loading_service.rb index 3ed87c9..4b8d3d4 100644 --- a/app/services/card_image_loading_service.rb +++ b/app/services/card_image_loading_service.rb @@ -2,6 +2,8 @@ require 'ruby-progressbar' require 'ruby-progressbar/outputs/null' +require 'async/semaphore' +require 'async/barrier' # Class for card image loading service class CardImageLoadingService @@ -13,16 +15,26 @@ def initialize(logger: nil, suppress_progress: false) end def import - (1..22).each { |disk| import_disk(disk) } + Sync do + (1..22).each { |disk| import_disk(disk) } + end end def import_disk(disk) logger.info("Fetching disk #{disk} file list") filenames = disk_array(disk) - progress = progress_bar(filenames.count) - filenames.each do |file_name| - progress.increment - find_or_create_card_image(file_name) + barrier = Async::Barrier.new + Sync do + progress = progress_bar(filenames.count) + semaphore = Async::Semaphore.new(10, parent: barrier) + filenames.map do |file_name| + semaphore.async do + progress.increment + find_or_create_card_image(file_name) + end + end.map(&:wait) + ensure + barrier.stop end end