Fetch files asynchronously

This uses the `async` gem to parallelize fetching files from the GitHub API. The app is heavily IO-bound, so this is a good way to speed it up. In my tests, this decreased the time to print results from 25s to around 2s. Note: We're not taking advantage of the FiberScheduler here, because we would need Ruby 3.1 to run async 2.x.
MatheusRich · Sep 2, 2023 · 067298b · 067298b
1 parent dd9b22f
commit 067298b
Show file tree

Hide file tree

Showing 5 changed files with 136 additions and 48 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -2,6 +2,7 @@ PATH
   remote: .
   specs:
     end_of_life (0.3.0)
+      async
       dry-monads (~> 1.3)
       octokit (~> 4.22)
       pastel (~> 0.8.0)
@@ -11,68 +12,84 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
+    async (1.31.0)
+      console (~> 1.10)
+      nio4r (~> 2.3)
+      timers (~> 4.1)
+    base64 (0.1.1)
     climate_control (1.0.1)
     concurrent-ruby (1.2.2)
+    console (1.23.2)
+      fiber-annotation
+      fiber-local
     crack (0.4.5)
       rexml
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     docile (1.4.0)
-    dry-core (1.0.0)
+    dry-core (1.0.1)
       concurrent-ruby (~> 1.0)
       zeitwerk (~> 2.6)
     dry-monads (1.6.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 1.0, < 2)
       zeitwerk (~> 2.6)
-    faraday (2.7.4)
+    faraday (2.7.10)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
+    fiber-annotation (0.2.0)
+    fiber-local (1.0.0)
     hashdiff (1.0.1)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
+    lint_roller (1.1.0)
+    nio4r (2.5.9)
     octokit (4.25.1)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
-    parallel (1.22.1)
-    parser (3.2.2.0)
+    parallel (1.23.0)
+    parser (3.2.2.3)
       ast (~> 2.4.1)
+      racc
     pastel (0.8.0)
       tty-color (~> 0.5)
-    public_suffix (4.0.6)
+    public_suffix (5.0.3)
+    racc (1.7.1)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.7.0)
-    rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    regexp_parser (2.8.1)
+    rexml (3.2.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.48.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    rubocop (1.56.2)
+      base64 (~> 0.1.1)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.3)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
+    rubocop-ast (1.29.0)
       parser (>= 3.2.1.0)
-    rubocop-performance (1.16.0)
+    rubocop-performance (1.19.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.13.0)
@@ -85,16 +102,25 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    standard (1.26.0)
+    simplecov_json_formatter (0.1.4)
+    standard (1.31.0)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (~> 1.48.1)
-      rubocop-performance (~> 1.16.0)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.56.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.2)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.2.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.19.0)
     strings (0.2.1)
       strings-ansi (~> 0.2)
       unicode-display_width (>= 1.5, < 3.0)
       unicode_utils (~> 1.4)
     strings-ansi (0.2.0)
+    timers (4.3.5)
     tty-color (0.6.0)
     tty-cursor (0.7.1)
     tty-screen (0.8.1)
@@ -107,16 +133,14 @@ GEM
     unicode-display_width (2.4.2)
     unicode_utils (1.4.0)
     vcr (6.0.0)
-    webmock (3.13.0)
-      addressable (>= 2.3.6)
+    webmock (3.19.1)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    zeitwerk (2.6.7)
+    zeitwerk (2.6.11)
 
 PLATFORMS
-  x86_64-darwin-20
   x86_64-darwin-22
-  x86_64-linux
 
 DEPENDENCIES
   climate_control (~> 1.0.1)

diff --git a/end_of_life.gemspec b/end_of_life.gemspec
@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "async"
   spec.add_dependency "dry-monads", "~> 1.3"
   spec.add_dependency "octokit", "~> 4.22"
   spec.add_dependency "pastel", "~> 0.8.0"

diff --git a/lib/end_of_life.rb b/lib/end_of_life.rb
@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "async"
 require "dry-monads"
 require "json"
 require "octokit"
@@ -59,7 +60,11 @@ def fetch_repositories(options)
 
     def filter_repositories_with_end_of_life(repositories, max_eol_date:)
       with_loading_spinner("Searching for EOL Ruby in repositories...") do
-        repositories.filter { |repo| repo.eol_ruby?(at: max_eol_date) }
+        Sync do
+          repositories
+            .tap { |repos| repos.map { |repo| Async { repo.ruby_version } }.map(&:wait) }
+            .filter { |repo| repo.eol_ruby?(at: max_eol_date) }
+        end
       end
     end
 

diff --git a/lib/end_of_life/repository.rb b/lib/end_of_life/repository.rb
@@ -81,15 +81,22 @@ def ruby_version
     def ruby_versions
       return @ruby_versions if defined?(@ruby_versions)
 
-      @ruby_versions = begin
-        ruby_version_files = [
-          fetch_file(".ruby-version"),
-          fetch_file("Gemfile"),
-          fetch_file("Gemfile.lock"),
-          fetch_file(".tool-versions")
-        ].compact
-
-        ruby_version_files.filter_map { |file| parse_version_file(file) }
+      @ruby_versions = fetch_ruby_version_files.filter_map { |file|
+        parse_version_file(file)
+      }
+    end
+
+    POSSIBLE_RUBY_VERSION_FILES = [
+      ".ruby-version",
+      "Gemfile.lock",
+      "Gemfile",
+      ".tool-versions"
+    ]
+    def fetch_ruby_version_files
+      Sync do
+        POSSIBLE_RUBY_VERSION_FILES
+          .map { |file_path| Async { fetch_file(file_path) } }
+          .filter_map(&:wait)
       end
     end
 

diff --git a/spec/end_of_life/repository_spec.rb b/spec/end_of_life/repository_spec.rb
@@ -286,6 +286,51 @@
 
         expect(client).to have_received(:contents).with("thoughtbot/paperclip", path: ".tool-versions")
       end
+
+      it "fetches files asynchronously" do
+        seconds_of_sleep = 1
+        sleepy_github = build_client(
+          repo: "thoughtbot/paperclip",
+          contents: {
+            ".ruby-version" => {
+              content: lambda {
+                sleep(seconds_of_sleep)
+                nil
+              }
+            },
+            ".tool-versions" => {
+              content: lambda {
+                sleep(seconds_of_sleep)
+                "ruby 2.5.0"
+              }
+            },
+            "Gemfile" => {
+              content: lambda {
+                sleep(seconds_of_sleep)
+                nil
+              }
+            },
+            "Gemfile.lock" => {
+              content: lambda {
+                sleep(seconds_of_sleep)
+                nil
+              }
+            }
+          }
+        )
+        repo = EndOfLife::Repository.new(
+          full_name: "thoughtbot/paperclip",
+          url: "https://github.com/thoughtbot/paperclip",
+          github_client: sleepy_github
+        )
+
+        t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        repo.ruby_version
+        total_elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0
+
+        overhead = 0.1
+        expect(total_elapsed_time).to be_within(overhead).of(seconds_of_sleep)
+      end
     end
 
     private
@@ -307,15 +352,21 @@ def build_client(repo: nil, contents: [], search_results: [])
         if config[:content] == Octokit::NotFound
           allow(client).to receive(:contents).with(repo, path: path).and_raise(Octokit::NotFound)
         else
-          allow(client).to receive(:contents).with(repo, path: path).and_return(
+          allow(client).to receive(:contents).with(repo, path: path) do
             if config[:content]
+              file_content = if config[:content].is_a?(Proc)
+                config[:content].call
+              else
+                config[:content]
+              end
+
               OpenStruct.new(
-                content: encoder.call(config[:content]),
+                content: encoder.call(file_content) || "",
                 name: path,
                 encoding: config[:encoding]
               )
             end
-          )
+          end
         end
       end