From c0637fa6053ff8c1185ce86cf2221d04acce9997 Mon Sep 17 00:00:00 2001 From: Joshua Liebowitz Date: Mon, 25 Jun 2018 15:13:03 -0700 Subject: [PATCH] [WIP] GitRepos get individual work queues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we’ve made progress on https://github.com/ruby-git/ruby-git/pull/372 We can limit mutexes to be individual repo-scoped instead of 1 giant mutex for all. --- app/features-json/build_json_controller.rb | 1 - app/features/build_runner/build_runner.rb | 6 +++--- app/features/project/project_controller.rb | 1 - app/shared/models/git_repo.rb | 10 +++++----- app/workers/github_worker_base.rb | 1 - launch.rb | 5 +---- 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/app/features-json/build_json_controller.rb b/app/features-json/build_json_controller.rb index 2594f92a..ca24929b 100644 --- a/app/features-json/build_json_controller.rb +++ b/app/features-json/build_json_controller.rb @@ -78,7 +78,6 @@ class BuildJSONController < APIController sha: current_sha, github_service: FastlaneCI::GitHubService.new(provider_credential: current_user_provider_credential), notification_service: FastlaneCI::Services.notification_service, - work_queue: FastlaneCI::GitRepo.git_action_queue, # using the git repo queue because of https://github.com/ruby-git/ruby-git/issues/355 trigger: project.find_triggers_of_type(trigger_type: :manual).first, git_fork_config: git_fork_config ) diff --git a/app/features/build_runner/build_runner.rb b/app/features/build_runner/build_runner.rb index 683148e9..c8927811 100644 --- a/app/features/build_runner/build_runner.rb +++ b/app/features/build_runner/build_runner.rb @@ -58,7 +58,7 @@ def initialize( sha:, github_service:, notification_service:, - work_queue:, + work_queue: nil, trigger:, git_fork_config:, local_build_folder: nil @@ -86,8 +86,6 @@ def initialize( # TODO: provider credential should determine what exact CodeHostingService gets instantiated @code_hosting_service = github_service - @work_queue = work_queue - prepare_build_object(trigger: trigger) local_folder = @local_build_folder || File.join(project.local_repo_path, "builds", sha) @@ -98,6 +96,8 @@ def initialize( notification_service: notification_service, async_start: false ) + + @work_queue = @repo.git_action_queue end # Access the build number of that specific BuildRunner diff --git a/app/features/project/project_controller.rb b/app/features/project/project_controller.rb index a9c528ea..47a7712f 100644 --- a/app/features/project/project_controller.rb +++ b/app/features/project/project_controller.rb @@ -72,7 +72,6 @@ class ProjectController < AuthenticatedControllerBase sha: current_sha, github_service: FastlaneCI::GitHubService.new(provider_credential: current_github_provider_credential), notification_service: FastlaneCI::Services.notification_service, - work_queue: FastlaneCI::GitRepo.git_action_queue, # using the git repo queue because of https://github.com/ruby-git/ruby-git/issues/355 trigger: project.find_triggers_of_type(trigger_type: :manual).first, git_fork_config: git_fork_config, local_build_folder: checkout_folder diff --git a/app/shared/models/git_repo.rb b/app/shared/models/git_repo.rb index 28ae9390..d4321065 100644 --- a/app/shared/models/git_repo.rb +++ b/app/shared/models/git_repo.rb @@ -70,6 +70,9 @@ class GitRepo # @return [proc(GitRepo)] attr_accessor :callback + # Each repo has a serial queue for actions that need to be ordered + attr_reader :git_action_queue + class << self def pushes_disabled? push_state = ENV["FASTLANE_CI_DISABLE_PUSHES"] @@ -81,8 +84,6 @@ def pushes_disabled? return true end - attr_reader :git_action_queue - # Loads the octokit cache stack for speed-up calls to github service. # As explained in: https://github.com/octokit/octokit.rb#caching def load_octokit_cache_stack @@ -96,8 +97,6 @@ def load_octokit_cache_stack end end - @git_action_queue = TaskQueue::TaskQueue.new(name: "GitRepo task queue") - # Initializer for GitRepo class # @param git_config [GitConfig] # @param provider_credential [ProviderCredential] @@ -114,6 +113,7 @@ def initialize( callback: nil, notification_service: ) + @git_action_queue = TaskQueue::TaskQueue.new(name: "GitRepo task queue for #{local_folder}") GitRepo.load_octokit_cache_stack logger.debug("Creating repo in #{local_folder} for a copy of #{git_config.git_url}") @@ -614,7 +614,7 @@ def status # `ensure_block`: block that you want executed after the `&block` finishes executed, even on error def git_action_with_queue(ensure_block: nil, &block) git_task = TaskQueue::Task.new(work_block: block, ensure_block: ensure_block) - GitRepo.git_action_queue.add_task_async(task: git_task) + git_action_queue.add_task_async(task: git_task) return git_task end diff --git a/app/workers/github_worker_base.rb b/app/workers/github_worker_base.rb index bef04f70..0d5344dc 100644 --- a/app/workers/github_worker_base.rb +++ b/app/workers/github_worker_base.rb @@ -72,7 +72,6 @@ def create_and_queue_build_task(sha:, trigger:, git_fork_config: nil, notificati sha: current_sha, github_service: github_service, notification_service: notification_service, - work_queue: FastlaneCI::GitRepo.git_action_queue, # using the git repo queue because of https://github.com/ruby-git/ruby-git/issues/355 git_fork_config: git_fork_config, trigger: trigger ) diff --git a/launch.rb b/launch.rb index 6a9295f7..8037a965 100644 --- a/launch.rb +++ b/launch.rb @@ -3,7 +3,6 @@ require_relative "app/shared/logging_module" require_relative "app/shared/models/job_trigger" require_relative "app/shared/models/git_fork_config" -require_relative "app/shared/models/git_repo" # for GitRepo.git_action_queue module FastlaneCI # Launch is responsible for spawning up the whole @@ -266,7 +265,7 @@ def self.run_pending_github_builds(projects: nil, github_service: nil) github_service: github_service, notification_service: Services.notification_service, # using the git repo queue because of https://github.com/ruby-git/ruby-git/issues/355 - work_queue: FastlaneCI::GitRepo.git_action_queue, + work_queue: nil, git_fork_config: git_fork_config, trigger: project.find_triggers_of_type(trigger_type: :commit).first ) @@ -318,8 +317,6 @@ def self.enqueue_builds_for_open_github_prs_with_no_status(projects: nil, github sha: open_pr.current_sha, github_service: github_service, notification_service: Services.notification_service, - # using the git repo queue because of https://github.com/ruby-git/ruby-git/issues/355 - work_queue: FastlaneCI::GitRepo.git_action_queue, git_fork_config: git_fork_config, trigger: project.find_triggers_of_type(trigger_type: :commit).first )