Permalink
Browse files

Don't do threading from MetaRepo.

This is just really innefficient, considering that each thread is
making grit shell out to a different process.
  • Loading branch information...
cespare committed Oct 4, 2012
1 parent afddc08 commit 5749fae57e0b9caa0c015cf422c7a2cfc578eca7
Showing with 8 additions and 24 deletions.
  1. +1 −1 Rakefile
  2. +7 −23 lib/meta_repo.rb
View
@@ -40,4 +40,4 @@ namespace :resque do
task :require_resque_tasks do
Dir.glob("resque_jobs/*.rb").each { |file_name| require file_name }
end
-end
+end
View
@@ -101,8 +101,7 @@ def search_options_match_commit?(repo_id_or_name, commit_sha, search_options)
repos = search_options[:repos].blank? ? @repos : repos_which_match(search_options[:repos])
commit_matches_search = false
- # TODO(philc): Abort searches in other threads when we find a match.
- parallel_each_repos(repos) do |repo, mutex|
+ repos.each do |repo|
commit_ids = GitHelper.rev_list(repo, git_command_options).map(&:sha)
commit_matches_search = true if commit_ids.include?(grit_commit.sha)
end
@@ -211,29 +210,27 @@ def commits_from_repos(repos, git_command_options, limit, retain = :first, commi
git_command_options[:after] += 1 if git_command_options[:after]
commits = []
- parallel_each_repos(repos) do |repo, mutex|
+ repos.each do |repo|
local_results = commits_from_repo(repo, git_command_options, limit, retain, commit_filter_proc)
# If two commits have the same timestamp, we want to order them as they were originally ordered by
# GitHelper.commits_with_limit. We could just sort by timestamp if Ruby's sort was stable, but it's not.
# Instead, we must remember the array position of each commit so that we can use this later to sort.
- commit_tuples = local_results.each_with_index.map do |commit, i|
+ commits += local_results.each_with_index.map do |commit, i|
[commit, [commit.timestamp, commit.repo_name, i]]
end
- mutex.synchronize { commits += commit_tuples }
end
# Hokay, now let's add in all the boundary commits (if necessary)
boundary_commits = []
if original_timestamp
git_command_options[:before] = git_command_options[:after] = original_timestamp
- parallel_each_repos(repos) do |repo, mutex|
+ repos.each do |repo|
# Hopefully there aren't > 1000 commits with a single timestamp...
local_results = commits_from_repo(repo, git_command_options, 1000, retain, commit_filter_proc)
- commit_tuples = local_results.each_with_index.map do |commit, i|
+ boundary_commits += local_results.each_with_index.map do |commit, i|
[commit, [commit.timestamp, commit.repo_name, i]]
end
- mutex.synchronize { boundary_commits += commit_tuples }
end
end
@@ -308,25 +305,12 @@ def count_commits_to_token(search_options, token)
count = 0
# TODO(caleb): Fix the case where we've paged back > 10000 commits into a single repo.
- parallel_each_repos(repos) do |repo, mutex|
- local_count = GitHelper.commits_with_limit(repo, git_command_options, 10_000, :count, :first)
- mutex.synchronize { count += local_count }
+ repos.each do |repo|
+ count += GitHelper.commits_with_limit(repo, git_command_options, 10_000, :count, :first)
end
count
end
- # Perform some operation with a thread for each repo.
- # Caller gets a thread-local repo and a mutex.
- def parallel_each_repos(repos, &block)
- mutex = Mutex.new
- threads = repos.map do |r|
- Thread.new(r) do |repo|
- yield repo, mutex
- end
- end
- threads.each(&:join)
- end
-
# Compare two commit tuples: [timestamp, repo_name, index]
# (Index represents the git ordering). We want to order by decreasing timestamp, and break ties by
# increasing (repo, index). This preserves the git order nicely across commits spanning multiple repos.

0 comments on commit 5749fae

Please sign in to comment.