Merge pull request #24 from avsej/gerrit-speedup

speed up gerrit grabber a bit, from avsej

Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem 'couchbase'
+gem 'dalli'
+gem 'octokit'
+gem 'nokogiri'
+gem 'yam'

Gemfile.lock

@@ -0,0 +1,49 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.7)
+    bcrypt (3.1.10)
+    bcrypt-ruby (3.1.5)
+      bcrypt (>= 3.1.3)
+    connection_pool (2.1.1)
+    couchbase (1.3.11)
+      connection_pool (>= 1.0.0, <= 3.0.0)
+      multi_json (~> 1.0)
+      yaji (~> 0.3, >= 0.3.2)
+    dalli (2.7.2)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    mime-types (2.4.3)
+    multi_json (1.10.1)
+    multipart-post (2.0.0)
+    netrc (0.10.2)
+    nokogiri (1.6.6.2)
+    oauth2-client (2.0.0)
+      addressable (~> 2.3)
+      bcrypt-ruby (~> 3.0)
+    octokit (3.8.0)
+      sawyer (~> 0.6.0, >= 0.5.3)
+    oj (2.11.4)
+    rest-client (1.7.3)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    sawyer (0.6.0)
+      addressable (~> 2.3.5)
+      faraday (~> 0.8, < 0.10)
+    yaji (0.3.5)
+    yam (2.4.1)
+      addressable (~> 2.3)
+      multi_json (~> 1.10)
+      oauth2-client (~> 2.0)
+      oj (~> 2.0)
+      rest-client (~> 1.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  couchbase
+  dalli
+  nokogiri
+  octokit
+  yam

README.md

@@ -40,10 +40,8 @@ And, at least ruby 1.9.3...
 
     rvm use 1.9.3 --default
 
-    gem install dalli
-    gem install octokit
-    gem install nokogiri
-    gem install yam
+    gem install bundler
+    bundle install
 
 To get leveldb...
 

grab-gerrit

@@ -4,36 +4,39 @@
 # couchbase bucket with the latest updates.
 
 require 'cgi'
-require 'dalli'
+require 'couchbase'
 require 'json'
 require 'net/http'
 require 'uri'
+require 'optparse'
+require 'stringio'
 
-args = {}
-ARGV.each do |arg|
-  k, v = arg.split('=')
-  args[k] = v or ""
-end
-
-mc_addr = 'localhost:11211'
-mc_addr = args['--mc-addr'] if args.include?('--mc-addr')
+options = {
+  couchbase: 'http://localhost:8091',
+  gerrit: 'http://review.couchbase.org'
+}
 
-gerrit_url = 'http://review.couchbase.org'
-gerrit_url = args['--gerrit_url'] if args.include?('--gerrit_url')
+OptionParser.new do |opts|
+  opts.banner = 'Usage: grab-gerrit [options]'
+  opts.on('-c', '--couchbase=URL', "URL of Couchbase (#{options[:couchbase]})") do |val|
+    options[:couchbase] = val
+  end
+  opts.on('-g', '--gerrit=URL', "URL of Gerrit (#{options[:gerrit]})") do |val|
+    options[:gerrit] = val
+  end
+end.parse!
 
 # -----------------------------------------------------------------------------
 
-mc = Dalli::Client.new(mc_addr, :serializer => JSON)
-
-def populate_mc(mc, gerrit_url, project)
-  print "project: #{project}\n"
+def populate_cb(cb, gerrit_url, project, out)
+  out.puts "project: #{project}"
 
   res = Net::HTTP.get(URI(gerrit_url + '/changes/?q=project:' + project))
-  changes = JSON.parse(res.sub(/^\)\]\}\'/, ""))
+  changes = JSON.parse(res.sub(/^\)\]\}\'/, ''))
 
-  changes = changes.sort {|a, b| b["updated"] <=> a["updated"]}
+  changes = changes.sort { |a, b| b['updated'] <=> a['updated'] }
   if changes.length > 0
-    print "  latest: #{changes[0]['updated']}\n"
+    out.puts "  latest: #{changes[0]['updated']}"
   end
 
   i = 0
@@ -55,37 +58,42 @@ def populate_mc(mc, gerrit_url, project)
     #     "name": "Chiyoung Seo"
     #   }
     # }
-    key = "gerrit/" + change["id"]
+    key = 'gerrit/' + change['id']
     doc = {
-      "type" => "gerrit/change",
-      "key" => key,
-      "url" => "#{gerrit_url}/#{change['_number']}",
-      "id" => change["id"],
-      "project" => change["project"],
-      "branch" => change["branch"],
-      "change_id" => change["change_id"],
-      "title" => change["subject"], # Relabeled.
-      "status" => change["status"],
-      "createdDate" => change["created"], # Relabeled.
-      "updatedDate" => change["updated"], # Relabeled.
-      "owner" => change["owner"]["name"], # Relabeled.
+      type: 'gerrit/change',
+      key: key,
+      url: "#{gerrit_url}/#{change['_number']}",
+      id: change['id'],
+      project: change['project'],
+      branch: change['branch'],
+      change_id: change['change_id'],
+      title: change['subject'], # Relabeled.
+      status: change['status'],
+      createdDate: change['created'], # Relabeled.
+      updatedDate: change['updated'], # Relabeled.
+      owner: change['owner']['name'], # Relabeled.
     }
 
-    ok = mc.set(key, doc)
-    if not ok
-      exit "mc.set failed"
-    end
+    ok = cb.set(key, doc)
+    abort 'cb.set failed' unless ok
 
-    i = i + 1
+    i += 1
   end
 
-  return changes.length, i
+  [changes.length, i]
 end
 
-res = Net::HTTP.get(URI(gerrit_url + '/projects/'))
-projects = JSON.parse(res.sub(/^\)\]\}\'/, ""))
-projects.each do |project, project_obj|
-  changes_total, changes_set = populate_mc(mc, gerrit_url, project)
-  print "  changes_total: #{changes_total}\n"
-  print "  changes_set: #{changes_set}\n"
+res = Net::HTTP.get(URI(options[:gerrit] + '/projects/'))
+projects = JSON.parse(res.sub(/^\)\]\}\'/, ''))
+threads = []
+projects.each do |project, _project_obj|
+  threads << Thread.new do
+    cb = Couchbase.connect(options[:couchbase])
+    out = StringIO.new
+    changes_total, changes_set = populate_cb(cb, options[:gerrit], project, out)
+    out.puts "  changes_total: #{changes_total}"
+    out.puts "  changes_set: #{changes_set}"
+    puts out.string
+  end
 end
+threads.each(&:join)