Permalink
Browse files

Merge branch 'master' of github.com:twitter/gizzmo

  • Loading branch information...
2 parents 82f238a + a021765 commit 6fa06e1d74aec2226e6a76c77308ae32f3d4bd78 Kyle Maxwell committed Oct 1, 2010
Showing with 76 additions and 22 deletions.
  1. +1 −1 VERSION
  2. +5 −4 gizzmo.gemspec
  3. +2 −1 lib/gizzard.rb
  4. +25 −7 lib/gizzard/commands.rb
  5. +13 −0 lib/gizzard/digest.rb
  6. +21 −4 lib/gizzmo.rb
  7. +9 −5 test/test.sh
View
@@ -1 +1 @@
-0.7.0
+0.7.3
View
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = %q{gizzmo}
- s.version = "0.7.0"
+ s.version = "0.7.3"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Kyle Maxwell"]
- s.date = %q{2010-08-25}
+ s.date = %q{2010-09-08}
s.default_executable = %q{gizzmo}
s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
s.email = %q{kmaxwell@twitter.com}
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
"gizzmo.gemspec",
"lib/gizzard.rb",
"lib/gizzard/commands.rb",
+ "lib/gizzard/hash.rb",
"lib/gizzard/thrift.rb",
"lib/gizzmo.rb",
"lib/vendor/thrift_client/simple.rb",
@@ -56,7 +57,7 @@ Gem::Specification.new do |s|
s.homepage = %q{http://github.com/twitter/gizzmo}
s.rdoc_options = ["--charset=UTF-8"]
s.require_paths = ["lib"]
- s.rubygems_version = %q{1.3.6}
+ s.rubygems_version = %q{1.3.7}
s.summary = %q{Gizzmo is a command-line client for managing gizzard clusters.}
s.test_files = [
"test/helper.rb"
@@ -66,7 +67,7 @@ Gem::Specification.new do |s|
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
s.specification_version = 3
- if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+ if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
else
end
else
View
@@ -1,4 +1,5 @@
$: << File.dirname(__FILE__)
module Gizzard; end
require "gizzard/thrift"
-require "gizzard/commands"
+require "gizzard/commands"
+require "gizzard/digest"
View
@@ -5,13 +5,13 @@ class Command
attr_reader :buffer
- def self.run(command_name, global_options, argv, subcommand_options, log)
+ def self.run(command_name, global_options, argv, subcommand_options, log, service=nil)
command_class = Gizzard.const_get("#{classify(command_name)}Command")
- service = command_class.make_service(global_options, log)
+ service = command_class.make_service(global_options, log) if service.nil?
command = command_class.new(service, global_options, argv, subcommand_options)
command.run
if command.buffer && command_name = global_options.render.shift
- run(command_name, service, global_options, command.buffer, OpenStruct.new)
+ run(command_name, global_options, command.buffer, OpenStruct.new, log, service)
end
end
@@ -86,7 +86,7 @@ def run
end.reject do |forwarding|
@command_options.table_ids && !@command_options.table_ids.include?(forwarding.table_id)
end.each do |forwarding|
- output [ forwarding.table_id, forwarding.base_id, forwarding.shard_id.to_unix ].join("\t")
+ output [ forwarding.table_id, @command_options.hex ? ("%016x" % forwarding.base_id) : forwarding.base_id, forwarding.shard_id.to_unix ].join("\t")
end
end
end
@@ -425,9 +425,15 @@ def run
class LookupCommand < ShardCommand
def run
- table_id, source_id = @argv
- help!("Requires table id and source id") unless table_id && source_id
- shard = service.find_current_forwarding(table_id.to_i, source_id.to_i)
+ table_id, source = @argv
+ help!("Requires table id and source") unless table_id && source
+ case @command_options.hash_function
+ when :fnv
+ source_id = Digest.fnv1a_64(source)
+ else
+ source_id = source.to_i
+ end
+ shard = service.find_current_forwarding(table_id.to_i, source_id)
output shard.id.to_unix
end
end
@@ -529,4 +535,16 @@ def run
STDERR.print "\n"
end
end
+
+ class FlushCommand < JobCommand
+ def run
+ args = @argv[0]
+ help!("Requires --all, or a job priority id.") unless args || command_options.flush_all
+ if command_options.flush_all
+ service.retry_errors()
+ else
+ service.retry_errors_for(args.to_i)
+ end
+ end
+ end
end
View
@@ -0,0 +1,13 @@
+module Gizzard
+ module Digest
+ def self.fnv1a_64(data)
+ prime = 1099511628211
+ rv = 0xcbf29ce484222325
+ data.each_byte do |byte|
+ rv = ((rv ^ (byte & 0xff)) * prime) & 0xffffffffffffffff
+ end
+ # trim to 60 bits for gizzard.
+ rv & 0x0fffffffffffffff
+ end
+ end
+end
View
@@ -9,7 +9,9 @@ class HelpNeededError < RuntimeError; end
DOC_STRINGS = {
"create" => "Create shard(s) of a given Java/Scala class. If you don't know the list of available classes, you can just try a bogus class, and the exception will include a list of valid classes.",
"wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
- "inject" => "Inject jobs (as literal json) into the server. Jobs can be linefeed-terminated from stdin, or passed as arguments. Priority is server-defined, but typically lower numbers (like 1) are lower priority."
+ "inject" => "Inject jobs (as literal json) into the server. Jobs can be linefeed-terminated from stdin, or passed as arguments. Priority is server-defined, but typically lower numbers (like 1) are lower priority.",
+ "lookup" => "Lookup the shard id that holds the record for a given table / source_id.",
+ "flush" => "Flush error queue for a given priority."
}
ORIGINAL_ARGV = ARGV.dup
@@ -132,6 +134,9 @@ def separators(opts, string)
subcommand_options.table_ids ||= []
subcommand_options.table_ids += table_ids.split(",").map { |s| s.to_i }
end
+ opts.on("-x", "--hex", "Show base ids in hex") do
+ subcommand_options.hex = true
+ end
end,
'unwrap' => OptionParser.new do |opts|
opts.banner = "Usage: #{zero} unwrap SHARD_ID_TO_REMOVE [MORE SHARD_IDS]"
@@ -170,8 +175,12 @@ def separators(opts, string)
separators(opts, DOC_STRINGS["unlink"])
end,
'lookup' => OptionParser.new do |opts|
- opts.banner = "Usage: #{zero} lookup TABLE_ID SOURCE_ID"
+ opts.banner = "Usage: #{zero} lookup [options] TABLE_ID SOURCE"
separators(opts, DOC_STRINGS["lookup"])
+
+ opts.on("--fnv", "Use FNV1A_64 hash on source") do
+ subcommand_options.hash_function = :fnv
+ end
end,
'copy' => OptionParser.new do |opts|
opts.banner = "Usage: #{zero} copy SOURCE_SHARD_ID DESTINATION_SHARD_ID"
@@ -192,6 +201,14 @@ def separators(opts, string)
'inject' => OptionParser.new do |opts|
opts.banner = "Usage: #{zero} inject PRIORITY JOBS..."
separators(opts, DOC_STRINGS["inject"])
+ end,
+ 'flush' => OptionParser.new do |opts|
+ opts.banner = "Usage: #{zero} flush --all|PRIORITY"
+ separators(opts, DOC_STRINGS["flush"])
+
+ opts.on("--all", "Flush all error queues.") do
+ subcommand_options.flush_all = true
+ end
end
}
@@ -231,7 +248,7 @@ def separators(opts, string)
end
opts.on("-P", "--port=PORT", "PORT of remote thrift service") do |port|
- global_options.port = port
+ global_options.port = port.to_i
end
opts.on("-r", "--retry=TIMES", "TIMES to retry the command") do |r|
@@ -342,4 +359,4 @@ def process_nested_parsers(global, subcommands)
# seeing the backtrace is annoying!
rescue Interrupt
exit 1
-end
+end
View
@@ -6,7 +6,7 @@ function g {
}
function expect {
- diff -u - "expected/$1" && echo " success." || echo " failed." && exit 1
+ diff -u - "expected/$1" && echo " success." || (echo " failed." && exit 1)
}
function expect-string {
@@ -19,9 +19,9 @@ g find -hlocalhost | expect empty-file.txt
for i in {0..9}
do
- g create localhost "table_repl_$i" com.twitter.service.flock.edges.ReplicatingShard
- g create localhost "table_a_$i" com.twitter.service.flock.edges.SqlShard --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
- g create localhost "table_b_$i" com.twitter.service.flock.edges.SqlShard --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
+ g create com.twitter.service.flock.edges.ReplicatingShard localhost/table_repl_$i
+ g create com.twitter.service.flock.edges.SqlShard localhost/table_a_$i --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
+ g create com.twitter.service.flock.edges.SqlShard localhost/table_b_$i --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
g addlink "localhost/table_repl_$i" "localhost/table_a_$i" 2
g addlink "localhost/table_repl_$i" "localhost/table_b_$i" 1
done
@@ -52,6 +52,7 @@ g links localhost/replicating_table_b_0 | expect links-for-replicating_table_b_0
g --subtree --info find -Hlocalhost | expect subtree-info.txt
g lookup 1 100 | expect-string "localhost/forward_1"
+g lookup --fnv 1 "hello" | expect-string "localhost/forward_1"
g unwrap localhost/replicating_table_b_0 | expect unwrapped-replicating_table_b_0.txt
g links localhost/table_b_0 | expect unwrapped-table_b_0.txt
@@ -72,4 +73,7 @@ do
g addlink "localhost/table_deep_repl_$last" "localhost/table_deep_repl_$i" 2
done
-g subtree localhost/table_deep_repl_5 | expect deep.txt
+g subtree localhost/table_deep_repl_5 | expect deep.txt
+
+g flush --all
+g flush 1

0 comments on commit 6fa06e1

Please sign in to comment.