Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'master' into copy_repair_merge

  • Loading branch information...
commit 37841fcd68b65d063f6684195f7104e8304c3cc7 2 parents 69606d9 + 898b2ce
Josh Hull authored
View
2  VERSION
@@ -1 +1 @@
-0.11.3
+0.12.0
View
4 gizzmo.gemspec
@@ -5,11 +5,11 @@
Gem::Specification.new do |s|
s.name = %q{gizzmo}
- s.version = "0.11.3"
+ s.version = "0.12.0"
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
s.authors = ["Kyle Maxwell"]
- s.date = %q{2011-01-26}
+ s.date = %q{2011-05-05}
s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
s.email = %q{kmaxwell@twitter.com}
s.executables = ["gizzmo", "setup_shards"]
View
106 lib/gizzard/commands.rb
@@ -892,4 +892,110 @@ def run
scheduler_options
end
end
+
+
+ class CreateTableCommand < Command
+
+ DEFAULT_NUM_SHARDS = 1024
+ DEFAULT_BASE_NAME = "shard"
+
+ FORWARDING_SPACE = 2 ** 60
+ FORWARDING_SPACE_MIN = 0
+ FORWARDING_SPACE_MAX = 2 ** 60 - 1
+
+ def generate_base_ids(num_shards, min_id, max_id)
+ srand(42) # consistent randomization
+
+ id_space = max_id - min_id + 1
+ step_size = id_space / num_shards
+
+ enums = (0...num_shards).to_a
+ ids = enums.map {|i| min_id + (i * step_size) }.sort_by { rand }
+
+ enums.zip(ids)
+ end
+
+ def parse_templates_and_weights(arr)
+ templates_and_weights = {}
+
+ arr.each_slice(2) do |(weight_s, to_template_s)|
+ to = ShardTemplate.parse(to_template_s)
+ weight = weight_s.to_i
+
+ templates_and_weights[to] = weight
+ end
+
+ templates_and_weights
+ end
+
+ # This is all super hacky but I don't have time to generalize right now
+
+ def run
+ help!("must have an even number of arguments") unless @argv.length % 2 == 0
+
+ base_name = command_options.base_name || DEFAULT_BASE_NAME
+ num_shards = (command_options.shards || DEFAULT_NUM_SHARDS).to_i
+ max_id = (command_options.max_id || FORWARDING_SPACE_MAX).to_i
+ min_id = (command_options.min_id || FORWARDING_SPACE_MIN).to_i
+
+ be_quiet = global_options.force && command_options.quiet
+
+ templates_and_weights = parse_templates_and_weights(@argv)
+ total_weight = templates_and_weights.values.inject {|a,b| a + b }
+ templates = templates_and_weights.keys
+
+ base_ids = generate_base_ids(num_shards, min_id, max_id)
+
+ templates_and_base_ids = templates_and_weights.inject({}) do |h, (template, weight)|
+ share = (weight.to_f / total_weight * num_shards).floor
+ ids = []
+ share.times { ids << base_ids.pop }
+
+ h.update template => ids
+ end
+
+ # divvy up the remainder across all templates.
+ base_ids.each_with_index do |base, idx|
+ templates_and_base_ids.values[idx % templates_and_base_ids.length] << base
+ end
+
+ proto = templates.first
+ transform = Transformation.new(proto, proto)
+
+ op_sets = templates.inject({}) do |h, template|
+ ops = transform.create_tree(template).sort
+ h.update template => ops
+ end
+
+ unless be_quiet
+ puts "Create tables #{global_options.tables.join(", ")}:"
+ templates_and_base_ids.each do |template, base_ids|
+ puts " #{template.inspect}"
+ puts " for #{base_ids.length} base ids:"
+ base_ids.each {|(enum, base_id)| puts " #{base_id}" }
+ end
+ puts ""
+ end
+
+ unless global_options.force
+ print "Continue? (y/n) "; $stdout.flush
+ exit unless $stdin.gets.chomp == "y"
+ puts ""
+ end
+
+ global_options.tables.each do |table_id|
+ templates_and_base_ids.each do |template, base_ids|
+ ops = op_sets[template]
+
+ base_ids.each do |(enum, base_id)|
+ table_prefix = Shard.canonical_table_prefix(enum, table_id, base_name)
+ ops.each do |op|
+ puts "#{op.inspect}: #{table_prefix}"
+ op.apply(manager, table_id, base_id, table_prefix, {})
+ end
+ end
+ end
+ end
+ end
+ end
end
View
8 lib/gizzard/nameserver.rb
@@ -87,7 +87,7 @@ class Nameserver
include ParallelMap
DEFAULT_PORT = 7917
- DEFAULT_RETRIES = 20
+ DEFAULT_RETRIES = 10
PARALLELISM = 10
attr_reader :hosts, :logfile, :dryrun, :framed
@@ -174,11 +174,9 @@ def create_client(host)
def with_retry
times ||= @retries
yield
- rescue ThriftClient::Simple::ThriftException, NoMethodError, Gizzard::GizzardException => e
- raise if e.is_a? Gizzard::GizzardException and e.message !~ /Communications link failure/
-
+ rescue Exception => e
times -= 1
- (times < 0) ? raise : (sleep 2; retry)
+ (times < 0) ? raise : (sleep 0.1; retry)
end
class Manifest
View
8 lib/gizzard/shard_template.rb
@@ -126,7 +126,13 @@ def link_eql?(other)
def shared_host?(other)
raise ArgumentError, "other is not a ShardTemplate" unless other.is_a? ShardTemplate
- (self.concrete_descendants & other.concrete_descendants).length > 0
+ self.concrete_descendants.each do |s|
+ other.concrete_descendants.each do |o|
+ return true if s.shard_eql? o
+ end
+ end
+
+ false
end
def hash
View
3  lib/gizzard/transformation.rb
@@ -147,8 +147,9 @@ def expand_jobs(jobs)
def copies_required?
return @copies_required unless @copies_required.nil?
+
@copies_required = !from.nil? &&
- to.concrete_descendants.reject {|d| from.shared_host? d }.length > 0
+ to.concrete_descendants.select {|d| !from.shared_host? d }.length > 0
end
def involved_in_copy?(template)
View
30 lib/gizzmo.rb
@@ -345,6 +345,30 @@ def add_scheduler_opts(subcommand_options, opts)
opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
subcommand_options.quiet = true
end
+ end,
+ 'create-table' => OptionParser.new do |opts|
+ opts.banner = "Usage: #{zero} create-table [options] WEIGHT TEMPLATE ..."
+ separators(opts, DOC_STRINGS["create-table"])
+
+ opts.on("--shards=COUNT", "Create COUNT shards for each table.") do |count|
+ subcommand_options.shards = count.to_i
+ end
+
+ opts.on("--min-id=NUM", "Set lower bound on the id space to NUM (default min signed long: -1 * 2^63)") do |min_id|
+ subcommand_options.min_id = min_id.to_i
+ end
+
+ opts.on("--max-id=NUM", "Set upper bound on the id space to NUM (default max signed long: 2^63 - 1)") do |max_id|
+ subcommand_options.max_id = max_id.to_i
+ end
+
+ opts.on("--base-name=NAME", "Use NAME as the base prefix for each shard's table prefix (default 'shard')") do |base_name|
+ subcommand_options.base_name = base_name
+ end
+
+ opts.on("-q", "--quiet", "Do not display table creation info (only valid with --force)") do
+ subcommand_options.quiet = true
+ end
end
}
@@ -388,12 +412,6 @@ def add_scheduler_opts(subcommand_options, opts)
global_options.hosts = hosts.split(",").map {|h| h.strip }
end
- opts.on("-H", "--host=HOST", "HOST of application servers") do |hosts|
- global_options.hosts = hosts.split(",").map {|h| h.strip }
- end
-
-
-
opts.on("-P", "--port=PORT", "PORT of remote manager service. default 7920") do |port|
global_options.port = port.to_i
end
View
119 test/gizzmo_spec.rb
@@ -440,6 +440,38 @@ def nameserver_db
link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1)]
end
+ it "properly re-weights shards" do
+ 1.upto(2) do |i|
+ gizzmo "create TestShard -s Int -d Int localhost/s_0_00#{i}_a"
+ gizzmo "create ReplicatingShard localhost/s_0_00#{i}_replicating"
+ gizzmo "addlink localhost/s_0_00#{i}_replicating localhost/s_0_00#{i}_a 1"
+ gizzmo "addforwarding 0 #{i} localhost/s_0_00#{i}_replicating"
+ end
+ gizzmo "-f reload"
+
+ gizzmo('-f -T0 transform --no-progress --poll-interval=1 --max-copies=1 \
+"ReplicatingShard -> TestShard(localhost,1,Int,Int)" \
+"ReplicatingShard -> TestShard(localhost,3,Int,Int)"').should == <<-EOF
+ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int) :
+ PREPARE
+ add_link(ReplicatingShard -> TestShard/localhost)
+ remove_link(ReplicatingShard -> TestShard/localhost)
+Applied to 2 shards:
+ [0] 1 = localhost/s_0_001_replicating
+ [0] 2 = localhost/s_0_002_replicating
+
+STARTING:
+ [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+FINISHING:
+ [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+STARTING:
+ [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+FINISHING:
+ [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+2 transformations applied. Total time elapsed: 2 seconds
+EOF
+ end
+
it "works with multiple pages" do
1.upto(2) do |i|
gizzmo "create TestShard -s Int -d Int localhost/s_0_00#{i}_a"
@@ -625,4 +657,91 @@ def nameserver_db
EOF
end
end
+
+ describe "create-table" do
+ it "works" do
+ gizzmo('-f -T0,1 create-table --shards=4 --base-name=s \
+1 "ReplicatingShard -> TestShard(127.0.0.1,1)" \
+1 "ReplicatingShard -> TestShard(localhost,1)"').should match(Regexp.new(Regexp.escape(<<-EOF).gsub("X", "\\d")))
+Create tables 0, 1:
+ ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+ for 2 base ids:
+ 288230376151711744
+ 576460752303423488
+ ReplicatingShard(1) -> TestShard(localhost,1)
+ for 2 base ids:
+ 864691128455135232
+ 0
+
+create_shard(ReplicatingShard): s_0_0003
+create_shard(TestShard/127.0.0.1): s_0_0003
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_0_0003
+set_forwarding(ReplicatingShard): s_0_0003
+create_shard(ReplicatingShard): s_0_0002
+create_shard(TestShard/127.0.0.1): s_0_0002
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_0_0002
+set_forwarding(ReplicatingShard): s_0_0002
+create_shard(ReplicatingShard): s_0_0001
+create_shard(TestShard/localhost): s_0_0001
+add_link(ReplicatingShard -> TestShard/localhost): s_0_0001
+set_forwarding(ReplicatingShard): s_0_0001
+create_shard(ReplicatingShard): s_0_0000
+create_shard(TestShard/localhost): s_0_0000
+add_link(ReplicatingShard -> TestShard/localhost): s_0_0000
+set_forwarding(ReplicatingShard): s_0_0000
+create_shard(ReplicatingShard): s_1_0003
+create_shard(TestShard/127.0.0.1): s_1_0003
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_1_0003
+set_forwarding(ReplicatingShard): s_1_0003
+create_shard(ReplicatingShard): s_1_0002
+create_shard(TestShard/127.0.0.1): s_1_0002
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_1_0002
+set_forwarding(ReplicatingShard): s_1_0002
+create_shard(ReplicatingShard): s_1_0001
+create_shard(TestShard/localhost): s_1_0001
+add_link(ReplicatingShard -> TestShard/localhost): s_1_0001
+set_forwarding(ReplicatingShard): s_1_0001
+create_shard(ReplicatingShard): s_1_0000
+create_shard(TestShard/localhost): s_1_0000
+add_link(ReplicatingShard -> TestShard/localhost): s_1_0000
+set_forwarding(ReplicatingShard): s_1_0000
+ EOF
+
+ nameserver_db[:shards].should == [info("127.0.0.1", "s_0_0002", "TestShard"),
+ info("127.0.0.1", "s_0_0003", "TestShard"),
+ info("127.0.0.1", "s_1_0002", "TestShard"),
+ info("127.0.0.1", "s_1_0003", "TestShard"),
+ info("localhost", "s_0_0000", "TestShard"),
+ info("localhost", "s_0_0000_replicating", "ReplicatingShard"),
+ info("localhost", "s_0_0001", "TestShard"),
+ info("localhost", "s_0_0001_replicating", "ReplicatingShard"),
+ info("localhost", "s_0_0002_replicating", "ReplicatingShard"),
+ info("localhost", "s_0_0003_replicating", "ReplicatingShard"),
+ info("localhost", "s_1_0000", "TestShard"),
+ info("localhost", "s_1_0000_replicating", "ReplicatingShard"),
+ info("localhost", "s_1_0001", "TestShard"),
+ info("localhost", "s_1_0001_replicating", "ReplicatingShard"),
+ info("localhost", "s_1_0002_replicating", "ReplicatingShard"),
+ info("localhost", "s_1_0003_replicating", "ReplicatingShard")]
+
+ nameserver_db[:links].should == [link(id("localhost", "s_0_0000_replicating"), id("localhost", "s_0_0000"), 1),
+ link(id("localhost", "s_0_0001_replicating"), id("localhost", "s_0_0001"), 1),
+ link(id("localhost", "s_0_0002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
+ link(id("localhost", "s_0_0003_replicating"), id("127.0.0.1", "s_0_0003"), 1),
+ link(id("localhost", "s_1_0000_replicating"), id("localhost", "s_1_0000"), 1),
+ link(id("localhost", "s_1_0001_replicating"), id("localhost", "s_1_0001"), 1),
+ link(id("localhost", "s_1_0002_replicating"), id("127.0.0.1", "s_1_0002"), 1),
+ link(id("localhost", "s_1_0003_replicating"), id("127.0.0.1", "s_1_0003"), 1)]
+
+ nameserver_db[:forwardings].should == [forwarding(0, 0, id("localhost", "s_0_0000_replicating")),
+ forwarding(1, 0, id("localhost", "s_1_0000_replicating")),
+ forwarding(0, 288230376151711744, id("localhost", "s_0_0003_replicating")),
+ forwarding(1, 288230376151711744, id("localhost", "s_1_0003_replicating")),
+ forwarding(0, 576460752303423488, id("localhost", "s_0_0002_replicating")),
+ forwarding(1, 576460752303423488, id("localhost", "s_1_0002_replicating")),
+ forwarding(0, 864691128455135232, id("localhost", "s_0_0001_replicating")),
+ forwarding(1, 864691128455135232, id("localhost", "s_1_0001_replicating"))]
+
+ end
+ end
end
View
6 test/spec_helper.rb
@@ -130,7 +130,7 @@ def as_link(h)
end
def as_forwarding(h)
- Gizzard::Forwarding.new(h['table_id'].to_i, h['base_id'].to_i, as_shard_id(h, 'shard'))
+ Gizzard::Forwarding.new(h['table_id'].to_i, h['base_source_id'].to_i, as_shard_id(h, 'shard'))
end
def as_host(h)
@@ -144,14 +144,14 @@ def gizzmo(cmd)
end
def nameserver
- @nameserver ||= Gizzard::Nameserver.new('localhost:' + MANAGER_PORT.to_s)
+ @nameserver ||= Gizzard::Nameserver.new('localhost:' + MANAGER_PORT.to_s, :retries => 5)
end
alias ns nameserver
# setup
-mysql_connect!("localhost", '', '')
+mysql_connect!("localhost", 'root', '')
reset_databases!
unless ENV['EXTERNAL_TEST_SERVER']
Please sign in to comment.
Something went wrong with that request. Please try again.