Skip to content
This repository
Browse code

wip

  • Loading branch information...
commit 0a674637adefcd63b6d220241bae6b757a6c2a39 1 parent a16f09c
Matt Freels freels authored
1  gizzmo.gemspec
@@ -82,6 +82,7 @@ Gem::Specification.new do |s|
82 82 "test/gizzmo_spec.rb",
83 83 "test/helper.rb",
84 84 "test/nameserver_spec.rb",
  85 + "test/scheduler_spec.rb",
85 86 "test/shard_template_spec.rb",
86 87 "test/spec_helper.rb",
87 88 "test/test_server/target/gen-rb/test_server.rb",
35 lib/gizzard/commands.rb
@@ -812,7 +812,7 @@ def run
812 812 end
813 813
814 814 unless global_options.force
815   - puts "Continue? (y/n)"
  815 + print "Continue? (y/n) "; $stdout.flush
816 816 exit unless $stdin.gets.chomp == "y"
817 817 puts ""
818 818 end
@@ -826,33 +826,40 @@ def run
826 826
827 827 class TransformCommand < Command
828 828 def run
829   - help!("wrong number of arguments") unless @argv.length == 2
  829 + help!("must have an even number of arguments") unless @argv.length % 2 == 0
  830 +
  831 + manifest = manager.manifest(*global_options.tables)
  832 + transformations = {}
  833 +
  834 + @argv.each_slice(2) do |(from_template_s, to_template_s)|
  835 + from, to = [from_template_s, to_template_s].map {|s| ShardTemplate.parse(s) }
  836 + transformation = Transformation.new(from, to)
  837 + forwardings = manifest.templates[from]
  838 + trees = manifest.trees.reject {|(f, s)| !forwardings.include?(f) }
830 839
831   - from_template_s, to_template_s = @argv
  840 + transformations[transformation] = trees
  841 + end
832 842
833   - from, to = [from_template_s, to_template_s].map {|s| ShardTemplate.parse(s) }
834   - manifest = manager.manifest(*global_options.tables)
835   - forwardings = manifest.templates[from]
836   - transformation = Transformation.new(from, to)
837   - trees = manifest.trees.reject {|(f, s)| !forwardings.include?(f) }
838   - base_name = trees.values.first.id.table_prefix.split('_').first
  843 + base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
839 844
840 845 unless global_options.force && command_options.quiet
841   - puts transformation.inspect
842   - puts "Applied to:"
843   - forwardings.sort.each {|f| puts " #{f.inspect}" }
  846 + transformations.each do |transformation, trees|
  847 + puts transformation.inspect
  848 + puts "Applied to:"
  849 + trees.keys.sort.each {|f| puts " #{f.inspect}" }
  850 + end
844 851 puts ""
845 852 end
846 853
847 854 unless global_options.force
848   - puts "Continue? (y/n)"
  855 + print "Continue? (y/n) "; $stdout.flush
849 856 exit unless $stdin.gets.chomp == "y"
850 857 puts ""
851 858 end
852 859
853 860 Gizzard.schedule! manager,
854 861 base_name,
855   - { transformation => trees },
  862 + transformations,
856 863 command_options.scheduler_options
857 864 end
858 865 end
2  lib/gizzard/thrift.rb
@@ -86,7 +86,7 @@ def <=>(o)
86 86 end
87 87
88 88 def inspect
89   - "[#{table_id}] #{base_id.to_s(16)} -> #{shard_id.inspect}"
  89 + "[#{table_id}] #{base_id.to_s(16)} = #{shard_id.inspect}"
90 90 end
91 91 end
92 92
28 lib/gizzard/transformation.rb
@@ -201,27 +201,15 @@ def involved_hosts(phase = :copy)
201 201 involved_shards(phase).map {|s| s.hostname }.uniq
202 202 end
203 203
204   - def inspect(phase = nil)
205   - base = "#{@forwarding.inspect}: #{from.inspect} => #{to.inspect}"
206   - op_inspect = transformation.operations.inject({}) do |h, (p, ops)|
207   - h.update p => ops.map {|job| " #{job.inspect}" }.join("\n")
208   - end
209   -
210   - prepare_inspect = op_inspect[:prepare].empty? ? "" : " PREPARE\n#{op_inspect[:prepare]}\n"
211   - copy_inspect = op_inspect[:copy].empty? ? "" : " COPY\n#{op_inspect[:copy]}\n"
212   - cleanup_inspect = op_inspect[:cleanup].empty? ? "" : " CLEANUP\n#{op_inspect[:cleanup]}\n"
  204 + def inspect
  205 + "#{@forwarding.inspect}: #{from.inspect} => #{to.inspect}"
  206 + end
213 207
214   - case phase
215   - when :all
216   - [base, "\n", prepare_inspect, copy_inspect, cleanup_inspect].join
217   - when :prepare
218   - [base, "\n", prepare_inspect].join
219   - when :copy
220   - [base, "\n", copy_inspect].join
221   - when :cleanup
222   - [base, "\n", cleanup_inspect].join
223   - else
224   - base
  208 + def copy_descs
  209 + transformation.operations[:copy].map do |copy|
  210 + from_id = copy.from.to_shard_id(@table_prefix, @translations)
  211 + to_id = copy.to.to_shard_id(@table_prefix, @translations)
  212 + "#{from_id.inspect} -> #{to_id.inspect}"
225 213 end
226 214 end
227 215
48 lib/gizzard/transformation_scheduler.rb
@@ -9,9 +9,9 @@ class Transformation::Scheduler
9 9 attr_reader :max_copies, :copies_per_host
10 10
11 11 DEFAULT_OPTIONS = {
12   - :max_copies => 30,
  12 + :max_copies => 30,
13 13 :copies_per_host => 8,
14   - :poll_interval => 5
  14 + :poll_interval => 10
15 15 }.freeze
16 16
17 17 def initialize(nameserver, base_name, transformations, options = {})
@@ -44,8 +44,16 @@ def initialize(nameserver, base_name, transformations, options = {})
44 44 # 4. schedule a new job or reload app servers.
45 45
46 46 def apply!
  47 + @start_time = Time.now
  48 +
47 49 loop do
48   - reload_busy_shards
  50 + begin
  51 + reload_busy_shards
  52 + rescue GizzardException
  53 + sleep 10
  54 + next
  55 + end
  56 +
49 57 cleanup_jobs
50 58 schedule_jobs(max_copies - busy_shards.length)
51 59
@@ -61,7 +69,7 @@ def apply!
61 69
62 70 nameserver.reload_config
63 71
64   - log "All transformations applied. Have a nice day!"
  72 + log "All transformations applied. Total time elapsed: #{time_elapsed}"
65 73 end
66 74
67 75 def schedule_jobs(num_to_schedule)
@@ -78,23 +86,22 @@ def schedule_jobs(num_to_schedule)
78 86 end
79 87
80 88 job
81   - end.compact
  89 + end.compact.sort_by {|t| t.forwarding }
82 90
83 91 unless jobs.empty?
84   - log "Jobs starting:"
85   - jobs.each {|j| log " #{j.inspect(:prepare)}" }
  92 + log "STARTING:"
  93 + jobs.each {|j| log " #{j.inspect}" }
86 94
87 95 jobs.each {|j| j.prepare!(nameserver) }
88 96
89   - log "Reloading nameserver configuration."
90 97 nameserver.reload_config
91 98
92 99 copy_jobs = jobs.select {|j| j.copy_required? }
93 100
94 101 unless copy_jobs.empty?
95   - log "Scheduling copies:"
  102 + log "COPIES:"
96 103 copy_jobs.each do |j|
97   - log " #{j.inspect(:copy)}"
  104 + j.copy_descs.each {|d| log " #{d}" }
98 105 j.copy!(nameserver)
99 106 end
100 107 end
@@ -108,8 +115,8 @@ def cleanup_jobs
108 115 @jobs_in_progress -= jobs
109 116
110 117 unless jobs.empty?
111   - log "Jobs finishing:"
112   - jobs.each {|j| log " #{j.inspect(:cleanup)}" }
  118 + log "FINISHING:"
  119 + jobs.each {|j| log " #{j.inspect}" }
113 120 end
114 121
115 122 jobs.each {|j| j.cleanup!(nameserver) }
@@ -141,7 +148,7 @@ def busy_hosts(extra_hosts = [])
141 148 h.update(host => 1) {|_,a,b| a + b }
142 149 end
143 150
144   - copies_count_map.select {|_, count| count >= @max_copies }.map {|(host, _)| host }
  151 + copies_count_map.select {|_, count| count >= @copies_per_host }.map {|(host, _)| host }
145 152 end
146 153
147 154 def reset_progress_string
@@ -151,6 +158,17 @@ def reset_progress_string
151 158 end
152 159 end
153 160
  161 + def time_elapsed
  162 + s = (Time.now - @start_time).to_i
  163 +
  164 + days = s / (60 * 60 * 24) if s >= 60 * 60 * 24
  165 + hours = (s % (60 * 60 * 24)) / (60 * 60) if s >= 60 * 60
  166 + minutes = (s % (60 * 60)) / 60 if s >= 60
  167 + seconds = s % 60
  168 +
  169 + [days,hours,minutes,seconds].compact.map {|i| "%0.2i" % i }.join(":")
  170 + end
  171 +
154 172 def log(*args)
155 173 reset_progress_string
156 174 puts *args
@@ -164,10 +182,10 @@ def put_copy_progress
164 182 unless @jobs_in_progress.empty? || @busy_shards.empty?
165 183 if @progress_string
166 184 print "\r"
167   - print " " * @progress_string.length + 10
  185 + print " " * (@progress_string.length + 10)
168 186 print "\r"
169 187 end
170   - @progress_string = "#{spinner} Copies in progress: #{@busy_shards.length}"
  188 + @progress_string = "#{spinner} Copies in progress: #{@busy_shards.length} Time elapsed: #{time_elapsed}"
171 189 print @progress_string; $stdout.flush
172 190 end
173 191 end
82 test/gizzmo_spec.rb
@@ -334,7 +334,7 @@ def nameserver
334 334 gizzmo "addforwarding 0 1 localhost/s_0_001_replicating"
335 335 gizzmo "-f reload"
336 336
337   - gizzmo('-f transform-tree "ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))" localhost/s_0_001_replicating').should == <<-EOF
  337 + gizzmo('-f transform-tree --poll-interval=1 "ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))" localhost/s_0_001_replicating').should == <<-EOF
338 338 ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1)) :
339 339 PREPARE
340 340 create_shard(TestShard/127.0.0.1)
@@ -349,26 +349,13 @@ def nameserver
349 349 remove_link(ReplicatingShard -> WriteOnlyShard)
350 350 delete_shard(WriteOnlyShard)
351 351
352   -Jobs starting:
353   - [0] 1 -> localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
354   - PREPARE
355   - create_shard(TestShard/127.0.0.1)
356   - create_shard(WriteOnlyShard)
357   - add_link(WriteOnlyShard -> TestShard/127.0.0.1)
358   - add_link(ReplicatingShard -> WriteOnlyShard)
359   -Reloading nameserver configuration.
360   -Scheduling copies:
361   - [0] 1 -> localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
362   - COPY
363   - copy_shard(TestShard/127.0.0.1)
364   -Jobs finishing:
365   - [0] 1 -> localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
366   - CLEANUP
367   - add_link(ReplicatingShard -> TestShard/127.0.0.1)
368   - remove_link(WriteOnlyShard -> TestShard/127.0.0.1)
369   - remove_link(ReplicatingShard -> WriteOnlyShard)
370   - delete_shard(WriteOnlyShard)
371   -All transformations applied. Have a nice day!
  352 +STARTING:
  353 + [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
  354 +COPIES:
  355 + localhost/s_0_001_a -> 127.0.0.1/s_0_0001
  356 +FINISHING:
  357 + [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
  358 +All transformations applied. Total time elapsed: 10
372 359 EOF
373 360
374 361 nameserver[:shards].should == [ info("127.0.0.1", "s_0_0001", "TestShard"),
@@ -392,7 +379,7 @@ def nameserver
392 379 end
393 380 gizzmo "-f reload"
394 381
395   - gizzmo('-f -T 0 transform "ReplicatingShard -> TestShard(localhost,1,Int,Int)" "ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
  382 + gizzmo('-f -T0 transform --poll-interval=1 "ReplicatingShard -> TestShard(localhost,1,Int,Int)" "ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
396 383 ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1)) :
397 384 PREPARE
398 385 create_shard(TestShard/127.0.0.1)
@@ -407,44 +394,19 @@ def nameserver
407 394 remove_link(ReplicatingShard -> WriteOnlyShard)
408 395 delete_shard(WriteOnlyShard)
409 396 Applied to:
410   - [0] 1 -> localhost/s_0_001_replicating
411   - [0] 2 -> localhost/s_0_002_replicating
412   -
413   -Jobs starting:
414   - [0] 1 -> localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
415   - PREPARE
416   - create_shard(TestShard/127.0.0.1)
417   - create_shard(WriteOnlyShard)
418   - add_link(WriteOnlyShard -> TestShard/127.0.0.1)
419   - add_link(ReplicatingShard -> WriteOnlyShard)
420   - [0] 2 -> localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
421   - PREPARE
422   - create_shard(TestShard/127.0.0.1)
423   - create_shard(WriteOnlyShard)
424   - add_link(WriteOnlyShard -> TestShard/127.0.0.1)
425   - add_link(ReplicatingShard -> WriteOnlyShard)
426   -Reloading nameserver configuration.
427   -Scheduling copies:
428   - [0] 1 -> localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
429   - COPY
430   - copy_shard(TestShard/127.0.0.1)
431   - [0] 2 -> localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
432   - COPY
433   - copy_shard(TestShard/127.0.0.1)
434   -Jobs finishing:
435   - [0] 1 -> localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
436   - CLEANUP
437   - add_link(ReplicatingShard -> TestShard/127.0.0.1)
438   - remove_link(WriteOnlyShard -> TestShard/127.0.0.1)
439   - remove_link(ReplicatingShard -> WriteOnlyShard)
440   - delete_shard(WriteOnlyShard)
441   - [0] 2 -> localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
442   - CLEANUP
443   - add_link(ReplicatingShard -> TestShard/127.0.0.1)
444   - remove_link(WriteOnlyShard -> TestShard/127.0.0.1)
445   - remove_link(ReplicatingShard -> WriteOnlyShard)
446   - delete_shard(WriteOnlyShard)
447   -All transformations applied. Have a nice day!
  397 + [0] 1 = localhost/s_0_001_replicating
  398 + [0] 2 = localhost/s_0_002_replicating
  399 +
  400 +STARTING:
  401 + [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
  402 + [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
  403 +COPIES:
  404 + localhost/s_0_001_a -> 127.0.0.1/s_0_0001
  405 + localhost/s_0_002_a -> 127.0.0.1/s_0_0002
  406 +FINISHING:
  407 + [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
  408 + [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
  409 +All transformations applied. Total time elapsed: 01
448 410 EOF
449 411
450 412 nameserver[:shards].should == [ info("127.0.0.1", "s_0_0001", "TestShard"),
59 test/scheduler_spec.rb
... ... @@ -0,0 +1,59 @@
  1 +require File.expand_path('../spec_helper', __FILE__)
  2 +
  3 +describe Gizzard::Transformation::Scheduler do
  4 +
  5 + before do
  6 + @nameserver = stub!.subject
  7 + stub(@nameserver).dryrun? { false }
  8 +
  9 + @transformations = {}
  10 + @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations)
  11 + end
  12 +
  13 + describe "busy_shards" do
  14 + it "memoizes" do
  15 + shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
  16 + mock(@nameserver).get_busy_shards { shards }
  17 +
  18 + @scheduler.busy_shards.should == shards.map {|s| s.id }
  19 + @scheduler.busy_shards.should == shards.map {|s| s.id }
  20 + end
  21 +
  22 + it "resets after calling reload_busy_shards" do
  23 + shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
  24 + mock(@nameserver).get_busy_shards { shards }.twice
  25 +
  26 + @scheduler.busy_shards.should == shards.map {|s| s.id }
  27 + @scheduler.reload_busy_shards
  28 + @scheduler.busy_shards.should == shards.map {|s| s.id }
  29 + end
  30 + end
  31 +
  32 + describe "busy_hosts" do
  33 + it "returns a list of hosts over the threshold of copies per host" do
  34 + shards = []
  35 + stub(@nameserver).get_busy_shards { shards }
  36 + @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations, :copies_per_host => 2)
  37 +
  38 + @scheduler.busy_hosts.should == []
  39 +
  40 + shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
  41 + @scheduler.reload_busy_shards
  42 + @scheduler.busy_hosts.should == []
  43 +
  44 + shards = [info('127.0.0.1', 't_0_0001', 'TestShard'), info('127.0.0.1', 't_0_0002', 'TestShard')]
  45 + @scheduler.reload_busy_shards
  46 + @scheduler.busy_hosts.should == ['127.0.0.1']
  47 + end
  48 +
  49 + it "respects passed in extra hosts" do
  50 + shards = []
  51 + stub(@nameserver).get_busy_shards { shards }
  52 + @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations, :copies_per_host => 2)
  53 +
  54 + @scheduler.busy_hosts.should == []
  55 + @scheduler.busy_hosts(["127.0.0.1"]).should == []
  56 + @scheduler.busy_hosts(["127.0.0.1", "127.0.0.1"]).should == ["127.0.0.1"]
  57 + end
  58 + end
  59 +end

0 comments on commit 0a67463

Please sign in to comment.
Something went wrong with that request. Please try again.