Skip to content
This repository has been archived by the owner on Sep 18, 2021. It is now read-only.

Commit

Permalink
carry around a config for generating table names.
Browse files Browse the repository at this point in the history
  • Loading branch information
Robey Pointer committed Aug 31, 2010
1 parent aade98f commit fe4ca69
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 11 deletions.
14 changes: 10 additions & 4 deletions lib/gizzard/migrator.rb
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
module Gizzard
class MigratorConfig
attr_accessor :namespace, :table_prefix, :graph_id, :source_type, :destination_type
end

class Migrator
BALANCE_TOLERANCE = 1

Expand All @@ -11,11 +15,12 @@ class Migrator
# populated via derive_changes
attr_reader :new_templates, :unrecognized_templates, :similar_templates, :unchanged_templates

def initialize(existing_map, config_templates, default_total_shards, forwarding_generator)
def initialize(existing_map, config_templates, default_total_shards, config, forwarding_generator)
@configured_templates = config_templates
@existing_map = existing_map
@existing_templates = existing_map.keys
@total_shards = @existing_map.values.map { |a| a.length }.inject { |a, b| a + b } || default_total_shards
@config = config
@forwarding_generator = forwarding_generator
derive_changes
end
Expand Down Expand Up @@ -66,12 +71,13 @@ def generate_new_forwardings(shard_count)
step_size = FORWARDING_SPACE / shard_count
(0...shard_count).map { |i| FORWARDING_SPACE_MIN + (i * step_size) }.sort_by { rand }
else
step_size = (1 << 60) / options[:count]
step_size = (1 << 60) / shard_count
(0...shard_count).map { |i| step_size * i }
end

bases.each_with_index do |base_id, i|
forwardings[base_id] = "status_%05d" % (i + 1)
table_name = [ @config.namespace, @config.table_prefix, @config.graph_id, "%04d" % i ].compact.join("_")
forwardings[base_id] = table_name
end

forwardings
Expand Down Expand Up @@ -136,7 +142,7 @@ def generate_transformations(existing, configured)
# transformation for each one.
(configured_shards.to_a - existing_shards.to_a).inject({}) do |transformations, (shard, to)|
from = existing_shards[shard]
(transformations[[from, to]] ||= Transformation.new(from, to, [])).shard_ids << shard
(transformations[[from, to]] ||= Transformation.new(from, to, [], @config)).shard_ids << shard
transformations
end.values
end
Expand Down
13 changes: 8 additions & 5 deletions lib/gizzard/shard_template.rb
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ def replicating?
type =~ /ReplicatingShard/
end

def short_type
type.split(".").last
end

def identifier
replicating? ? type.to_s : "#{type}:#{host}"
end
Expand Down Expand Up @@ -91,13 +95,12 @@ def inspect

# Materialization

def to_shard_id(namespace, table_prefix, graph_id, shard_number)
table_name = [ namespace, table_prefix, graph_id, "%04d" % shard_number ].compact.join("_")
Thrift::ShardId.new(host, table_name)
def to_shard_id(table_name)
Thrift::ShardId.new(host, concrete? ? table_name : table_name + "_" + short_type)
end

def to_shard_info(namespace, table_prefix, graph_id, shard_number, source_type, destination_type)
Thrift::ShardInfo.new(to_shard_id(namespace, table_prefix, graph_id, shard_number), type, source_type, destination_type, 0)
def to_shard_info(config, table_name)
Thrift::ShardInfo.new(to_shard_id(table_name), type, config.source_type, config.destination_type, 0)
end


Expand Down
5 changes: 3 additions & 2 deletions lib/gizzard/transformation.rb
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,11 @@ class Transformation
:copy_shard => 4
}

def initialize(from_template, to_template, shard_ids)
def initialize(from_template, to_template, shard_ids, config)
@from = from_template
@to = to_template
@shard_ids = shard_ids
@config = config
end

def paginate(page_size = DEFAULT_CONCURRENT_COPIES)
Expand Down Expand Up @@ -264,7 +265,7 @@ def id(shard)

def info(shard)
shard_id = @current_shard_id or raise "no current shard id!"
shard.to_shard_info(shard_id)
shard.to_shard_info(@config, shard_id)
end

def copy_destination?(shard)
Expand Down

0 comments on commit fe4ca69

Please sign in to comment.