Skip to content
Browse files

removed gizzmo code, moved to separate repo, pending amac approval.

  • Loading branch information...
1 parent 114554c commit 784487ec282805a4e8ef04b44205a052d84a8912 Kyle Maxwell committed
View
3 script/gizzmo
@@ -1,3 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__) + "/../src/main/ruby"
-load "gizzmo.rb"
View
6 src/main/ruby/gizzard.rb
@@ -1,6 +0,0 @@
-$: << File.dirname(__FILE__)
-module Gizzard; end
-require "gizzard/consts"
-require "gizzard/thrift"
-require "gizzard/commands"
-require "gizzard/dsl"
View
57 src/main/ruby/gizzard/_deprecated.rb
@@ -1,57 +0,0 @@
-module Gizzard
- module Recipes
- def setup_migration_shards(source, destination)
- raise "Cannot migrate to the same shard" if source == destination
-
- write_only_shard_id = shard_id("localhost", source.table_prefix + "_migrate_write_only")
- write_only_shard = shard_info(write_only_shard_id, Consts::WRITE_ONLY_SHARD, "", "", 0)
- create_shard(write_only_shard)
- add_link(write_only_shard_id, destination)
-
- replicating_shard_id = shard_id("localhost", source.table_prefix + "_migrate_replicating")
- replicating_shard = shard_info(replicating_shard_id, Consts::REPLICATING_SHARD, "", "", 0)
- create_shard(replicating_shard)
-
- list_upward_links(source).each do |link|
- add_link(link.up_id, replicating_shard_id, link.weight)
- remove_link(link.up_id, link.down_id)
- end
-
- add_link(replicating_shard_id, source, 1)
- add_link(replicating_shard_id, write_only_shard_id, 1)
- replace_forwarding(source, replicating_shard_id)
-
- puts "Replication structure set up: #{source} => #{destination}."
- end
-
- def finish_migration(source, destination)
- replicating_shard_id = shard_id("localhost", source.table_prefix + "_migrate_replicating")
- write_only_shard_id = shard_id("localhost", source.table_prefix + "migrate_write_only")
- list_upward_links(replicating_shard_id).each do |link|
- add_link(link.up_id, destination, link.weight)
- remove_link(link.up_id, replicating_shard_id)
- end
-
- replace_forwarding(replicating_shard_id, destination)
- delete_shard(replicating_shard_id)
- delete_shard(write_only_shard_id)
- delete_shard(source_id)
- end
- end
-
- class Manager
- include DSL
- include Recipes
-
- def initialize(hostname, port = 7917)
- @client = hostname.is_a?(Gizzard::Thrift::ShardManager) ? hostname : Gizzard::Thrift::ShardManager.new(hostname, port)
- end
-
- def method_missing(method, *args, &block)
- @client.send(method, *args, &block)
- end
-
- def inspect_shard_tree(root_shard, prefix = '')
- end
- end
-end
View
150 src/main/ruby/gizzard/commands.rb
@@ -1,150 +0,0 @@
-module Gizzard
- class Command
- include Thrift
-
- def self.run(command_name, *args)
- Gizzard.const_get("#{classify(command_name)}Command").new(*args).run
- end
-
- def self.classify(string)
- string.split(/\W+/).map{|s| s.capitalize }.join("")
- end
-
- attr_reader :service, :global_options, :argv, :command_options
- def initialize(service, global_options, argv, command_options)
- @service = service
- @global_options = global_options
- @argv = argv
- @command_options = command_options
- end
-
- def help!(message = nil)
- raise HelpNeededError, message
- end
- end
-
- class ReloadCommand < Command
- def run
- puts "Are you sure? Reloading will affect production services immediately! (Type 'yes')"
- if gets.chomp == "yes"
- service.reload_forwardings
- else
- STDERR.puts "aborted"
- end
- end
- end
-
- class LinkCommand < Command
- def run
- up_id, down_id, weight = argv
- help! if argv.length != 3
- weight = weight.to_i
- up_id = ShardId.parse(up_id)
- down_id = ShardId.parse(down_id)
- link = LinkInfo.new(up_id, down_id, weight)
- service.add_link(link.up_id, link.down_id, link.weight)
- puts link.to_unix
- end
- end
-
- class UnlinkCommand < Command
- def run
- up_id, down_id = argv
- up_id = ShardId.parse(up_id)
- down_id = ShardId.parse(down_id)
- service.remove_link(up_id, down_id)
- end
- end
-
- class UnwrapCommand < Command
- def run
- shard_ids = argv
- help! "No shards specified" if shard_ids.empty?
- shard_ids.each do |shard_id_string|
- shard_id = ShardId.parse(shard_id_string)
- service.list_upward_links(shard_id).each do |uplink|
- service.list_downward_links(shard_id).each do |downlink|
- service.add_link(uplink.up_id, downlink.down_id, uplink.weight)
- new_link = LinkInfo.new(uplink.up_id, downlink.down_id, uplink.weight)
- service.remove_link(uplink.up_id, uplink.down_id)
- service.remove_link(downlink.up_id, downlink.down_id)
- puts new_link.to_unix
- end
- end
- end
- end
- end
-
- class CreateCommand < Command
- def run
- help! if argv.length != 3
- host, table, class_name = argv
- busy = 0
- source_type = command_options.source_type || ""
- destination_type = command_options.destination_type || ""
- service.create_shard(ShardInfo.new(shard_id = ShardId.new(host, table), class_name, source_type, destination_type, busy))
- service.get_shard(shard_id)
- puts shard_id.to_unix
- end
- end
-
- class LinksCommand < Command
- def run
- shard_ids = @argv
- shard_ids.each do |shard_id_text|
- shard_id = ShardId.parse(shard_id_text)
- service.list_upward_links(shard_id).each do |link_info|
- puts link_info.to_unix
- end
- service.list_downward_links(shard_id).each do |link_info|
- puts link_info.to_unix
- end
- end
- end
- end
-
- class InfoCommand < Command
- def run
- shard_ids = @argv
- shard_ids.each do |shard_id|
- shard_info = service.get_shard(ShardId.parse(shard_id))
- puts shard_info.to_unix
- end
- end
- end
-
- class WrapCommand < Command
- def self.derive_wrapper_shard_id(shard_info, wrapping_class_name)
- prefix_prefix = wrapping_class_name.split(".").last.downcase.gsub("shard", "") + "_"
- ShardId.new(shard_info.id.hostname, prefix_prefix + shard_info.id.table_prefix)
- end
-
- def run
- class_name, *shard_ids = @argv
- help! "No shards specified" if shard_ids.empty?
- shard_ids.each do |shard_id_string|
- shard_id = ShardId.parse(shard_id_string)
- shard_info = service.get_shard(shard_id)
- service.create_shard(ShardInfo.new(wrapper_id = self.class.derive_wrapper_shard_id(shard_info, class_name), class_name, "", "", 0))
-
- existing_links = service.list_upward_links(shard_id)
- service.add_link(wrapper_id, shard_id, 1)
- existing_links.each do |link_info|
- service.add_link(link_info.up_id, wrapper_id, link_info.weight)
- service.remove_link(link_info.up_id, link_info.down_id)
- end
- puts wrapper_id.to_unix
- end
- end
- end
-
- class FindCommand < Command
- def run
- help!("host is a required option") unless command_options.shard_host
- service.shards_for_hostname(command_options.shard_host).each do |shard|
- next if command_options.shard_type && shard.class_name !~ Regexp.new(command_options.shard_type)
- puts shard.id.to_unix
- end
- end
- end
-end
View
6 src/main/ruby/gizzard/consts.rb
@@ -1,6 +0,0 @@
-module Gizzard
- module Consts
- WRITE_ONLY_SHARD = "com.twitter.gizzard.shards.WriteOnlyShard"
- REPLICATING_SHARD = "com.twitter.gizzard.shards.ReplicatingShard"
- end
-end
View
13 src/main/ruby/gizzard/dsl.rb
@@ -1,13 +0,0 @@
-module Gizzard
- module DSL
- [:shard_id, :shard_info, :link_info, :shard_migration, :forwarding].each do |method|
- struct_class = method.to_s.capitalize.gsub(/_(.)/) { $1.upcase }
- module_eval "def #{method}(*args); Thrift::#{struct_class}.new(*args) end"
- end
-
- def shard(hostname, table_prefix, *rest)
- id = shard_id(hostname, table_prefix)
- shard_info(id, *rest)
- end
- end
-end
View
123 src/main/ruby/gizzard/thrift.rb
@@ -1,123 +0,0 @@
-#!/usr/bin/env ruby
-require 'vendor/thrift_client/simple'
-
-module Gizzard
- module Thrift
- T = ThriftClient::Simple
-
- def self.struct(*args)
- T::StructType.new(*args)
- end
-
- ShardException = T.make_exception(:ShardException,
- T::Field.new(:description, T::STRING, 1)
- )
-
- ShardId = T.make_struct(:ShardId,
- T::Field.new(:hostname, T::STRING, 1),
- T::Field.new(:table_prefix, T::STRING, 2)
- )
-
- class ShardId
- def inspect
- "#{hostname}/#{table_prefix}"
- end
-
- alias_method :to_unix, :inspect
-
- def self.parse(string)
- new(*string.split("/"))
- end
- end
-
- ShardInfo = T.make_struct(:ShardInfo,
- T::Field.new(:id, struct(ShardId), 1),
- T::Field.new(:class_name, T::STRING, 2),
- T::Field.new(:source_type, T::STRING, 3),
- T::Field.new(:destination_type, T::STRING, 4),
- T::Field.new(:busy, T::I32, 5)
- )
-
- class ShardInfo
- def busy?
- busy && busy > 0
- end
-
- def inspect(short = false)
- "#{id.inspect}" + (busy? ? " (BUSY)" : "")
- end
-
- def to_unix
- [id.to_unix, class_name, busy? ? "busy" : "unbusy"].join("\t")
- end
- end
-
- LinkInfo = T.make_struct(:LinkInfo,
- T::Field.new(:up_id, struct(ShardId), 1),
- T::Field.new(:down_id, struct(ShardId), 2),
- T::Field.new(:weight, T::I32, 3)
- )
-
- class LinkInfo
- def inspect
- "#{up_id.inspect} -> #{down_id.inspect}" + (weight == 1 ? "" : " <#{weight}>")
- end
-
- def to_unix
- [up_id.to_unix, down_id.to_unix, weight].join("\t")
- end
-
- end
-
- ShardMigration = T.make_struct(:ShardMigration,
- T::Field.new(:source_id, struct(ShardId), 1),
- T::Field.new(:destination_id, struct(ShardId), 2)
- )
-
- Forwarding = T.make_struct(:Forwarding,
- T::Field.new(:table_id, T::I32, 1),
- T::Field.new(:base_id, T::I64, 2),
- T::Field.new(:shard_id, struct(ShardId), 3)
- )
-
- class Forwarding
- #FIXME table_id is not human-readable
- def inspect
- "[#{table_id}] #{base_id.to_s(16)} -> #{shard_id.inspect}"
- end
- end
-
- class ShardManager < T::ThriftService
- thrift_method :create_shard, void, field(:shard, struct(ShardInfo), 1), :throws => exception(ShardException)
- thrift_method :delete_shard, void, field(:id, struct(ShardId), 1)
- thrift_method :get_shard, struct(ShardInfo), field(:id, struct(ShardId), 1)
-
- thrift_method :add_link, void, field(:up_id, struct(ShardId), 1), field(:down_id, struct(ShardId), 2), field(:weight, i32, 3)
- thrift_method :remove_link, void, field(:up_id, struct(ShardId), 1), field(:down_id, struct(ShardId), 2)
-
- thrift_method :list_upward_links, list(struct(LinkInfo)), field(:id, struct(ShardId), 1)
- thrift_method :list_downward_links, list(struct(LinkInfo)), field(:id, struct(ShardId), 1)
-
- thrift_method :get_child_shards_of_class, list(struct(ShardInfo)), field(:parent_id, struct(ShardId), 1), field(:class_name, string, 2)
-
- thrift_method :mark_shard_busy, void, field(:id, struct(ShardId), 1), field(:busy, i32, 2)
- thrift_method :copy_shard, void, field(:source_id, struct(ShardId), 1), field(:destination_id, struct(ShardId), 2)
-
- thrift_method :set_forwarding, void, field(:forwarding, struct(Forwarding), 1)
- thrift_method :replace_forwarding, void, field(:old_id, struct(ShardId), 1), field(:new_id, struct(ShardId), 2)
-
- thrift_method :get_forwarding, struct(Forwarding), field(:table_id, i32, 1), field(:base_id, i64, 2)
- thrift_method :get_forwarding_for_shard, struct(Forwarding), field(:shard_id, struct(ShardId), 1)
-
- thrift_method :get_forwardings, list(struct(Forwarding))
- thrift_method :reload_forwardings, void
-
- thrift_method :find_current_forwarding, struct(ShardInfo), field(:table_id, i32, 1), field(:id, i64, 2)
-
- thrift_method :shards_for_hostname, list(struct(ShardInfo)), field(:hostname, string, 1)
- thrift_method :get_busy_shards, list(struct(ShardInfo))
-
- thrift_method :rebuild_schema, void
- end
- end
-end
View
172 src/main/ruby/gizzmo.rb
@@ -1,172 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)
-class HelpNeededError < RuntimeError; end
-require "optparse"
-require "ostruct"
-require "gizzard"
-require "yaml"
-
-
-# Container for parsed options
-global_options = OpenStruct.new
-subcommand_options = OpenStruct.new
-
-# Leftover arguments
-argv = nil
-
-begin
- YAML.load_file(File.join(ENV["HOME"], ".gizzmorc")).each do |k, v|
- global_options.send("#{k}=", v)
- end
-rescue Errno::ENOENT
- # Do nothing...
-rescue => e
- abort "Unknown error loading ~/.gizzmorc: #{e.message}"
-end
-
-subcommands = {
- 'create' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} create [options] HOST TABLE_PREFIX CLASS_NAME"
-
- opts.on("-s", "--source-type=TYPE") do |s|
- subcommand_options.source_type = s
- end
-
- opts.on("-d", "--destination-type=TYPE") do |s|
- subcommand_options.destination_type = s
- end
- end,
- 'wrap' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} wrap CLASS_NAME SHARD_ID_TO_WRAP [MORE SHARD_IDS...]"
- end,
- 'unwrap' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} unwrap SHARD_ID_TO_REMOVE [MORE SHARD_IDS]"
- end,
- 'find' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} find [options]"
-
- opts.on("-t", "--type=TYPE", "Return only shards of the specified TYPE") do |shard_type|
- subcommand_options.shard_type = shard_type
- end
-
- opts.on("-H", "--host=HOST", "HOST of shard") do |shard_host|
- subcommand_options.shard_host = shard_host
- end
- end,
- 'links' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} links SHARD_ID [MORE SHARD_IDS...]"
- end,
- 'info' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} info SHARD_ID [MORE SHARD_IDS...]"
- end,
- 'reload' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} reload"
- end,
- 'link' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} link PARENT_SHARD_ID CHILD_SHARD_ID WEIGHT"
- end,
- 'unlink' => OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} unlink PARENT_SHARD_ID CHILD_SHARD_ID"
- end
-}
-
-global = OptionParser.new do |opts|
- opts.banner = "Usage: #{$0} [global-options] SUBCOMMAND [subcommand-options]"
- opts.separator ""
- opts.separator "Subcommands:"
- subcommands.keys.compact.sort.each do |sc|
- opts.separator " #{sc}"
- end
- opts.separator ""
- opts.separator "You can type `#{$0} help SUBCOMMAND` for help on a specific subcommand."
- opts.separator ""
- opts.separator "Global options:"
-
- opts.on("-H", "--host=HOSTNAME", "HOSTNAME of remote thrift service") do |host|
- global_options.host = host
- end
-
- opts.on("-P", "--port=PORT", "PORT of remote thrift service") do |port|
- global_options.port = port
- end
-
- opts.on("-d", "--dry-run", "") do |port|
- global_options.dry = true
- end
-
- opts.on("-C", "--config=YAML_FILE", "YAML_FILE of option key/values") do |file|
- YAML.load(File.open(file)).each do |k, v|
- global_options.send("#{k}=", v)
- end
- end
-end
-
-# Print banner if no args
-if ARGV.length == 0
- STDERR.puts global
- exit 1
-end
-
-# This
-def process_nested_parsers(global, subcommands)
- begin
- global.order!(ARGV) do |subcommand_name|
- # puts args.inspect
- subcommand = subcommands[subcommand_name]
- argv = subcommand ? subcommand.parse!(ARGV) : ARGV
- return subcommand_name, argv
- end
- rescue => e
- STDERR.puts e.message
- exit 1
- end
-end
-
-
-subcommand_name, argv = process_nested_parsers(global, subcommands)
-
-# Print help sub-banners
-if subcommand_name == "help"
- STDERR.puts subcommands[argv.shift] || global
- exit 1
-end
-
-unless subcommands.include?(subcommand_name)
- STDERR.puts "Subcommand not found: #{subcommand_name}"
- exit 1
-end
-
-if global_options.dry
- puts "Connecting to service on #{global_options.host}:#{global_options.port}"
- puts "Sending #{subcommand_name} with #{argv.inspect}, #{subcommand_options.inspect}"
-else
- service = Gizzard::Thrift::ShardManager.new(global_options.host, global_options.port)
- begin
- Gizzard::Command.run(subcommand_name, service, global_options, argv, subcommand_options)
- rescue HelpNeededError => e
- if e.class.name != e.message
- STDERR.puts("=" * 80)
- STDERR.puts e.message
- STDERR.puts("=" * 80)
- end
- STDERR.puts subcommands[subcommand_name]
- exit 1
- rescue ThriftClient::Simple::ThriftException => e
- STDERR.puts e.message
- exit 1
- end
-
- # include Gizzard::Thrift
- # 20.times do |i|
- # i += 20
- # repl = service.create_shard(ShardInfo.new(repl_id = ShardId.new("localhost", "table_repl_#{i}"), "com.twitter.service.flock.edges.ReplicatingShard", "", "", 0))
- # a = service.create_shard(ShardInfo.new(a_id = ShardId.new("localhost", "table_a_#{i}"), "com.twitter.service.flock.edges.SqlShard", "INT UNSIGNED", "INT UNSIGNED", 0))
- # b = service.create_shard(ShardInfo.new(b_id = ShardId.new("localhost", "table_b_#{i}"), "com.twitter.service.flock.edges.SqlShard", "INT UNSIGNED", "INT UNSIGNED", 0))
- #
- # service.add_link(repl_id, a_id, 2)
- # service.add_link(repl_id, b_id, 1)
- #
- # service.set_forwarding(Forwarding.new(0, i * 1000, repl_id))
- # end
-
-end
View
334 src/main/ruby/vendor/thrift_client/simple.rb
@@ -1,334 +0,0 @@
-require 'socket'
-require 'getoptlong'
-
-class ThriftClient
-
- # This is a simplified form of thrift, useful for clients only, and not
- # making any attempt to have good performance. It's intended to be used by
- # small command-line tools that don't want to install a dozen ruby files.
- module Simple
- VERSION_1 = 0x8001
-
- # message types
- CALL, REPLY, EXCEPTION = (1..3).to_a
-
- # value types
- STOP, VOID, BOOL, BYTE, DOUBLE, _, I16, _, I32, _, I64, STRING, STRUCT, MAP, SET, LIST = (0..15).to_a
-
- FORMATS = {
- BYTE => "c",
- DOUBLE => "G",
- I16 => "n",
- I32 => "N",
- }
-
- SIZES = {
- BYTE => 1,
- DOUBLE => 8,
- I16 => 2,
- I32 => 4,
- }
-
- module ComplexType
- module Extends
- def type_id=(n)
- @type_id = n
- end
-
- def type_id
- @type_id
- end
- end
-
- module Includes
- def to_i
- self.class.type_id
- end
-
- def to_s
- args = self.values.map { |v| self.class.type_id == STRUCT ? v.name : v.to_s }.join(", ")
- "#{self.class.name}.new(#{args})"
- end
- end
- end
-
- def self.make_type(type_id, name, *args)
- klass = Struct.new("STT_#{name}", *args)
- klass.send(:extend, ComplexType::Extends)
- klass.send(:include, ComplexType::Includes)
- klass.type_id = type_id
- klass
- end
-
- ListType = make_type(LIST, "ListType", :element_type)
- MapType = make_type(MAP, "MapType", :key_type, :value_type)
- SetType = make_type(SET, "SetType", :element_type)
- StructType = make_type(STRUCT, "StructType", :struct_class)
-
- class << self
- def pack_value(type, value)
- case type
- when BOOL
- [ value ? 1 : 0 ].pack("c")
- when STRING
- [ value.size, value ].pack("Na*")
- when I64
- [ value >> 32, value & 0xffffffff ].pack("NN")
- when ListType
- [ type.element_type.to_i, value.size ].pack("cN") + value.map { |item| pack_value(type.element_type, item) }.join("")
- when MapType
- [ type.key_type.to_i, type.value_type.to_i, value.size ].pack("ccN") + value.map { |k, v| pack_value(type.key_type, k) + pack_value(type.value_type, v) }.join("")
- when SetType
- [ type.element_type.to_i, value.size ].pack("cN") + value.map { |item| pack_value(type.element_type, item) }.join("")
- when StructType
- value._pack
- else
- [ value ].pack(FORMATS[type])
- end
- end
-
- def pack_request(method_name, arg_struct, request_id=0)
- [ VERSION_1, CALL, method_name.to_s.size, method_name.to_s, request_id, arg_struct._pack ].pack("nnNa*Na*")
- end
-
- def read_value(s, type)
- case type
- when BOOL
- s.read(1).unpack("c").first != 0
- when STRING
- len = s.read(4).unpack("N").first
- s.read(len)
- when I64
- hi, lo = s.read(8).unpack("NN")
- rv = (hi << 32) | lo
- (rv >= (1 << 63)) ? (rv - (1 << 64)) : rv
- when LIST
- read_list(s)
- when MAP
- read_map(s)
- when STRUCT
- read_struct(s, UnknownStruct)
- when ListType
- read_list(s, type.element_type)
- when MapType
- read_map(s, type.key_type, type.value_type)
- when StructType
- read_struct(s, type.struct_class)
- else
- rv = s.read(SIZES[type]).unpack(FORMATS[type]).first
- case type
- when I16
- (rv >= (1 << 15)) ? (rv - (1 << 16)) : rv
- when I32
- (rv >= (1 << 31)) ? (rv - (1 << 32)) : rv
- else
- rv
- end
- end
- end
-
- def read_list(s, element_type=nil)
- etype, len = s.read(5).unpack("cN")
- expected_type = (element_type and element_type.to_i == etype.to_i) ? element_type : etype
- rv = []
- len.times do
- rv << read_value(s, expected_type)
- end
- rv
- end
-
- def read_map(s, key_type=nil, value_type=nil)
- ktype, vtype, len = s.read(6).unpack("ccN")
- rv = {}
- expected_key_type, expected_value_type = if key_type and value_type and key_type.to_i == ktype and value_type.to_i == vtype
- [ key_type, value_type ]
- else
- [ ktype, vtype ]
- end
- len.times do
- key = read_value(s, expected_key_type)
- value = read_value(s, expected_value_type)
- rv[key] = value
- end
- rv
- end
-
- def read_struct(s, struct_class)
- struct = struct_class.new
- while true
- ftype = s.read(1).unpack("c").first
- return struct if ftype == STOP
- fid = s.read(2).unpack("n").first
-
- if field = struct_class._fields.find { |f| (f.fid == fid) and (f.type.to_i == ftype) }
- struct[field.name] = read_value(s, field.type)
- else
- $stderr.puts "Warning: Unknown struct field encountered. (recieved id: #{fid})"
- raise "Warning: Unknown struct field encountered. (recieved id: #{fid})"
- read_value(s, ftype)
- end
- end
- end
-
- def read_response(s, rv_class)
- version, message_type, method_name_len = s.read(8).unpack("nnN")
- method_name = s.read(method_name_len)
- seq_id = s.read(4).unpack("N").first
- if message_type == EXCEPTION
- exception = read_struct(s, ExceptionStruct)
- raise ThriftException, exception.message
- end
- response = read_struct(s, rv_class)
- raise response.ex if response.respond_to?(:ex) and response.ex
- [ method_name, seq_id, response.rv ]
- end
- end
-
- ## ----------------------------------------
-
- class Field
- attr_accessor :name, :type, :fid
-
- def initialize(name, type, fid)
- @name = name
- @type = type
- @fid = fid
- end
-
- def pack(value)
- value.nil? ? "" : [ type.to_i, fid, ThriftClient::Simple.pack_value(type, value) ].pack("cna*")
- end
- end
-
- class ThriftException < RuntimeError
- def initialize(reason)
- @reason = reason
- end
-
- def to_s
- "ThriftException(#{@reason.inspect})"
- end
- end
-
- module ThriftStruct
- module Include
- def _pack
- self.class._fields.map { |f| f.pack(self[f.name]) }.join + [ STOP ].pack("c")
- end
- end
-
- module Extend
- def _fields
- @fields
- end
-
- def _fields=(f)
- @fields = f
- end
- end
- end
-
- def self.make_struct(name, *fields)
- st_name = "ST_#{name.to_s.tr(':', '_')}"
- if Struct.constants.include?(st_name)
- warn "#{caller[0]}: Struct::#{st_name} is already defined; returning original class."
- Struct.const_get(st_name)
- else
- names = fields.map { |f| f.name.to_sym }
- klass = Struct.new(st_name, *names)
- klass.send(:include, ThriftStruct::Include)
- klass.send(:extend, ThriftStruct::Extend)
- klass._fields = fields
- klass
- end
- end
-
- def self.make_exception(name, *fields)
- struct_class = self.make_struct(name, *fields)
- ex_class = Class.new(StandardError)
-
- (class << struct_class; self end).send(:define_method, :exception_class) { ex_class }
- (class << ex_class; self end).send(:define_method, :struct_class) { struct_class }
-
- ex_class.class_eval do
- attr_reader :struct
-
- def initialize
- @struct = self.class.struct_class.new
- end
-
- def self._fields
- struct_class._fields
- end
-
- def to_s
- method = [:message, :description].find {|m| struct.respond_to? m }
- struct.send method || :to_s
- end
-
- alias message to_s
-
- def method_missing(method, *args)
- struct.send(method, *args)
- end
- end
-
- ex_class
- end
-
- ExceptionStruct = make_struct(:ProtocolException, Field.new(:message, STRING, 1), Field.new(:type, I32, 2))
- UnknownStruct = make_struct(:Unknown)
-
- class ThriftService
- def initialize(host, port)
- @host = host
- @port = port
- end
-
- def self._arg_structs
- @_arg_structs = {} if @_arg_structs.nil?
- @_arg_structs
- end
-
- def self.thrift_method(name, rtype, *args)
- options = args.last.is_a?(Hash) ? args.pop : {}
- fields = [ ThriftClient::Simple::Field.new(:rv, rtype, 0),
- (options[:throws] ? ThriftClient::Simple::Field.new(:ex, options[:throws], 1) : nil)
- ].compact
-
- arg_struct = ThriftClient::Simple.make_struct("Args__#{self.name}__#{name}", *args)
- rv_struct = ThriftClient::Simple.make_struct("Retval__#{self.name}__#{name}", *fields)
-
- _arg_structs[name.to_sym] = [ arg_struct, rv_struct ]
-
- arg_names = args.map { |a| a.name.to_s }.join(", ")
- class_eval "def #{name}(#{arg_names}); _proxy(:#{name}#{args.size > 0 ? ', ' : ''}#{arg_names}); end"
- end
-
- def _proxy(method_name, *args)
- cls = self.class.ancestors.find { |cls| cls.respond_to?(:_arg_structs) and cls._arg_structs[method_name.to_sym] }
- arg_class, rv_class = cls._arg_structs[method_name.to_sym]
- arg_struct = arg_class.new(*args)
- sock = TCPSocket.new(@host, @port)
- sock.write(ThriftClient::Simple.pack_request(method_name, arg_struct))
- rv = ThriftClient::Simple.read_response(sock, rv_class)
- sock.close
- rv[2]
- end
-
- # convenience. robey is lazy.
- { :field => "Field.new",
- :struct => "StructType.new",
- :exception => "StructType.new",
- :list => "ListType.new",
- :map => "MapType.new",
- }.each do |new_name, old_name|
- class_eval "def self.#{new_name}(*args); ThriftClient::Simple::#{old_name}(*args); end"
- end
-
-# alias exception struct
-
- [ :void, :bool, :byte, :double, :i16, :i32, :i64, :string ].each { |sym| class_eval "def self.#{sym}; ThriftClient::Simple::#{sym.to_s.upcase}; end" }
- end
- end
-end
View
2 src/test/bash/config.yaml
@@ -1,2 +0,0 @@
-host: localhost
-port: 7917
View
0 src/test/bash/expected/empty-file.txt
No changes.
View
20 src/test/bash/expected/find-only-sql-shard-type.txt
@@ -1,20 +0,0 @@
-localhost/table_a_0
-localhost/table_a_1
-localhost/table_a_2
-localhost/table_a_3
-localhost/table_a_4
-localhost/table_a_5
-localhost/table_a_6
-localhost/table_a_7
-localhost/table_a_8
-localhost/table_a_9
-localhost/table_b_0
-localhost/table_b_1
-localhost/table_b_2
-localhost/table_b_3
-localhost/table_b_4
-localhost/table_b_5
-localhost/table_b_6
-localhost/table_b_7
-localhost/table_b_8
-localhost/table_b_9
View
1 src/test/bash/expected/help-info.txt
@@ -1 +0,0 @@
-Usage: ../../../script/gizzmo info SHARD_ID [MORE SHARD_IDS...]
View
30 src/test/bash/expected/info.txt
@@ -1,30 +0,0 @@
-localhost/table_a_0 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_1 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_2 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_3 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_4 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_5 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_6 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_7 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_8 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_a_9 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_0 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_1 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_2 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_3 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_4 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_5 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_6 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_7 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_8 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_b_9 com.twitter.service.flock.edges.SqlShard unbusy
-localhost/table_repl_0 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_1 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_2 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_3 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_4 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_5 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_6 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_7 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_8 com.twitter.service.flock.edges.ReplicatingShard unbusy
-localhost/table_repl_9 com.twitter.service.flock.edges.ReplicatingShard unbusy
View
2 src/test/bash/expected/links-for-replicating_table_b_0.txt
@@ -1,2 +0,0 @@
-localhost/table_repl_0 localhost/replicating_table_b_0 1
-localhost/replicating_table_b_0 localhost/table_b_0 1
View
1 src/test/bash/expected/links-for-table_b_0.txt
@@ -1 +0,0 @@
-localhost/replicating_table_b_0 localhost/table_b_0 1
View
2 src/test/bash/expected/links-for-table_repl_0.txt
@@ -1,2 +0,0 @@
-localhost/table_repl_0 localhost/table_a_0 2
-localhost/table_repl_0 localhost/replicating_table_b_0 1
View
30 src/test/bash/expected/original-find.txt
@@ -1,30 +0,0 @@
-localhost/table_a_0
-localhost/table_a_1
-localhost/table_a_2
-localhost/table_a_3
-localhost/table_a_4
-localhost/table_a_5
-localhost/table_a_6
-localhost/table_a_7
-localhost/table_a_8
-localhost/table_a_9
-localhost/table_b_0
-localhost/table_b_1
-localhost/table_b_2
-localhost/table_b_3
-localhost/table_b_4
-localhost/table_b_5
-localhost/table_b_6
-localhost/table_b_7
-localhost/table_b_8
-localhost/table_b_9
-localhost/table_repl_0
-localhost/table_repl_1
-localhost/table_repl_2
-localhost/table_repl_3
-localhost/table_repl_4
-localhost/table_repl_5
-localhost/table_repl_6
-localhost/table_repl_7
-localhost/table_repl_8
-localhost/table_repl_9
View
1 src/test/bash/expected/unwrapped-replicating_table_b_0.txt
@@ -1 +0,0 @@
-localhost/table_repl_0 localhost/table_b_0 1
View
1 src/test/bash/expected/unwrapped-table_b_0.txt
@@ -1 +0,0 @@
-localhost/table_repl_0 localhost/table_b_0 1
View
1 src/test/bash/expected/wrap-table_b_0.txt
@@ -1 +0,0 @@
-localhost/replicating_table_b_0
View
32 src/test/bash/recreate.sql
@@ -1,32 +0,0 @@
-CREATE TABLE IF NOT EXISTS shards (
- class_name VARCHAR(125) NOT NULL,
- table_prefix VARCHAR(125) NOT NULL,
- hostname VARCHAR(25) NOT NULL,
- source_type VARCHAR(125),
- destination_type VARCHAR(125),
- busy TINYINT NOT NULL DEFAULT 0,
-
- PRIMARY KEY primary_key_table_prefix_hostname (hostname, table_prefix)
-) ENGINE=INNODB;
-
-CREATE TABLE IF NOT EXISTS shard_children (
- parent_hostname VARCHAR(125) NOT NULL,
- parent_table_prefix VARCHAR(125) NOT NULL,
- child_hostname VARCHAR(125) NOT NULL,
- child_table_prefix VARCHAR(125) NOT NULL,
- weight INT NOT NULL DEFAULT 1,
-
- PRIMARY KEY primary_key_family (parent_hostname, parent_table_prefix, child_hostname, child_table_prefix),
- INDEX child (child_hostname, child_table_prefix)
-) ENGINE=INNODB;
-
-CREATE TABLE IF NOT EXISTS forwardings (
- base_source_id BIGINT NOT NULL,
- table_id INT NOT NULL,
- shard_hostname VARCHAR(125) NOT NULL,
- shard_table_prefix VARCHAR(125) NOT NULL,
-
- PRIMARY KEY (base_source_id, table_id),
-
- UNIQUE unique_shard (shard_hostname, shard_table_prefix)
-) ENGINE=INNODB;
View
52 src/test/bash/test.sh
@@ -1,52 +0,0 @@
-#!/bin/bash
-cd `dirname $0`
-function g {
- # echo "> g $@" >&2
- ../../../script/gizzmo -Cconfig.yaml "$@" 2>&1
-}
-function expect {
- diff -u - "expected/$1" && echo " success." || echo " failed." && exit 1
-}
-
-# set -ex
-
-if ["$FLOCK_ENV" -eq ""]; then
- FLOCK_ENV=development
-fi
-
-for i in 1 2
-do
- for type in edges groups
- do
- db="flock_${type}_${FLOCK_ENV}_${i}"
- echo "drop database if exists $db; create database $db; " | mysql -u"$DB_USERNAME" --password="$DB_PASSWORD"
- cat recreate.sql | mysql -u"$DB_USERNAME" --password="$DB_PASSWORD" "$db"
- done
-done
-
-for i in {0..9}
-do
- g create localhost "table_repl_$i" com.twitter.service.flock.edges.ReplicatingShard
- g create localhost "table_a_$i" com.twitter.service.flock.edges.SqlShard --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
- g create localhost "table_b_$i" com.twitter.service.flock.edges.SqlShard --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
- g link "localhost/table_repl_$i" "localhost/table_a_$i" 2
- g link "localhost/table_repl_$i" "localhost/table_b_$i" 1
-done
-
-for i in `g find -h localhost`; do g info $i; done | expect info.txt
-g find -hlocalhost | expect original-find.txt
-g find -hlocalhost -tSqlShard | expect find-only-sql-shard-type.txt
-
-g wrap com.twitter.service.flock.edges.ReplicatingShard localhost/table_b_0 | expect wrap-table_b_0.txt
-g links localhost/table_b_0 | expect links-for-table_b_0.txt
-g links localhost/table_repl_0 | expect links-for-table_repl_0.txt
-g links localhost/replicating_table_b_0 | expect links-for-replicating_table_b_0.txt
-
-g unwrap localhost/replicating_table_b_0 | expect unwrapped-replicating_table_b_0.txt
-g links localhost/table_b_0 | expect unwrapped-table_b_0.txt
-
-g unlink localhost/table_repl_0 localhost/table_b_0 | expect empty-file.txt
-g links localhost/table_b_0 | expect empty-file.txt
-
-
-

0 comments on commit 784487e

Please sign in to comment.
Something went wrong with that request. Please try again.