Permalink
Browse files

simple sharding support

  • Loading branch information...
1 parent 9526c32 commit a409239d1aaa2862803fefc5cf4a41e0ea337319 @rcarver committed Mar 17, 2012
Showing with 184 additions and 1 deletion.
  1. +80 −0 features/sharding.feature
  2. +1 −0 lib/orel.rb
  3. +1 −1 lib/orel/schema_generator.rb
  4. +102 −0 lib/orel/sharding.rb
View
@@ -0,0 +1,80 @@
+@shard @mysql
+Feature: Automatically shard data into multiple tables
+
+ Scenario: The table is created as a template
+ Given I have these class definitions:
+ """
+ class Count
+ extend Orel::Relation
+ extend Orel::Sharding
+ heading do
+ key { day / thing }
+ att :day, Orel::Domains::String
+ att :thing, Orel::Domains::String
+ att :count, Orel::Domains::Integer
+ end
+ shard_table_on(:day) do |day|
+ {
+ :append_table_name => day[0, 6]
+ }
+ end
+ end
+ """
+ When I use Orel to fill my database with tables
+ Then my database looks like:
+ """
+ CREATE TABLE `counts_template` (
+ `day` varchar(255) NOT NULL,
+ `thing` varchar(255) NOT NULL,
+ `count` int(11) NOT NULL,
+ UNIQUE KEY `c_d_t_139dbd63abf1bc7926aee62e8ac5e276` (`day`,`thing`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+ """
+ When I run some Orel code:
+ """
+ days = %w(20120101 20120102 20120201)
+ months = %w(201201 201202)
+ days.each do |day|
+ Count.table.insert(:day => day, :thing => "ideas", :count => 10)
+ end
+ months.each do |month|
+ shard = Count.shard(month)
+ puts [month, shard.row_count].join(", ")
+ end
+ rows = Count.table.query { |q, table|
+ q.project table[:day], table[:thing], table[:count]
+ q.where table[:day].in(days)
+ }
+ rows.each { |row|
+ puts [row[:day], row[:thing], row[:count]].join(", ")
+ }
+ """
+ Then the output should contain:
+ """
+ 201201, 2
+ 201202, 1
+ """
+ And my database looks like:
+ """
+ CREATE TABLE `counts_201201` (
+ `day` varchar(255) NOT NULL,
+ `thing` varchar(255) NOT NULL,
+ `count` int(11) NOT NULL,
+ UNIQUE KEY `c_d_t_139dbd63abf1bc7926aee62e8ac5e276` (`day`,`thing`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+ CREATE TABLE `counts_201202` (
+ `day` varchar(255) NOT NULL,
+ `thing` varchar(255) NOT NULL,
+ `count` int(11) NOT NULL,
+ UNIQUE KEY `c_d_t_139dbd63abf1bc7926aee62e8ac5e276` (`day`,`thing`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+ CREATE TABLE `counts_template` (
+ `day` varchar(255) NOT NULL,
+ `thing` varchar(255) NOT NULL,
+ `count` int(11) NOT NULL,
+ UNIQUE KEY `c_d_t_139dbd63abf1bc7926aee62e8ac5e276` (`day`,`thing`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+ """
+
View
@@ -32,6 +32,7 @@
require 'orel/schema_generator'
require 'orel/simple_associations'
require 'orel/query'
+require 'orel/sharding'
require 'orel/table'
require 'orel/validator'
@@ -8,7 +8,7 @@ module SchemaGenerator
#
# Returns an Array of Strings.
def self.class_creation_statements(classes)
- headings = classes.map { |klass| klass.relation_set.to_a }.flatten
+ headings = classes.map { |klass| headings = klass.relation_set.to_a }.flatten
creation_statements(headings)
end
View
@@ -0,0 +1,102 @@
+module Orel
+ module Sharding
+
+ class TableSuffix
+ def initialize(namer, suffix)
+ @namer = namer
+ @suffix = suffix
+ end
+ def table_name
+ [@namer.table_name, "_", @suffix].join.to_sym
+ end
+ def method_missing(message, *args, &block)
+ @namer.send(message, *args, &block)
+ end
+ end
+
+ class ShardedTable
+ def initialize(relation, attribute)
+ @relation = relation
+ @attribute = attribute
+ end
+ def insert(attributes)
+ value = attributes[@attribute]
+ table = @relation.shard(value)
+ table.insert(attributes)
+ end
+ def query(&block)
+ []
+ end
+ end
+
+ def shard_table_on(attribute, &block)
+ heading = get_heading
+ @shard_attribute = attribute
+ @shard_block = block
+ @shard_namer = heading.namer.clone
+ heading.namer = TableSuffix.new(heading.namer, "template")
+ end
+
+ def table(child_name=nil)
+ if child_name
+ super
+ else
+ ShardedTable.new(self, @shard_attribute)
+ end
+ end
+
+ def shard(value)
+ table = Table.new(shard_namer(value).table_name, get_heading, shard_connection(value))
+ # TODO: we need to create the shard based on the actual template table to
+ # account for indices and other table tweak that aren't captured by orel.
+ create_shard(get_heading.with_namer(shard_namer(value)))
+ #Sharding.create_table_from_template(heading.table_name, shard_heading.table_name)
+ table
+ end
+
+ #
+ # Internal
+ #
+
+ def shard_namer(value)
+ instructions = @shard_block.call(value)
+ case
+ when instructions[:append_table_name]
+ TableSuffix.new(@shard_namer, instructions[:append_table_name])
+ else
+ raise ArgumentError, "Unhandled shard instructions: #{instructions.inspect}"
+ end
+ end
+
+ def shard_connection(value)
+ connection
+ end
+
+ def shard_schema_connection(value)
+ connection
+ end
+
+ def create_shard(heading)
+ begin
+ Orel::SchemaGenerator.creation_statements([heading]).each { |statement|
+ shard_schema_connection(heading).execute(statement)
+ }
+ rescue ActiveRecord::StatementInvalid => e
+ raise unless e.message =~ /already exists/
+ end
+ end
+
+ #def self.create_table_from_template(template_name, table_name)
+ #access = MysqlInspector::AR::Access.new(schema_connection)
+ #tables = access.tables
+ #table = tables.find { |t| t.name.to_s == table_name.to_s }
+ #template = tables.find { |t| t.name.to_s == template_name.to_s }
+ #if !table
+ #table = MysqlInspector::Table.new(template.to_sql)
+ #table.name = table_name
+ #access.load(table.to_sql)
+ #end
+ #end
+
+ end
+end

0 comments on commit a409239

Please sign in to comment.