Permalink
Browse files

Merge pull request #115 from nearbuy/composite_columns

Initial support for composite column names
  • Loading branch information...
2 parents b703b50 + a7e1004 commit 4bca4c8f0dc7392306197b47ad5163fc16439540 @ryanking ryanking committed Mar 29, 2012
View
@@ -61,6 +61,9 @@
"SuperUUID":{
"subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
"comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
- "column_type":"Super"}
+ "column_type":"Super"},
+ "CompositeColumnConversion":{
+ "comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)",
+ "column_type":"Standard"}
}
}
View
@@ -48,4 +48,4 @@ create keyspace TypeConversions with
use TypeConversions;
create column family UUIDColumnConversion with comparator = TimeUUIDType;
create column family SuperUUID with comparator = TimeUUIDType and column_type = Super;
-
+create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)';
View
@@ -61,6 +61,9 @@
"SuperUUID":{
"subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
"comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType",
- "column_type":"Super"}
+ "column_type":"Super"},
+ "CompositeColumnConversion":{
+ "comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)",
+ "column_type":"Standard"}
}
}
View
@@ -48,4 +48,4 @@ create keyspace TypeConversions with
use TypeConversions;
create column family UUIDColumnConversion with comparator = TimeUUIDType;
create column family SuperUUID with comparator = TimeUUIDType and column_type = Super;
-
+create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)';
View
@@ -23,6 +23,7 @@ class Cassandra ; end
require 'cassandra/time'
require 'cassandra/comparable'
require 'cassandra/long'
+require 'cassandra/composite'
require 'cassandra/ordered_hash'
require 'cassandra/columns'
require 'cassandra/protocol'
View
@@ -18,10 +18,11 @@ def sub_column_name_class(column_family)
def column_name_class_for_key(column_family, comparator_key)
property = column_family_property(column_family, comparator_key)
- property =~ /.*\.(.*?)$/
+ property =~ /[^(]*\.(.*?)$/
case $1
when "LongType" then Long
when "LexicalUUIDType", "TimeUUIDType" then SimpleUUID::UUID
+ when /^CompositeType\(/ then Composite
else
String # UTF8, Ascii, Bytes, anything else
end
View
@@ -0,0 +1,118 @@
+
+class Cassandra
+ class Composite
+ include ::Comparable
+ attr_reader :parts
+ attr_reader :column_slice
+
+ def initialize(*parts)
+ options = {}
+ if parts.last.is_a?(Hash)
+ options = parts.pop
+ end
+ @column_slice = options[:slice]
+ raise ArgumentError if @column_slice != nil && ![:before, :after].include?(@column_slice)
+
+ if parts.length == 1 && parts[0].instance_of?(self.class)
+ @column_slice = parts[0].column_slice
+ @parts = parts[0].parts
+ elsif parts.length == 1 && parts[0].instance_of?(String) && @column_slice.nil? && valid_packed_composite?(parts[0])
+ unpack(parts[0])
+ else
+ @parts = parts
+ end
+ end
+
+ def [](*args)
+ return @parts[*args]
+ end
+
+ def pack
+ packed = @parts.map do |part|
+ [part.length].pack('n') + part + "\x00"
+ end
+ if @column_slice
+ part = @parts[-1]
+ packed[-1] = [part.length].pack('n') + part + slice_end_of_component
+ end
+ return packed.join('')
+ end
+
+ def to_s
+ return pack
+ end
+
+ def <=>(other)
+ if !other.instance_of?(self.class)
+ return @parts.first <=> other
+ end
+ eoc = slice_end_of_component.unpack('c')[0]
+ other_eoc = other.slice_end_of_component.unpack('c')[0]
+ @parts.zip(other.parts).each do |a, b|
+ next if a == b
+ if a.nil? && b.nil?
+ return eoc <=> other_eoc
+ end
+
+ if a.nil?
+ return @column_slice == :after ? 1 : -1
+ end
+ if b.nil?
+ return other.column_slice == :after ? -1 : 1
+ end
+ return -1 if a < b
+ return 1 if a > b
+ end
+ return 0
+ end
+
+ def inspect
+ return "#<Composite:#{@column_slice} #{@parts.inspect}>"
+ end
+
+ def slice_end_of_component
+ return "\x01" if @column_slice == :after
+ return "\xFF" if @column_slice == :before
+ return "\x00"
+ end
+
+ private
+ def unpack(packed_string)
+ parts = []
+ end_of_component = nil
+ while packed_string.length > 0
+ length = packed_string.slice(0, 2).unpack('n')[0]
+ parts << packed_string.slice(2, length)
+ end_of_component = packed_string.slice(2 + length, 1)
+
+ packed_string = packed_string.slice(3 + length, packed_string.length)
+ end
+ @column_slice = :after if end_of_component == "\x01"
+ @column_slice = :before if end_of_component == "\xFF"
+ @parts = parts
+ end
+
+ def valid_packed_composite?(packed_string)
+ while packed_string.length > 0
+ length = packed_string.slice(0, 2).unpack('n')[0]
+ return false if length.nil? || length + 3 > packed_string.length
+
+ end_of_component = packed_string.slice(2 + length, 1)
+ if length + 3 != packed_string.length
+ return false if end_of_component != "\x00"
+ end
+
+ packed_string = packed_string.slice(3 + length, packed_string.length)
+ end
+ return true
+ end
+
+ def hash
+ return to_s.hash
+ end
+
+ def eql?(other)
+ return to_s == other.to_s
+ end
+ end
+end
@@ -22,6 +22,12 @@ def setup
@uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
@longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
+ @composites = [
+ Cassandra::Composite.new([5].pack('N'), "zebra"),
+ Cassandra::Composite.new([5].pack('N'), "aardvark"),
+ Cassandra::Composite.new([1].pack('N'), "elephant"),
+ Cassandra::Composite.new([10].pack('N'), "kangaroo"),
+ ]
end
def test_setup
View
@@ -21,6 +21,12 @@ def setup
@uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) }
@longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) }
+ @composites = [
+ Cassandra::Composite.new([5].pack('N'), "zebra"),
+ Cassandra::Composite.new([5].pack('N'), "aardvark"),
+ Cassandra::Composite.new([1].pack('N'), "elephant"),
+ Cassandra::Composite.new([10].pack('N'), "kangaroo"),
+ ]
end
def test_inspect
@@ -43,7 +49,7 @@ def test_setting_default_consistency
end
def test_get_key
-
+
@twitter.insert(:Users, key, {'body' => 'v', 'user' => 'v'})
assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Users, key))
assert_equal(['body', 'user'].sort, @twitter.get(:Users, key).timestamps.keys.sort)
@@ -501,12 +507,12 @@ def test_count_keys
def test_count_columns
columns = (1..200).inject(Hash.new){|h,v| h['column' + v.to_s] = v.to_s; h;}
-
+
@twitter.insert(:Statuses, key, columns)
assert_equal 200, @twitter.count_columns(:Statuses, key, :count => 200)
- assert_equal 100, @twitter.count_columns(:Statuses, key)
+ assert_equal 100, @twitter.count_columns(:Statuses, key)
assert_equal 55, @twitter.count_columns(:Statuses, key, :count => 55)
-
+
end
def test_count_super_columns
@@ -556,34 +562,34 @@ def test_batch_mutate
assert_equal({}, @twitter.get(:Users, k + '2')) # Not yet written
assert_equal({}, @twitter.get(:Statuses, k + '3')) # Not yet written
- @twitter.remove(:Users, k + '1') # Full row
+ @twitter.remove(:Users, k + '1') # Full row
assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Not yet removed
@twitter.remove(:Users, k + '0', 'delete_me') # A single column of the row
assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Not yet removed
-
+
@twitter.remove(:Users, k + '4')
@twitter.insert(:Users, k + '4', {'body' => 'v4', 'user' => 'v4'})
assert_equal({}, @twitter.get(:Users, k + '4')) # Not yet written
# SuperColumns
# Add and delete new sub columns to the user timeline supercolumn
- @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns })
+ @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns })
@twitter.remove(:StatusRelationships, k, 'user_timelines' , subcolumn_to_delete ) # Delete the first of the initial_subcolumns from the user_timeline supercolumn
assert_equal(initial_subcolumns, @twitter.get(:StatusRelationships, k, 'user_timelines')) # No additions or deletes reflected yet
- # Delete a complete supercolumn
+ # Delete a complete supercolumn
@twitter.remove(:StatusRelationships, k, 'dummy_supercolumn' ) # Delete the full dummy supercolumn
- assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted
+ assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted
end
assert_equal({'body' => 'v2', 'user' => 'v2'}, @twitter.get(:Users, k + '2')) # Written
assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}, @twitter.get(:Users, k + '3')) # Written and compacted
assert_equal({'body' => 'v4', 'user' => 'v4'}, @twitter.get(:Users, k + '4')) # Written
assert_equal({'body' => 'v'}, @twitter.get(:Statuses, k + '3')) # Written
assert_equal({}, @twitter.get(:Users, k + '1')) # Removed
-
+
assert_equal({ 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # 'delete_me' column removed
-
+
assert_equal({'body' => 'v2', 'user' => 'v2'}.keys.sort, @twitter.get(:Users, k + '2').timestamps.keys.sort) # Written
assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}.keys.sort, @twitter.get(:Users, k + '3').timestamps.keys.sort) # Written and compacted
@@ -593,7 +599,7 @@ def test_batch_mutate
# Final result: initial_subcolumns - initial_subcolumns.first + new_subcolumns
resulting_subcolumns = initial_subcolumns.merge(new_subcolumns).reject{|k2,v| k2 == subcolumn_to_delete }
assert_equal(resulting_subcolumns, @twitter.get(:StatusRelationships, key, 'user_timelines'))
- assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted
+ assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted
end
@@ -834,8 +840,47 @@ def test_adding_getting_value_in_multiple_counters_with_super_columns
assert_equal(1, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'today'))
assert_equal(2, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'tomorrow'))
end
+
+ def test_composite_column_type_conversion
+ columns = {}
+ @composites.each_with_index do |c, index|
+ columns[c] = "value-#{index}"
+ end
+ @type_conversions.insert(:CompositeColumnConversion, key, columns)
+ columns_in_order = [
+ Cassandra::Composite.new([1].pack('N'), "elephant"),
+ Cassandra::Composite.new([5].pack('N'), "aardvark"),
+ Cassandra::Composite.new([5].pack('N'), "zebra"),
+ Cassandra::Composite.new([10].pack('N'), "kangaroo"),
+ ]
+ assert_equal(columns_in_order, @type_conversions.get(:CompositeColumnConversion, key).keys)
+
+ column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+ :start => Cassandra::Composite.new([1].pack('N')),
+ :finish => Cassandra::Composite.new([10].pack('N')),
+ ).keys
+ assert_equal(columns_in_order[0..-2], column_slice)
+
+ column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+ :start => Cassandra::Composite.new([5].pack('N')),
+ :finish => Cassandra::Composite.new([5].pack('N'), :slice => :after),
+ ).keys
+ assert_equal(columns_in_order[1..2], column_slice)
+
+ column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+ :start => Cassandra::Composite.new([5].pack('N'), :slice => :after).to_s,
+ ).keys
+ assert_equal([columns_in_order[-1]], column_slice)
+
+ column_slice = @type_conversions.get(:CompositeColumnConversion, key,
+ :finish => Cassandra::Composite.new([10].pack('N'), :slice => :before).to_s,
+ ).keys
+ assert_equal(columns_in_order[0..-2], column_slice)
+
+ assert_equal('value-2', @type_conversions.get(:CompositeColumnConversion, key, columns_in_order.first))
+ end
end
-
+
def test_column_timestamps
base_time = Time.now
@twitter.insert(:Statuses, "time-key", { "body" => "value" })
@@ -0,0 +1,29 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+class CompositeTypesTest < Test::Unit::TestCase
+ include Cassandra::Constants
+
+ def setup
+ @col_parts = [[363].pack('N'), 'extradites-mulling', SimpleUUID::UUID.new().bytes]
+ @col = Cassandra::Composite.new(*@col_parts)
+ end
+
+ def test_creation_from_parts
+ assert_equal(@col_parts[0], @col[0])
+ assert_equal(@col_parts[1], @col[1])
+ assert_equal(@col_parts[2], @col[2])
+ end
+
+ def test_packing_and_unpacking
+ part0_length = 2 + 4 + 1 # size + int + end_term
+ part1_length = 2 + @col_parts[1].length + 1 # size + string_len + end_term
+ part2_length = 2 + @col_parts[2].length + 1 # size + uuid_bytes + end_term
+ assert_equal(part0_length + part1_length + part2_length, @col.pack.length)
+
+ col2 = Cassandra::Composite.new(@col.pack)
+ assert_equal(@col_parts[0], col2[0])
+ assert_equal(@col_parts[1], col2[1])
+ assert_equal(@col_parts[2], col2[2])
+ assert_equal(@col, col2)
+ end
+end

0 comments on commit 4bca4c8

Please sign in to comment.