Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Round trip tests #14

Merged
merged 7 commits into from

2 participants

@psanford

Add round trip tests.

This specifically tests that casting numeric types works properly.
The setup is based on the round trip tests found in https://github.com/fauna/cassandra.

psanford added some commits
@psanford psanford Added setup for running round-trip tests.
This is borrowed from https://github.com/fauna/cassandra.
9a30a9d
@psanford psanford Added round trip tests for integer types.
Fix how we handle ints that are shorter than 4 bytes.
The fix breaks handling of negative values.
b1c9cf8
@psanford psanford Fixed negative value integer handling.
The int cast code is now very similar to the python library.
6d8e555
@psanford psanford Add round trip tests for long type. 99e03a3
@psanford psanford Added round trip test for counter column type. 5073774
@psanford psanford Moved roundtrip tests into one keyspace.
Removed the unused keyspaces from fauna.
2d0ff43
@psanford psanford Removed old int cast tests.
These tests didn't do what they claimed to and have been replaced
by tests in roundtrip_spec.
37e1f7f
@kreynolds kreynolds merged commit 257dcf6 into from
@kreynolds
Owner

Nice work

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Sep 11, 2011
  1. @psanford
  2. @psanford

    Added round trip tests for integer types.

    psanford authored
    Fix how we handle ints that are shorter than 4 bytes.
    The fix breaks handling of negative values.
  3. @psanford

    Fixed negative value integer handling.

    psanford authored
    The int cast code is now very similar to the python library.
Commits on Sep 12, 2011
  1. @psanford
  2. @psanford
  3. @psanford

    Moved roundtrip tests into one keyspace.

    psanford authored
    Removed the unused keyspaces from fauna.
  4. @psanford

    Removed old int cast tests.

    psanford authored
    These tests didn't do what they claimed to and have been replaced
    by tests in roundtrip_spec.
This page is out of date. Refresh to see the latest.
View
129 Rakefile
@@ -18,5 +18,134 @@ end
task :default => :spec
+CassandraBinaries = {
+ '0.8' => 'http://archive.apache.org/dist/cassandra/0.8.4/apache-cassandra-0.8.4-bin.tar.gz'
+}
+
+CASSANDRA_HOME = ENV['CASSANDRA_HOME'] || "#{ENV['HOME']}/cassandra"
+CASSANDRA_VERSION = ENV['CASSANDRA_VERSION'] || '0.8'
+CASSANDRA_PIDFILE = ENV['CASSANDRA_PIDFILE'] || "#{CASSANDRA_HOME}/cassandra.pid"
+
+def setup_cassandra_version(version = CASSANDRA_VERSION)
+ FileUtils.mkdir_p CASSANDRA_HOME
+
+ destination_directory = File.join(CASSANDRA_HOME, 'cassandra-' + CASSANDRA_VERSION)
+
+ unless File.exists?(File.join(destination_directory, 'bin','cassandra'))
+ download_source = CassandraBinaries[CASSANDRA_VERSION]
+ download_destination = File.join("/tmp", File.basename(download_source))
+ untar_directory = File.join(CASSANDRA_HOME, File.basename(download_source,'-bin.tar.gz'))
+
+ puts "downloading cassandra"
+ sh "curl -L -o #{download_destination} #{download_source}"
+
+ sh "tar xzf #{download_destination} -C #{CASSANDRA_HOME}"
+ sh "mv #{untar_directory} #{destination_directory}"
+ end
+end
+
+def setup_environment
+ env = ""
+ if !ENV["CASSANDRA_INCLUDE"]
+ env << "CASSANDRA_INCLUDE=#{File.expand_path(Dir.pwd)}/conf/#{CASSANDRA_VERSION}/cassandra.in.sh "
+ env << "CASSANDRA_HOME=#{CASSANDRA_HOME}/cassandra-#{CASSANDRA_VERSION} "
+ env << "CASSANDRA_CONF=#{File.expand_path(Dir.pwd)}/conf/#{CASSANDRA_VERSION}"
+ else
+ env << "CASSANDRA_INCLUDE=#{ENV['CASSANDRA_INCLUDE']} "
+ env << "CASSANDRA_HOME=#{ENV['CASSANDRA_HOME']} "
+ env << "CASSANDRA_CONF=#{ENV['CASSANDRA_CONF']}"
+ end
+
+ env
+end
+
+def running?(pid_file = nil)
+ pid_file ||= CASSANDRA_PIDFILE
+
+ if File.exists?(pid_file)
+ pid = File.new(pid_file).read.to_i
+ begin
+ Process.kill(0, pid)
+ return true
+ rescue
+ File.delete(pid_file)
+ end
+ end
+
+ false
+end
+
+namespace :cassandra do
+ desc "Start Cassandra"
+ task :start, [:daemonize] => :java do |t, args|
+ args.with_defaults(:daemonize => true)
+
+ setup_cassandra_version
+
+ env = setup_environment
+
+ Dir.chdir(File.join(CASSANDRA_HOME, "cassandra-#{CASSANDRA_VERSION}")) do
+ sh("env #{env} bin/cassandra #{'-f' unless args.daemonize} -p #{CASSANDRA_PIDFILE}")
+ end
+ end
+
+ desc "Stop Cassandra"
+ task :stop => :java do
+ setup_cassandra_version
+ env = setup_environment
+ sh("kill $(cat #{CASSANDRA_PIDFILE})")
+ end
+end
+
+desc "Start Cassandra"
+task :cassandra => :java do
+ begin
+ Rake::Task["cassandra:start"].invoke(false)
+ rescue RuntimeError => e
+ raise e unless e.message =~ /Command failed with status \(130\)/ # handle keyboard interupt errors
+ end
+end
+
+desc "Run the Cassandra CLI"
+task :cli do
+ Dir.chdir(File.join(CASSANDRA_HOME, "cassandra-#{CASSANDRA_VERSION}")) do
+ sh("bin/cassandra-cli -host localhost -port 9160")
+ end
+end
+
+desc "Check Java version"
+task :java do
+ unless `java -version 2>&1`.split("\n").first =~ /java version "1.6/ #"
+ puts "You need to configure your environment for Java 1.6."
+ puts "If you're on OS X, just export the following environment variables:"
+ puts ' JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home"'
+ puts ' PATH="/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home/bin:$PATH"'
+ exit(1)
+ end
+end
+
+namespace :data do
+ desc "Reset test data"
+ task :reset do
+ puts "Resetting test data"
+ sh("rm -rf #{File.join(CASSANDRA_HOME, "cassandra-#{CASSANDRA_VERSION}", 'data')}")
+ end
+
+ desc "Load test data structures."
+ task :load do
+ schema_path = "#{File.expand_path(Dir.pwd)}/conf/#{CASSANDRA_VERSION}/schema.txt"
+ puts "Loading test data structures."
+ Dir.chdir(File.join(CASSANDRA_HOME, "cassandra-#{CASSANDRA_VERSION}")) do
+ begin
+ sh("bin/cassandra-cli --host localhost --batch < #{schema_path}")
+ rescue
+ puts "Schema already loaded."
+ end
+ end
+ end
+end
+
+task :spec => 'data:load'
+
require 'yard'
YARD::Rake::YardocTask.new
View
41 conf/0.8/cassandra.in.sh
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_HOME" = "x" ]; then
+ CASSANDRA_HOME=`dirname $0`/..
+fi
+
+# The directory where Cassandra's configs live (required)
+if [ "x$CASSANDRA_CONF" = "x" ]; then
+ CASSANDRA_CONF=$CASSANDRA_HOME/conf
+fi
+
+# This can be the path to a jar file, or a directory containing the
+# compiled classes. NOTE: This isn't needed by the startup script,
+# it's just used here in constructing the classpath.
+cassandra_bin=$CASSANDRA_HOME/build/classes/main
+cassandra_bin=$cassandra_bin:$CASSANDRA_HOME/build/classes/thrift
+#cassandra_bin=$cassandra_home/build/cassandra.jar
+
+# JAVA_HOME can optionally be set here
+#JAVA_HOME=/usr/local/jdk6
+
+# The java classpath (required)
+CLASSPATH=$CASSANDRA_CONF:$cassandra_bin
+
+for jar in $CASSANDRA_HOME/lib/*.jar; do
+ CLASSPATH=$CLASSPATH:$jar
+done
View
61 conf/0.8/cassandra.yaml
@@ -0,0 +1,61 @@
+# Cassandra storage config YAML
+cluster_name: 'Test'
+initial_token:
+auto_bootstrap: false
+hinted_handoff_enabled: true
+max_hint_window_in_ms: 3600000 # one hour
+hinted_handoff_throttle_delay_in_ms: 50
+authenticator: org.apache.cassandra.auth.AllowAllAuthenticator
+authority: org.apache.cassandra.auth.AllowAllAuthority
+partitioner: org.apache.cassandra.dht.RandomPartitioner
+
+# directories where Cassandra should store data on disk.
+data_file_directories:
+ - data/data
+commitlog_directory: data/commitlog
+
+# saved caches
+saved_caches_directory: data/saved_caches
+
+commitlog_rotation_threshold_in_mb: 128
+commitlog_sync: periodic
+commitlog_sync_period_in_ms: 10000
+seed_provider:
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ - seeds: "127.0.0.1"
+flush_largest_memtables_at: 0.75
+reduce_cache_sizes_at: 0.85
+reduce_cache_capacity_to: 0.6
+concurrent_reads: 32
+concurrent_writes: 32
+memtable_flush_queue_size: 4
+sliced_buffer_size_in_kb: 64
+storage_port: 7000
+listen_address: localhost
+rpc_address: localhost
+rpc_port: 9160
+rpc_keepalive: true
+thrift_framed_transport_size_in_mb: 15
+thrift_max_message_length_in_mb: 16
+incremental_backups: false
+snapshot_before_compaction: false
+column_index_size_in_kb: 64
+in_memory_compaction_limit_in_mb: 64
+concurrent_compactors: 1
+compaction_throughput_mb_per_sec: 16
+compaction_preheat_key_cache: true
+rpc_timeout_in_ms: 10000
+endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
+dynamic_snitch: true
+dynamic_snitch_update_interval_in_ms: 100
+dynamic_snitch_reset_interval_in_ms: 600000
+dynamic_snitch_badness_threshold: 0.0
+request_scheduler: org.apache.cassandra.scheduler.NoScheduler
+index_interval: 128
+encryption_options:
+ internode_encryption: none
+ keystore: conf/.keystore
+ keystore_password: cassandra
+ truststore: conf/.truststore
+ truststore_password: cassandra
View
40 conf/0.8/log4j-server.properties
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# for production, you should probably set pattern to %c instead of %l.
+# (%l is slower.)
+
+# output messages into a rolling log file as well as stdout
+log4j.rootLogger=DEBUG,stdout,R
+
+# stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p %d{HH:mm:ss,SSS} %m%n
+
+# rolling log file
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.maxFileSize=20MB
+log4j.appender.R.maxBackupIndex=50
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%5p [%t] %d{ISO8601} %F (line %L) %m%n
+# Edit the next line to point to your logs directory
+log4j.appender.R.File=data/logs/system.log
+
+# Application logging options
+#log4j.logger.org.apache.cassandra=DEBUG
+#log4j.logger.org.apache.cassandra.db=DEBUG
+#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG
View
10 conf/0.8/schema.txt
@@ -0,0 +1,10 @@
+create keyspace TypeConversions with
+ placement_strategy = 'org.apache.cassandra.locator.LocalStrategy' AND
+ strategy_options = [{replication_factor:1}];
+use TypeConversions;
+create column family UUIDColumnConversion with comparator = TimeUUIDType;
+create column family SuperUUID with comparator = TimeUUIDType and column_type = Super;
+create column family IntegerConversion with comparator = 'IntegerType';
+create column family LongConversion with comparator = 'LongType';
+create column family CounterConversion with comparator = 'UTF8Type' and
+ default_validation_class = CounterColumnType;
View
27 lib/cassandra-cql/schema.rb
@@ -21,11 +21,11 @@ def method_missing(method, *args, &block)
def to_s
keyspace
end
-
+
def keyspace
name
end
-
+
def column_family_names
@column_families.keys
end
@@ -34,11 +34,11 @@ def column_family_names
class ColumnFamily
attr_reader :cf_def
-
+
def initialize(cf_def)
@cf_def = cf_def
end
-
+
def method_missing(method, *args, &block)
if @cf_def.respond_to?(method)
@cf_def.send(method)
@@ -49,7 +49,7 @@ def method_missing(method, *args, &block)
def columns
return @columns if @columns
-
+
@columns = Hash.new(default_validation_class)
@cf_def.column_metadata.each do |col|
@columns[col.name] = col.validation_class
@@ -58,7 +58,7 @@ def columns
@columns
end
-
+
def self.cast(value, type)
case type
when "org.apache.cassandra.db.marshal.TimeUUIDType"
@@ -66,12 +66,13 @@ def self.cast(value, type)
when "org.apache.cassandra.db.marshal.UUIDType"
UUID.new(value)
when "org.apache.cassandra.db.marshal.IntegerType"
- int = value.unpack('N')[0]
- if int & 2**31 == 2**31
- int - 2**32
- else
- int
+ int = 0
+ values = value.unpack('C*')
+ values.each {|v| int = int << 8; int += v; }
+ if value[0].ord & 128 != 0
+ int = int - (1 << value.length * 8)
end
+ int
when "org.apache.cassandra.db.marshal.LongType", "org.apache.cassandra.db.marshal.CounterColumnType"
ints = value.unpack("NN")
val = (ints[0] << 32) + ints[1]
@@ -102,9 +103,9 @@ def id
def standard?
type == 'Standard'
end
-
+
def super?
type == 'Super'
end
end
-end
+end
View
18 spec/column_family_spec.rb
@@ -27,7 +27,7 @@
}.to raise_error NoMethodError
end
end
-
+
context "with a standard column family" do
it "should be standard" do
standard_column_family.type.should eq("Standard")
@@ -55,14 +55,6 @@
ColumnFamily.cast(uuid.bytes, "org.apache.cassandra.db.marshal.UUIDType").should eq(uuid)
end
- it "should turn a packed integer into a Fixnum" do
- ColumnFamily.cast([0x7FFFFFFF].pack("N"), "org.apache.cassandra.db.marshal.IntegerType").should eq(0x7FFFFFFF)
- end
-
- it "should turn a packed negative integer into a negative Fixnum" do
- ColumnFamily.cast([-68047].pack("N"), "org.apache.cassandra.db.marshal.IntegerType").should eq(-68047)
- end
-
it "should turn a packed long into a number" do
number = 2**33
packed = [number >> 32, number].pack("N*")
@@ -90,19 +82,19 @@
obj.should_receive(:to_s)
ColumnFamily.cast(obj, "org.apache.cassandra.db.marshal.UTF8Type")
end
-
+
it "should return self with BytesType" do
obj = Object.new
ColumnFamily.cast(obj, "org.apache.cassandra.db.marshal.BytesType").object_id.should eq(obj.object_id)
end
end
-
+
context "validations classes" do
let(:column_family) { ColumnFamily.new(yaml_fixture(:standard_with_validations)) }
it "should have a hash of column_names and validations" do
column_family.columns.should be_kind_of(Hash)
end
-
+
it "should have a default validation class" do
column_family.columns.default.should eq(column_family.cf_def.default_validation_class)
end
@@ -112,4 +104,4 @@
column_family.columns[column_family.key_alias].should eq(column_family.key_validation_class)
end
end
-end
+end
View
101 spec/roundtrip_spec.rb
@@ -0,0 +1,101 @@
+require File.expand_path('spec_helper.rb', File.dirname(__FILE__))
+include CassandraCQL
+
+describe "RoundTrip tests" do
+ before(:all) do
+ def clear_keyspace!(connection)
+ connection.schema.column_family_names.each do |cf|
+ connection.execute("truncate #{cf}")
+ end
+ end
+
+ conn = ["127.0.0.1:9160"]
+ thrift_options = {:retries => 2, :timeout => 1}
+ @type_conversions = CassandraCQL::Database.new(conn, {:keyspace => 'TypeConversions'}, thrift_options)
+ clear_keyspace!(@type_conversions)
+ end
+
+ context "with comparator IntegerType" do
+ def create_and_fetch_integer_column(int_column_name)
+ cql = 'insert into IntegerConversion (KEY, ?) values (?, ?)'
+ row_key = 'opening-Melbourne'
+ column_value = 'silted-misunderstanding'
+ @type_conversions.execute(cql, int_column_name, row_key, column_value)
+ return @type_conversions.execute('select ? from IntegerConversion where KEY = ?', int_column_name, row_key).fetch
+ end
+
+ it "should properly convert integer values that fit into 1 byte" do
+ row = create_and_fetch_integer_column(1)
+ row.column_names.should eq([1])
+ end
+
+ it "should properly convert integer values that fit into 2 bytes" do
+ i = 2**8 + 80
+ row = create_and_fetch_integer_column(i)
+ row.column_names.should eq([i])
+ end
+
+ it "should properly convert integer values that fit into 3 bytes" do
+ i = 2**16 + 622
+ row = create_and_fetch_integer_column(i)
+ row.column_names.should eq([i])
+ end
+
+ it "should properly convert integer values that fit into 4 bytes" do
+ i = 2**24 + 45820
+ row = create_and_fetch_integer_column(i)
+ row.column_names.should eq([i])
+ end
+
+ it "should properly convert integer values that are negative" do
+ i = -20681
+ row = create_and_fetch_integer_column(i)
+ row.column_names.should eq([i])
+ end
+ end
+
+ context "with comparator LongType" do
+ def create_and_fetch_long_column(long_column_name)
+ cql = 'insert into LongConversion (KEY, ?) values (?, ?)'
+ row_key = 'rationalistic-hammock'
+ column_value = 'thyroid-hallucinogen'
+ @type_conversions.execute(cql, long_column_name, row_key, column_value)
+ return @type_conversions.execute('select ? from LongConversion where KEY = ?', long_column_name, row_key).fetch
+ end
+
+ it "should properly convert long values shorter than 4 bytes" do
+ i = 190
+ row = create_and_fetch_long_column(i)
+ row.column_names.should eq([i])
+ i = -i
+ row = create_and_fetch_long_column(i)
+ row.column_names.should eq([i])
+ end
+
+ it "should properly convert long values greater than 4 bytes" do
+ i = 2**32 + 618387
+ row = create_and_fetch_long_column(i)
+ row.column_names.should eq([i])
+ i = -i
+ row = create_and_fetch_long_column(i)
+ row.column_names.should eq([i])
+ end
+ end
+
+ context "with comparator CounterColumnType" do
+ it "should convert counters to long values" do
+ i = 2**32 + 280647
+ @type_conversions.execute("update CounterConversion set Montezuma = Montezuma + #{i} where KEY = 'Houston'")
+ row = @type_conversions.execute("select Montezuma from CounterConversion where KEY = 'Houston'").fetch
+ row.column_values.should eq([i])
+
+ @type_conversions.execute("update CounterConversion set Montezuma = Montezuma - #{i} where KEY = 'Houston'")
+ row = @type_conversions.execute("select Montezuma from CounterConversion where KEY = 'Houston'").fetch
+ row.column_values.should eq([0])
+
+ @type_conversions.execute("update CounterConversion set Montezuma = Montezuma - #{i} where KEY = 'Houston'")
+ row = @type_conversions.execute("select Montezuma from CounterConversion where KEY = 'Houston'").fetch
+ row.column_values.should eq([-i])
+ end
+ end
+end
Something went wrong with that request. Please try again.