diff --git a/README b/README index c48e270..dd4e864 100644 --- a/README +++ b/README @@ -3,17 +3,28 @@ This has been moved to a new location: http://github.com/devver/dm-adapter-simpledb/tree/master -This is the new version of the adapter now based on Right_Aws which has helped a lot with performance. The rest of this FAQ is a bit dated... +This is the new version of the adapter now based on Right_Aws which has helped a +lot with performance. The rest of this FAQ is a bit dated... Written by Jeremy Boles -Contributers Edward Ocampo-Gooding (edward), Dan Mayer (danmayer), Thomas Olausson (latompa) + +Contributers: + Edward Ocampo-Gooding (edward) + Dan Mayer (danmayer) + Thomas Olausson (latompa) + Avdi Grimm (avdi) A DataMapper adapter for SimpleDB. -This version combines elements of Jeremy's orginal project, adding in code from Edward Ocampo-Gooding. It also includes various bug fixes and updates from Dan Mayer. +This version combines elements of Jeremy's orginal project, adding in code from +Edward Ocampo-Gooding. It also includes various bug fixes and updates from Dan +Mayer. Avdi Grimm updated the code to work with DataMapper 0.10.*, along with +misc. other improvements. -Tested using Matthew Painter’s SimpleDB/dev http://code.google.com/p/simpledb-dev/ -Additional SimpleDB/dev setup notes found here: http://pandastream.tumblr.com/post/52779609/playing-with-panda-without-simpledb-account +Tested using Matthew Painter’s SimpleDB/dev +http://code.google.com/p/simpledb-dev/ Additional SimpleDB/dev setup notes found +here: +http://pandastream.tumblr.com/post/52779609/playing-with-panda-without-simpledb-account == Current state @@ -35,7 +46,7 @@ Additional SimpleDB/dev setup notes found here: http://pandastream.tumblr.com/po require 'rubygems' require 'dm-core' - DataMapper.setup(:default, 'simpledb://sdb.amazon.com/sweetapp_development', :access_key_id => 'a valid access key id', :secret_access_key => 'a valid secret access key id') + DataMapper.setup(:default, 'simpledb://ACCESS_KEY:SECRET_KEY@sdb.amazon.com/DOMAIN') [Same as the following, but skip the database.yml] @@ -47,13 +58,11 @@ Additional SimpleDB/dev setup notes found here: http://pandastream.tumblr.com/po adapter: simpledb database: 'default' - access_key_id: (a 20-character, alphanumeric sequence) - secret_access_key: (a 40-character sequence) - domain: 'sweetapp_development' + access_key: (a 20-character, alphanumeric sequence) + secret_key: (a 40-character sequence) + domain: 'my_amazon_sdb_domain' base_url: 'http://sdb.amazon.com' - Alternatively, - Create a model class Tree @@ -78,13 +87,27 @@ Additional SimpleDB/dev setup notes found here: http://pandastream.tumblr.com/po yanked_tree = Tree.remote(:name => "Acer rubrum") == Running the tests - add these two lines to your .bash_profile as the spec_helper relies on them - export AMAZON_ACCESS_KEY_ID='YOUR_ACCESS_KEY' - export AMAZON_SECRET_ACCESS_KEY='YOUR_SECRET_ACCESS_KEY' + Add these two lines to your .bash_profile as the spec_helper relies on them + + $ export AMAZON_ACCESS_KEY_ID='YOUR_ACCESS_KEY' + $ export AMAZON_SECRET_ACCESS_KEY='YOUR_SECRET_ACCESS_KEY' + + Configure the domain to use for integration tests. THIS DOMAIN WILL BE + DELETED AND RECREATED BY THE TESTS, so do not choose a domain which contains + data you care about. Configure the domain by creating a file named + THROW_AWAY_SDB_DOMAIN in the projet root: + + $ echo dm_simpledb_adapter_test > THROW_AWAY_SDB_DOMAIN + + Run the tests: - Create the test domain on SimpleDB, :domain => 'missionaries' as found in spec_helper. This can be done easiest via IRB or I went to a project that had a database.yml and called db:automigrate because of the added migration support to this adaprter. + rake spec - rake spec + NOTE: While every attempt has been made to make the tests robust, Amazon + SimpleDB is by it's nature an unreliable service. Sometimes it can take a + very long time for updates to be reflected by queries, and sometimes calls + just time out. If the tests fail, try them again a few times before reporting + it as a bug. Also try running the spec files individually. == Bibliography Relating to Amazon SimpleDB http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1292&ref=featured diff --git a/Rakefile b/Rakefile index f567657..6769e47 100644 --- a/Rakefile +++ b/Rakefile @@ -54,7 +54,7 @@ def setup_dm(access_key, secret_key) :adapter => 'simpledb', :access_key => access_key, :secret_key => secret_key, - :domain => 'missionaries' + :domain => 'dm_simpledb_adapter_test' }) end diff --git a/lib/simpledb_adapter.rb b/lib/simpledb_adapter.rb index a1b2742..bd59833 100644 --- a/lib/simpledb_adapter.rb +++ b/lib/simpledb_adapter.rb @@ -2,99 +2,172 @@ require 'dm-core' require 'digest/sha1' require 'dm-aggregates' -require 'right_aws' +require 'right_aws' +require 'uuidtools' +require File.expand_path('simpledb_adapter/sdb_array', File.dirname(__FILE__)) module DataMapper + + module Migrations + #integrated from http://github.com/edward/dm-simpledb/tree/master + module SimpledbAdapter + + module ClassMethods + + end + + def self.included(other) + other.extend ClassMethods + + DataMapper.extend(::DataMapper::Migrations::SingletonMethods) + + [ :Repository, :Model ].each do |name| + ::DataMapper.const_get(name).send(:include, Migrations.const_get(name)) + end + end + + # Returns whether the storage_name exists. + # @param storage_name a String defining the name of a domain + # @return true if the storage exists + def storage_exists?(storage_name) + domains = sdb.list_domains[:domains] + domains.detect {|d| d == storage_name }!=nil + end + + def create_model_storage(model) + sdb.create_domain(@sdb_options[:domain]) + end + + #On SimpleDB you probably don't want to destroy the whole domain + #if you are just adding fields it is automatically supported + #default to non destructive migrate, to destroy run + #rake db:automigrate destroy=true + def destroy_model_storage(model) + if ENV['destroy']!=nil && ENV['destroy']=='true' + sdb.delete_domain(@sdb_options[:domain]) + end + end + + end # module Migration + end # module Migration + module Adapters class SimpleDBAdapter < AbstractAdapter - #right_aws calls Array(your_string) on all string properties, which splits on \n and then sorts your string in a random order - #This is a value that can be used to replace \n before storing a string and get it back coming out of SDB - NEWLINE_REPLACE = "[[[NEWLINE]]]" + attr_reader :sdb_options + + # For testing purposes ONLY. Seriously, don't enable this for production + # code. + attr_accessor :consistency_policy - def initialize(name, opts = {}) - super - @opts = opts + def initialize(name, normalised_options) + super + @sdb_options = {} + @sdb_options[:access_key] = options.fetch(:access_key) { + options[:user] + } + @sdb_options[:secret_key] = options.fetch(:secret_key) { + options[:password] + } + @sdb_options[:logger] = options.fetch(:logger) { DataMapper.logger } + @sdb_options[:server] = options.fetch(:host) { 'sdb.amazonaws.com' } + @sdb_options[:port] = options[:port] || 443 # port may be set but nil + @sdb_options[:domain] = options.fetch(:domain) { + options[:path].to_s.gsub(%r{(^/+)|(/+$)},"") # remove slashes + } + @consistency_policy = + normalised_options.fetch(:wait_for_consistency) { false } end def create(resources) created = 0 time = Benchmark.realtime do resources.each do |resource| + uuid = UUIDTools::UUID.timestamp_create + initialize_serial(resource, uuid.to_i) item_name = item_name_for_resource(resource) sdb_type = simpledb_type(resource.model) attributes = resource.attributes.merge(:simpledb_type => sdb_type) attributes = adjust_to_sdb_attributes(attributes) - #attributes.reject!{|key,value| value.nil? || value == '' || value == []} + attributes.reject!{|name, value| value.nil?} sdb.put_attributes(domain, item_name, attributes) created += 1 end end DataMapper.logger.debug(format_log_entry("(#{created}) INSERT #{resources.inspect}", time)) + modified! created end - def delete(query) + def delete(collection) deleted = 0 time = Benchmark.realtime do - item_name = item_name_for_query(query) - sdb.delete_attributes(domain, item_name) - deleted += 1 - raise NotImplementedError.new('Only :eql on delete at the moment') if not_eql_query?(query) - end; DataMapper.logger.debug(format_log_entry("(#{deleted}) DELETE #{query.conditions.inspect}", time)) + collection.each do |resource| + item_name = item_name_for_resource(resource) + sdb.delete_attributes(domain, item_name) + deleted += 1 + end + raise NotImplementedError.new('Only :eql on delete at the moment') if not_eql_query?(collection.query) + end; DataMapper.logger.debug(format_log_entry("(#{deleted}) DELETE #{collection.query.conditions.inspect}", time)) + modified! deleted end - def read_many(query) + def read(query) + maybe_wait_for_consistency sdb_type = simpledb_type(query.model) - conditions, order = set_conditions_and_sort_order(query, sdb_type) + conditions, order, unsupported_conditions = + set_conditions_and_sort_order(query, sdb_type) results = get_results(query, conditions, order) - - Collection.new(query) do |collection| - results.each do |result| - data = query.fields.map do |property| - value = result.values[0][property.field.to_s] - if value != nil - - value = chunks_to_string(value) if property.type==String && value.size > 1 - #replace the newline placeholder with newlines - if property.type==String - value = value.gsub(NEWLINE_REPLACE,"\n") if value.is_a?(String) - value[0] = value[0].gsub(NEWLINE_REPLACE,"\n") if value.is_a?(Array) && value[0]!=nil - end - if value.size > 1 - value.map {|v| property.typecast(v) } + proto_resources = results.map do |result| + name, attributes = *result.to_a.first + proto_resource = query.fields.inject({}) do |proto_resource, property| + value = attributes[property.field.to_s] + if value != nil + if value.size > 1 + if property.type == String + value = chunks_to_string(value) else - property.typecast(value[0]) + value = value.map {|v| property.typecast(v) } end else - property.typecast(nil) + value = property.typecast(value.first) end + else + value = property.typecast(nil) end - collection.load(data) + proto_resource[property.name.to_s] = value + proto_resource end + proto_resource end + query.conditions.operands.reject!{ |op| + !unsupported_conditions.include?(op) + } + records = query.filter_records(proto_resources) + + records end - def read_one(query) - #already has limit defined as 1 return first/only result from collection - results = read_many(query) - results.inspect #force the lazy loading to actually load - results[0] - end - - def update(attributes, query) + def update(attributes, collection) updated = 0 + attrs_to_update, attrs_to_delete = prepare_attributes(attributes) time = Benchmark.realtime do - item_name = item_name_for_query(query) - attributes = attributes.to_a.map {|a| [a.first.name.to_s, a.last]}.to_hash - attributes = adjust_to_sdb_attributes(attributes) - sdb.put_attributes(domain, item_name, attributes, true) - updated += 1 - raise NotImplementedError.new('Only :eql on delete at the moment') if not_eql_query?(query) + collection.each do |resource| + item_name = item_name_for_resource(resource) + unless attrs_to_update.empty? + sdb.put_attributes(domain, item_name, attrs_to_update, :replace) + end + unless attrs_to_delete.empty? + sdb.delete_attributes(domain, item_name, attrs_to_delete) + end + updated += 1 + end + raise NotImplementedError.new('Only :eql on delete at the moment') if not_eql_query?(collection.query) end - DataMapper.logger.debug(format_log_entry("UPDATE #{query.conditions.inspect} (#{updated} times)", time)) + DataMapper.logger.debug(format_log_entry("UPDATE #{collection.query.conditions.inspect} (#{updated} times)", time)) + modified! updated end @@ -105,7 +178,7 @@ def query(query_call, query_limit = 999999999) def aggregate(query) raise ArgumentError.new("Only count is supported") unless (query.fields.first.operator == :count) sdb_type = simpledb_type(query.model) - conditions, order = set_conditions_and_sort_order(query, sdb_type) + conditions, order, unsupported_conditions = set_conditions_and_sort_order(query, sdb_type) query_call = "SELECT count(*) FROM #{domain} " query_call << "WHERE #{conditions.compact.join(' AND ')}" if conditions.length > 0 @@ -115,17 +188,30 @@ def aggregate(query) end; DataMapper.logger.debug(format_log_entry(query_call, time)) [results[:items][0].values.first["Count"].first.to_i] end - + + # For testing purposes only. + def wait_for_consistency + return unless @current_consistency_token + token = :none + begin + results = sdb.get_attributes(domain, '__dm_consistency_token', '__dm_consistency_token') + tokens = results[:attributes]['__dm_consistency_token'] + end until tokens.include?(@current_consistency_token) + end + private - #hack for converting and storing strings longer than 1024 - #one thing to note if you use string longer than 1019 chars you will loose the ability to do full text matching on queries - #as the string can be broken at any place during chunking + # hack for converting and storing strings longer than 1024 one thing to + # note if you use string longer than 1019 chars you will loose the ability + # to do full text matching on queries as the string can be broken at any + # place during chunking def adjust_to_sdb_attributes(attrs) attrs.each_pair do |key, value| - if value.is_a?(String) - value = value.gsub("\n",NEWLINE_REPLACE) - attrs[key] = value + if value.kind_of?(String) + # Strings need to be inside arrays in order to prevent RightAws from + # inadvertantly splitting them on newlines when it calls + # Array(value). + attrs[key] = [value] end if value.is_a?(String) && value.length > 1019 chunked = string_to_chunks(value) @@ -153,6 +239,7 @@ def chunks_to_string(value) end chunks.replace chunks.sort_by{|index, text| index} string_result = chunks.map!{|index, text| text}.join + string_result rescue ArgumentError, TypeError #return original value, they could have put strings in the system not using the adapter or previous versions #that are larger than chunk size, but less than 1024 @@ -162,11 +249,12 @@ def chunks_to_string(value) # Returns the domain for the model def domain - @uri[:domain] + @sdb_options[:domain] end #sets the conditions and order for the SDB query def set_conditions_and_sort_order(query, sdb_type) + unsupported_conditions = [] conditions = ["simpledb_type = '#{sdb_type}'"] # look for query.order.first and insure in conditions # raise if order if greater than 1 @@ -174,43 +262,59 @@ def set_conditions_and_sort_order(query, sdb_type) if query.order && query.order.length > 0 query_object = query.order[0] #anything sorted on must be a condition for SDB - conditions << "#{query_object.property.name} IS NOT NULL" - order = "ORDER BY #{query_object.property.name} #{query_object.direction}" + conditions << "#{query_object.target.name} IS NOT NULL" + order = "ORDER BY #{query_object.target.name} #{query_object.operator}" else order = "" end - - query.conditions.each do |operator, attribute, value| - operator = case operator - when :eql - if value.nil? - conditions << "#{attribute.name} IS NULL" - next - else - '=' - end - when :not - if value.nil? - conditions << "#{attribute.name} IS NOT NULL" - next - else - '!=' - end - when :gt then '>' - when :gte then '>=' - when :lt then '<' - when :lte then '<=' - when :like then 'like' - when :in - values = value.collect{|v| "'#{v}'"}.join(',') - values = "'__NULL__'" if values.empty? - conditions << "#{attribute.name} IN (#{values})" - next - else raise "Invalid query operator: #{operator.inspect}" - end - conditions << "#{attribute.name} #{operator} '#{value}'" + query.conditions.each do |op| + case op.slug + when :regexp + unsupported_conditions << op + when :eql + conditions << if op.value.nil? + "#{op.subject.name} IS NULL" + else + "#{op.subject.name} = '#{op.value}'" + end + when :not then + comp = op.operands.first + if comp.slug == :like + conditions << "#{comp.subject.name} not like '#{comp.value}'" + next + end + case comp.value + when Range, Set, Array, Regexp + unsupported_conditions << op + when nil + conditions << "#{comp.subject.name} IS NOT NULL" + else + conditions << "#{comp.subject.name} != '#{comp.value}'" + end + when :gt then conditions << "#{op.subject.name} > '#{op.value}'" + when :gte then conditions << "#{op.subject.name} >= '#{op.value}'" + when :lt then conditions << "#{op.subject.name} < '#{op.value}'" + when :lte then conditions << "#{op.subject.name} <= '#{op.value}'" + when :like then conditions << "#{op.subject.name} like '#{op.value}'" + when :in + case op.value + when Array, Set + values = op.value.collect{|v| "'#{v}'"}.join(',') + values = "'__NULL__'" if values.empty? + conditions << "#{op.subject.name} IN (#{values})" + when Range + if op.value.exclude_end? + unsupported_conditions << op + else + conditions << "#{op.subject.name} between '#{op.value.first}' and '#{op.value.last}'" + end + else + raise ArgumentError, "Unsupported inclusion op: #{op.value.inspect}" + end + else raise "Invalid query op: #{op.inspect}" + end end - [conditions,order] + [conditions,order,unsupported_conditions] end def select(query_call, query_limit) @@ -229,7 +333,8 @@ def select(query_call, query_limit) #gets all results or proper number of results depending on the :limit def get_results(query, conditions, order) - query_call = "SELECT * FROM #{domain} " + output_list = query.fields.map{|f| f.field}.join(', ') + query_call = "SELECT #{output_list} FROM #{domain} " query_call << "WHERE #{conditions.compact.join(' AND ')}" if conditions.length > 0 query_call << " #{order}" if query.limit!=nil @@ -240,7 +345,7 @@ def get_results(query, conditions, order) query_limit = 999999999 #TODO hack for query.limit being nil #query_call << " limit 2500" #this doesn't work with continuation keys as it halts at the limit passed not just a limit per query. end - select(query_call, query_limit) + records = select(query_call, query_limit) end # Creates an item name for a query @@ -263,7 +368,7 @@ def item_name_for_resource(resource) item_name = "#{sdb_type}+" keys = keys_for_model(resource.model) item_name += keys.map do |property| - resource.instance_variable_get(property.instance_variable_name) + property.get(resource) end.join('-') Digest::SHA1.hexdigest(item_name) @@ -276,16 +381,16 @@ def keys_for_model(model) def not_eql_query?(query) # Curosity check to make sure we are only dealing with a delete - conditions = query.conditions.map {|c| c[0] }.uniq + conditions = query.conditions.map {|c| c.slug }.uniq selectors = [ :gt, :gte, :lt, :lte, :not, :like, :in ] return (selectors - conditions).size != selectors.size end # Returns an SimpleDB instance to work with def sdb - access_key = @uri[:access_key] - secret_key = @uri[:secret_key] - @sdb ||= RightAws::SdbInterface.new(access_key,secret_key,@opts) + access_key = @sdb_options[:access_key] + secret_key = @sdb_options[:secret_key] + @sdb ||= RightAws::SdbInterface.new(access_key,secret_key,@sdb_options) @sdb end @@ -298,47 +403,67 @@ def format_log_entry(query, ms = 0) 'SDB (%.1fs) %s' % [ms, query.squeeze(' ')] end - #integrated from http://github.com/edward/dm-simpledb/tree/master - module Migration - # Returns whether the storage_name exists. - # @param storage_name a String defining the name of a domain - # @return true if the storage exists - def storage_exists?(storage_name) - domains = sdb.list_domains[:domains] - domains.detect {|d| d == storage_name }!=nil - end - - def create_model_storage(repository, model) - sdb.create_domain(@uri[:domain]) - end - - #On SimpleDB you probably don't want to destroy the whole domain - #if you are just adding fields it is automatically supported - #default to non destructive migrate, to destroy run - #rake db:automigrate destroy=true - def destroy_model_storage(repository, model) - if ENV['destroy']!=nil && ENV['destroy']=='true' - sdb.delete_domain(@uri[:domain]) - end + def prepare_attributes(attributes) + attributes = attributes.to_a.map {|a| [a.first.name.to_s, a.last]}.to_hash + attributes = adjust_to_sdb_attributes(attributes) + updates, deletes = attributes.partition{|name,value| + !value.nil? && !(value.respond_to?(:to_ary) && value.to_ary.empty?) + } + attrs_to_update = Hash[updates] + attrs_to_delete = Hash[deletes].keys + [attrs_to_update, attrs_to_delete] + end + + def update_consistency_token + @current_consistency_token = UUIDTools::UUID.timestamp_create.to_s + sdb.put_attributes( + domain, + '__dm_consistency_token', + {'__dm_consistency_token' => [@current_consistency_token]}) + end + + def maybe_wait_for_consistency + if consistency_policy == :automatic && @current_consistency_token + wait_for_consistency end - - #TODO look at github panda simpleDB for serials support? - module SQL - def supports_serial? - false - end + end + + # SimpleDB supports "eventual consistency", which mean your data will be + # there... eventually. Obviously this can make tests a little flaky. One + # option is to just wait a fixed amount of time after every write, but + # this can quickly add up to a lot of waiting. The strategy implemented + # here is based on the theory that while consistency is only eventual, + # chances are writes will at least be linear. That is, once the results of + # write #2 show up we can probably assume that the results of write #1 are + # in as well. + # + # When a consistency policy is enabled, the adapter writes a new unique + # "consistency token" to the database after every write (i.e. every + # create, update, or delete). If the policy is :manual, it only writes the + # consistency token. If the policy is :automatic, writes will not return + # until the token has been successfully read back. + # + # When waiting for the consistency token to show up, we use progressively + # longer timeouts until finally giving up and raising an exception. + def modified! + case @consistency_policy + when :manual, :automatic then + update_consistency_token + when false then + # do nothing + else + raise "Invalid :wait_for_consistency option: #{@consistency_policy.inspect}" end - - include SQL - - end # module Migration - - include Migration - + end + end # class SimpleDBAdapter # Required naming scheme. SimpledbAdapter = SimpleDBAdapter - + + const_added(:SimpledbAdapter) + end # module Adapters + + end # module DataMapper diff --git a/lib/simpledb_adapter/sdb_array.rb b/lib/simpledb_adapter/sdb_array.rb index 24f38b2..88c3824 100644 --- a/lib/simpledb_adapter/sdb_array.rb +++ b/lib/simpledb_adapter/sdb_array.rb @@ -1,19 +1,52 @@ +require 'dm-types' + +# NOTE: Do not try to clear SdbArray properties by assigning nil. Instead, +# assign an empty array: +# +# resource.array_prop = [] +# resource.save +# +# The reason has to do with DataMapper's lazy-load handling - a lazy-loaded +# property has a value of nil until it is loaded. If you assign nil, DM thinks +# that module DataMapper module Types class SdbArray < DataMapper::Type - primitive Text + primitive ::Object + lazy true def self.load(value, property) value end def self.dump(value, property) - value + dumped = ::Object.new + # This is a little screwy. DataMapper has a fixed list of values it + # considers primitives, and it insists that the value that comes out of + # a type's .dump() method MUST match one of these types. For SimpleDB + # Array is effectively a primitive because of the way it stores values, + # but DM doesn't include Array in it's list of valid primtive types. So + # we need to return an object which IS considered a primitive - in this + # case a plain 'ole Ruby Object. In order to convey the actual array + # value to the backend, we tack on a #to_ary method which returns the + # array data. RightAws calls Array() on all values before writing them, + # which in turn calls #to_ary(), and winds up with the correct data. In + # effect we are sneaking the array data through DataMapper inside a + # singleton method. + singleton_class = (class << dumped; self; end) + singleton_class.send(:define_method, :to_ary) do + value + end + singleton_class.send(:define_method, :to_s) do + value.to_s + end + dumped end def self.typecast(value, property) value end + end end end diff --git a/spec/associations_spec.rb b/spec/associations_spec.rb index 6b7ebdf..6662950 100644 --- a/spec/associations_spec.rb +++ b/spec/associations_spec.rb @@ -1,6 +1,5 @@ require 'pathname' require Pathname(__FILE__).dirname.expand_path + 'spec_helper' -require 'ruby-debug' describe 'associations' do it 'should work with belongs_to associations' diff --git a/spec/compliance_spec.rb b/spec/compliance_spec.rb new file mode 100644 index 0000000..787af83 --- /dev/null +++ b/spec/compliance_spec.rb @@ -0,0 +1,18 @@ +require 'pathname' +require Pathname(__FILE__).dirname.expand_path + 'spec_helper' + +require 'dm-core/spec/adapter_shared_spec' + +describe DataMapper::Adapters::SimpleDBAdapter do + before :all do + @adapter = DataMapper::Repository.adapters[:default] + @old_consistency_policy = @adapter.consistency_policy + @adapter.consistency_policy = :automatic + end + + after :all do + @adapter.consistency_policy = @old_consistency_policy + end + + it_should_behave_like 'An Adapter' +end diff --git a/spec/date_spec.rb b/spec/date_spec.rb index c64b8a4..f890052 100644 --- a/spec/date_spec.rb +++ b/spec/date_spec.rb @@ -15,26 +15,25 @@ class Professor describe 'with multiple records saved' do before(:each) do + @adapter.wait_for_consistency @person_attrs = { :id => "person-#{Time.now.to_f.to_s}", :name => 'Jeremy Boles', :age => 25, :wealth => 25.00, :birthday => Date.today } @jeremy = Professor.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Jeremy Boles", :age => 25)) @danielle = Professor.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Danille Boles", :age => 26)) @keegan = Professor.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Keegan Jones", :age => 20)) - sleep(0.4) #or the get calls might not have these created yet + @adapter.wait_for_consistency end after(:each) do @jeremy.destroy @danielle.destroy @keegan.destroy - sleep(0.4) #or might not be destroyed by the next test end it 'should handle DateTime' do - pending 'Need to figure out how to coerce DateTime' - time = Time.now + time = DateTime.civil(1970,1,1) @jeremy.created_at = time @jeremy.save - sleep(0.2) + @adapter.wait_for_consistency person = Professor.get!(@jeremy.id, @jeremy.name) person.created_at.should == time end diff --git a/spec/limit_and_order_spec.rb b/spec/limit_and_order_spec.rb index 99b52f5..ef6031b 100644 --- a/spec/limit_and_order_spec.rb +++ b/spec/limit_and_order_spec.rb @@ -11,7 +11,6 @@ class Hero property :birthday, Date property :created_at, DateTime - belongs_to :company end describe 'with multiple records saved' do @@ -20,14 +19,13 @@ class Hero @jeremy = Hero.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Jeremy Boles", :age => 25)) @danielle = Hero.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Danille Boles", :age => 26)) @keegan = Hero.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Keegan Jones", :age => 20, :wealth => 15.00)) - sleep(0.4) #or the get calls might not have these created yet + @adapter.wait_for_consistency end after(:each) do @jeremy.destroy @danielle.destroy @keegan.destroy - sleep(0.4) #or might not be destroyed by the next test end it 'should handle limit one case' do @@ -45,12 +43,6 @@ class Hero persons.length.should ==3 end - #it would be really slow to create over 100 entires to test this until we have batch creation - it 'should handle limits over the default SDB 100 results limit' - - #it would be really slow to create over 100 entires to test this until we have batch creation - it 'should get all results over the default SDB 100 results limit' - it 'should handle ordering asc results with a limit' do persons = Hero.all(:order => [:age.asc], :limit => 2) persons.inspect #can't access via array until loaded? Weird @@ -99,4 +91,20 @@ class Hero persons[1].should == @jeremy end + context "with many entries" do + before :each do + resources = [] + 111.times do |i| + resources << Hero.new(:id => i, :name => "Hero#{i}") + end + DataMapper.repository(:default).create(resources) + @adapter.wait_for_consistency + end + + it "should support limits over 100" do + results = Hero.all(:limit => 110) + results.should have(110).entries + end + end + end diff --git a/spec/migrations_spec.rb b/spec/migrations_spec.rb index 8cb1fb1..efe88a7 100644 --- a/spec/migrations_spec.rb +++ b/spec/migrations_spec.rb @@ -1,6 +1,6 @@ require 'pathname' require Pathname(__FILE__).dirname.expand_path + 'spec_helper' -require 'ruby-debug' +require 'dm-migrations' describe 'support migrations' do @@ -15,13 +15,8 @@ class Person property :birthday, Date property :created_at, DateTime - belongs_to :company end - before do - @adapter = repository(:default).adapter - end - # test can't be run simultanious make it delete a throwawaable storage model # instead of the one used by all the tests # it "should destroy model storage" do @@ -33,9 +28,13 @@ class Person # @adapter.storage_exists?("missionaries").should == true # end + before :all do + @sdb.delete_domain(@domain) + end + it "should create model storage" do - Person.auto_migrate! - @adapter.storage_exists?("missionaries").should == true + DataMapper.auto_migrate! + @adapter.storage_exists?(@domain).should == true end end diff --git a/spec/multiple_records_spec.rb b/spec/multiple_records_spec.rb index 88598a0..85a418a 100644 --- a/spec/multiple_records_spec.rb +++ b/spec/multiple_records_spec.rb @@ -25,19 +25,18 @@ class Company end describe 'with multiple records saved' do - before(:each) do + before(:all) do @person_attrs = { :id => "person-#{Time.now.to_f.to_s}", :name => 'Jeremy Boles', :age => 25, :wealth => 25.00, :birthday => Date.today } @jeremy = Person.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Jeremy Boles", :age => 25)) @danielle = Person.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Danille Boles", :age => 26)) @keegan = Person.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Keegan Jones", :age => 20)) - sleep(0.4) #or the get calls might not have these created yet + @adapter.wait_for_consistency end - after(:each) do + after(:all) do @jeremy.destroy @danielle.destroy @keegan.destroy - sleep(0.4) #or might not be destroyed by the next test end it 'should get all records' do @@ -56,7 +55,7 @@ class Company it 'should get records by not matcher' do people = Person.all(:age.not => 25) - people.length.should == 2 + people.should have(2).entries end it 'should get record by not matcher' do @@ -100,13 +99,13 @@ class Company end it 'should get records by the IN matcher' do - people = Person.all(:id.in => [@jeremy.id, @danielle.id]) + people = Person.all(:id => [@jeremy.id, @danielle.id]) people.should include(@jeremy) people.should include(@danielle) people.should_not include(@keegan) end it "should get no records if IN array is empty" do - people = Person.all(:id.in => []) + people = Person.all(:id => []) people.should be_empty end end diff --git a/spec/nils_spec.rb b/spec/nils_spec.rb index b23ecdd..e1add7f 100644 --- a/spec/nils_spec.rb +++ b/spec/nils_spec.rb @@ -1,6 +1,5 @@ require 'pathname' require Pathname(__FILE__).dirname.expand_path + 'spec_helper' -require 'ruby-debug' class Enemy include DataMapper::Resource @@ -11,36 +10,36 @@ class Enemy property :wealth, Float property :birthday, Date property :created_at, DateTime - - belongs_to :network end describe 'with nils records saved and retreived' do - before(:each) do + before(:all) do @person_attrs = { :id => "person-#{Time.now.to_f.to_s}", :name => 'Jeremy Boles', :age => 25, :wealth => 25.00, :birthday => Date.today } @jeremy = Enemy.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Jeremy Boles", :age => 25)) - @danielle = Enemy.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => nil, :age => 26, :birthday => nil)) - sleep(0.4) #or the get calls might not have these created yet + @danielle = Enemy.create(@person_attrs.merge(:id => Time.now.to_f.to_s, :name => "Danielle", :age => nil, :birthday => nil)) + @adapter.wait_for_consistency end - after(:each) do + after(:all) do @jeremy.destroy @danielle.destroy - sleep(0.4) #or might not be destroyed by the next test + @adapter.wait_for_consistency end it 'should get all records' do Enemy.all.length.should == 2 end - it 'should get retrieve nil values' do - people = Enemy.all(:age => 26) + it 'should retrieve nil values' do + records = people = Enemy.all(:name => "Danielle") people.length.should == 1 - people[0].name.should == nil + people[0].age.should == nil people[0].birthday.should == nil end - #fails but might work if we switch to using the helena lib - it 'should find based on nil values' + it 'should find based on nil values' do + @people = Enemy.all(:age => nil) + @people.should include(@danielle) + end end diff --git a/spec/sdb_array_spec.rb b/spec/sdb_array_spec.rb index f0a1402..c1086b9 100644 --- a/spec/sdb_array_spec.rb +++ b/spec/sdb_array_spec.rb @@ -1,6 +1,7 @@ require 'pathname' require Pathname(__FILE__).dirname.expand_path + 'spec_helper' require Pathname(__FILE__).dirname.expand_path + '../lib/simpledb_adapter/sdb_array' +require 'spec/autorun' describe 'with multiple records saved' do @@ -14,14 +15,13 @@ class Hobbyist @jeremy = Hobbyist.create(:name => "Jeremy Boles", :hobbies => ["biking", "diving", "chess"]) @danielle = Hobbyist.create(:name => "Danille Boles", :hobbies => ["swimming", "diving"]) @keegan = Hobbyist.create(:name => "Keegan Jones", :hobbies => ["painting"]) - sleep(0.4) + @adapter.wait_for_consistency end after(:each) do @jeremy.destroy @danielle.destroy @keegan.destroy - sleep(0.4) end it 'should store hobbies as array' do @@ -33,16 +33,16 @@ class Hobbyist person = Hobbyist.first(:name => 'Jeremy Boles') person.hobbies = ["lego"] person.save - + @adapter.wait_for_consistency lego_person = Hobbyist.first(:name => 'Jeremy Boles') lego_person.hobbies.should == "lego" end it 'should allow deletion of array' do person = Hobbyist.first(:name => 'Jeremy Boles') - person.hobbies = nil + person.hobbies = [] person.save - + @adapter.wait_for_consistency lego_person = Hobbyist.first(:name => 'Jeremy Boles') lego_person.hobbies.should == nil end @@ -55,7 +55,7 @@ class Hobbyist end it 'should find all records with painting hobby' do - people = Hobbyist.all(:hobbies => 'painting') + people = Hobbyist.all(:hobbies => ['painting']) people.should_not include(@jeremy) people.should_not include(@danielle) people.should include(@keegan) @@ -68,4 +68,4 @@ class Hobbyist people.should include(@keegan) end -end \ No newline at end of file +end diff --git a/spec/simpledb_adapter_spec.rb b/spec/simpledb_adapter_spec.rb index b6a173e..5a0a7c1 100644 --- a/spec/simpledb_adapter_spec.rb +++ b/spec/simpledb_adapter_spec.rb @@ -131,7 +131,7 @@ class Network describe '#query' do before(:each) do - @domain = Friend.repository(:default).adapter.uri[:domain] + @domain = Friend.repository(:default).adapter.sdb_options[:domain] end it "should return an array of records" do records = Friend.repository(:default).adapter.query("SELECT age, wealth from #{@domain} where age = '25'") diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 53de40c..17cc58e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,29 +1,67 @@ require 'pathname' require Pathname(__FILE__).dirname.parent.expand_path + 'lib/simpledb_adapter' -require 'ruby-debug' require 'logger' require 'fileutils' +require 'spec' -access_key = ENV['AMAZON_ACCESS_KEY_ID'] -secret_key = ENV['AMAZON_SECRET_ACCESS_KEY'] +DOMAIN_FILE_MESSAGE = < THROW_AWAY_SDB_DOMAIN + +END + +Spec::Runner.configure do |config| + access_key = ENV['AMAZON_ACCESS_KEY_ID'] + secret_key = ENV['AMAZON_SECRET_ACCESS_KEY'] + domain_file = File.expand_path('../THROW_AWAY_SDB_DOMAIN', File.dirname(__FILE__)) + test_domain = if File.exist?(domain_file) + File.read(domain_file).strip + else + warn DOMAIN_FILE_MESSAGE + exit 1 + end + + #For those that don't like to mess up their ENV + if access_key==nil && secret_key==nil + lines = File.readlines(File.join(File.dirname(__FILE__),'..','aws_config')) + access_key = lines[0].strip + secret_key = lines[1].strip + end -FileUtils.mkdir_p('log') unless File.exists?('log') -log_file = "log/dm-sdb.log" -FileUtils.touch(log_file) -log = Logger.new(log_file) - -DataMapper.logger.set_log(log_file, :debug) -DataMapper.setup(:default, { - :adapter => 'simpledb', - :access_key => access_key, - :secret_key => secret_key, - :domain => 'missionaries', - :logger => log -}) + # Run just once + config.before :suite do + FileUtils.mkdir_p('log') unless File.exists?('log') + log_file = "log/dm-sdb.log" + FileUtils.touch(log_file) + log = Logger.new(log_file) + + $control_sdb ||= RightAws::SdbInterface.new( + access_key, secret_key, :domain => test_domain) + + DataMapper.logger.set_log(log_file, :debug) + DataMapper.setup(:default, { + :adapter => 'simpledb', + :access_key => access_key, + :secret_key => secret_key, + :domain => test_domain, + :logger => log, + :wait_for_consistency => :manual + }) + end + + # Run before each group + config.before :all do + @adapter = DataMapper::Repository.adapters[:default] + @sdb ||= $control_sdb + @sdb.delete_domain(test_domain) + @sdb.create_domain(test_domain) + @domain = test_domain + end +end