diff --git a/README b/README index 9e2b83b..5d5978f 100644 --- a/README +++ b/README @@ -50,6 +50,15 @@ http://github.com/devver/dm-adapter-simpledb/ - Store floats using exponential notation * Option to store Date/Time/DateTime as ISO8601 * Full aggregate support (min/max/etc) + * Offset support + Note, from the SimpleDB documentation: + + "The next token returned by count(*) and select are interchangeable as long + as the where and order by clauses match. For example, if you want to return + the 200 items after the first 10,000 (similar to an offset), you can perform + a count with a limit clause of 10,000 and use the next token to return the + next 200 items with select." + * Option to use libxml if available * Parallelized queries for increased throughput * Support of normalized 1:1 table:domain schemes that works with associations diff --git a/Rakefile b/Rakefile index cc6ec38..2892642 100644 --- a/Rakefile +++ b/Rakefile @@ -4,33 +4,44 @@ require 'pathname' load 'tasks/devver.rake' ROOT = Pathname(__FILE__).dirname.expand_path -require ROOT + 'lib/simpledb_adapter' -task :default => [ :spec ] +task :default => [ 'spec:unit' ] -desc 'Run specifications' -Spec::Rake::SpecTask.new(:spec) do |t| - if File.exists?('spec/spec.opts') - t.spec_opts << '--options' << 'spec/spec.opts' +namespace :spec do + desc 'Run unit-level specifications' + Spec::Rake::SpecTask.new(:unit) do |t| + if File.exists?('spec/spec.opts') + t.spec_opts << '--options' << 'spec/spec.opts' + end + t.spec_files = Pathname.glob((ROOT + 'spec/unit/**/*_spec.rb').to_s) + + begin + t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true + t.rcov_opts << '--exclude' << 'spec' + t.rcov_opts << '--text-summary' + t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse' + rescue Exception + # rcov not installed + end end - t.spec_files = Pathname.glob((ROOT + 'spec/**/*_spec.rb').to_s) - - begin - t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true - t.rcov_opts << '--exclude' << 'spec' - t.rcov_opts << '--text-summary' - t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse' - rescue Exception - # rcov not installed - end -end -desc 'Run specifications without Rcov' -Spec::Rake::SpecTask.new(:spec_no_rcov) do |t| - if File.exists?('spec/spec.opts') - t.spec_opts << '--options' << 'spec/spec.opts' + desc 'Run integration-level specifications' + Spec::Rake::SpecTask.new(:integration) do |t| + if File.exists?('spec/spec.opts') + t.spec_opts << '--options' << 'spec/spec.opts' + end + t.spec_files = Pathname.glob((ROOT + 'spec/integration/**/*_spec.rb').to_s) + + begin + t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true + t.rcov_opts << '--exclude' << 'spec' + t.rcov_opts << '--text-summary' + t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse' + rescue Exception + # rcov not installed + end end - t.spec_files = Pathname.glob((ROOT + 'spec/**/*_spec.rb').to_s) + end begin @@ -68,6 +79,8 @@ END ] gem.add_dependency('dm-core', '~> 0.10.0') gem.add_dependency('dm-aggregates', '~> 0.10.0') + gem.add_dependency('dm-migrations', '~> 0.10.0') + gem.add_dependency('dm-types', '~> 0.10.0') gem.add_dependency('uuidtools', '~> 2.0') gem.add_dependency('right_aws', '~> 1.10') end diff --git a/lib/simpledb/chunked_string.rb b/lib/simpledb/chunked_string.rb new file mode 100644 index 0000000..1e9cae1 --- /dev/null +++ b/lib/simpledb/chunked_string.rb @@ -0,0 +1,54 @@ +module SimpleDB + class ChunkedString < String + MAX_CHUNK_SIZE = 1019 + + def self.valid?(values) + values.all?{|v| v =~ /^\d{4}:/} + end + + def initialize(string_or_array) + case string_or_array + when Array then super(chunks_to_string(string_or_array)) + else super(string_or_array) + end + end + + def to_ary + string_to_chunks(self) + end + + alias_method :to_a, :to_ary + + private + + def string_to_chunks(value) + return [value] if value.size <= 1019 + chunks = value.to_s.scan(%r/.{1,1019}/m) # 1024 - '1024:'.size + i = -1 + fmt = '%04d:' + chunks.map!{|chunk| [(fmt % (i += 1)), chunk].join} + raise ArgumentError, 'that is just too big yo!' if chunks.size >= 256 + chunks + end + + def chunks_to_string(value) + begin + chunks = + Array(value).flatten.map do |chunk| + index, text = chunk.split(%r/:/, 2) + [Float(index).to_i, text] + end + chunks.replace chunks.sort_by{|index, text| index} + string_result = chunks.map!{|index, text| text}.join + string_result + rescue ArgumentError, TypeError + #return original value, they could have put strings in the system not + #using the adapter or previous versions that are larger than chunk size, + #but less than 1024 + value + end + end + + + end +end diff --git a/lib/simpledb/rake.rb b/lib/simpledb/rake.rb new file mode 100644 index 0000000..671ada5 --- /dev/null +++ b/lib/simpledb/rake.rb @@ -0,0 +1,43 @@ +namespace :simpledb do + desc "Migrate records to be compatable with current DM/SimpleDB adapter" + task :migrate, :domain do |t, args| + raise "THIS IS A WORK IN PROGRESS AND WILL DESTROY YOUR DATA" + require 'progressbar' + require 'right_aws' + require 'simpledb/record' + + puts "Initializing connection..." + domain = args.domain + sdb = RightAws::SdbInterface.new + puts "Counting records..." + num_legacy_records = 0 + query = "select count(*) from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)" + next_token = nil + while(results = sdb.select(query, next_token)) do + next_token = results[:next_token] + count = results[:items].first["Domain"]["Count"].first.to_i + num_legacy_records += count + break if next_token.nil? + end + puts "Found #{num_legacy_records} to migrate" + + pbar = ProgressBar.new("migrate", num_legacy_records) + query = "select * from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)" + while(results = sdb.select(query, next_token)) do + next_token = results[:next_token] + items = results[:items] + items.each do |item| + legacy_record = SimpleDB::Record.from_simpledb_hash(item) + new_record = legacy_record.migrate + updates = new_record.writable_attributes + deletes = new_record.deletable_attributes + sdb.put_attributes(domain, new_record.item_name, updates) + sdb.delete_attributes(domain, new_record.item_name, deletes) + pbar.inc + end + break if next_token.nil? + end + pbar.finish + + end +end diff --git a/lib/simpledb/record.rb b/lib/simpledb/record.rb new file mode 100644 index 0000000..033b115 --- /dev/null +++ b/lib/simpledb/record.rb @@ -0,0 +1,318 @@ +require 'dm-core' +require 'simpledb/utils' +require 'simpledb/chunked_string' +require 'simpledb/table' + +# TODO +# * V1.1: Store type in __dm_metadata +# * V1.1: Store type as non-munged class name + +module SimpleDB + class Record + include Utils + + METADATA_KEY = "__dm_metadata" + STORAGE_NAME_KEY = "simpledb_type" + META_KEYS = [METADATA_KEY, STORAGE_NAME_KEY] + CURRENT_VERSION = "01.01.00" + + def self.from_simpledb_hash(hash) + data_version = data_version(simpledb_attributes(hash)) + versions.fetch(data_version) do + raise "Unknown data version for: #{hash.inspect}" + end.new(hash) + end + + def self.from_resource(resource) + versions.fetch(CURRENT_VERSION).new(resource) + end + + def self.register(klass, version) + versions[version] = klass + end + + def self.versions + @versions ||= {} + end + + def self.version(version=nil) + if version + Record.register(self, version) + @version = version + else + @version + end + end + + def self.data_version(simpledb_attributes) + simpledb_attributes.fetch(METADATA_KEY){[]}.grep(/v\d\d\.\d\d\.\d\d/) do + |version_stamp| + return version_stamp[1..-1] + end + return "00.00.00" + end + + def self.simpledb_attributes(hash) + hash.values.first + end + + attr_reader :simpledb_attributes + attr_reader :deletable_attributes + attr_reader :item_name + alias_method :writable_attributes, :simpledb_attributes + + def initialize(hash_or_resource) + case hash_or_resource + when DataMapper::Resource then + attrs_to_update, attrs_to_delete = extract_attributes(hash_or_resource) + @simpledb_attributes = attrs_to_update + @deletable_attributes = attrs_to_delete + @item_name = item_name_for_resource(hash_or_resource) + when Hash + hash = hash_or_resource + @item_name = hash.keys.first + @simpledb_attributes = hash.values.first + @deletable_attributes = [] + else + raise "Don't know how to initialize from #{hash_or_resource.inspect}" + end + end + + # Convert to a Hash suitable for initializing a Resource + # + # @param [PropertySet] fields + # The fields to extract + def to_resource_hash(fields) + result = transform_hash(fields) {|hash, property| + hash[property.name.to_s] = self[property.field, property] + } + result + end + + # Deprecated - we are moving the type information under the metadata key + def storage_name + simpledb_attributes[STORAGE_NAME_KEY].first + end + + def [](attribute, type) + values = Array(simpledb_attributes[attribute]) + coerce_to(values, type) + end + + def coerce_to(values, type_or_property) + case type_or_property + when DataMapper::Property + coerce_to_property(values, type_or_property) + when Class + coerce_to_type(values, type_or_property) + else raise "Should never get here" + end + end + + def coerce_to_property(value, property) + property.typecast(coerce_to_type(value, property.type)) + end + + def coerce_to_type(values, type) + case + when type <= String + case values.size + when 0 + nil + when 1 + values.first + else + ChunkedString.new(values) + end + when type <= Array, type <= DataMapper::Types::SdbArray + values + else + values.first + end + end + + def version + self.class.version || self.class.data_version(simpledb_attributes) + end + + def version_token + "v#{version}" + end + + # Returns the "Table" this record belongs to. SimpleDB has no concept of + # tables, but we fake it with metadata. + def table + Table.name_from_metadata(metadata) || + storage_name + end + + def metadata + simpledb_attributes[METADATA_KEY] + end + + # Returns a record of the current version + def migrate + new_record = Record.versions[CURRENT_VERSION].allocate + new_record.item_name = item_name + data = transform_hash(simpledb_attributes) { + |hash, key, values| + hash[key] = coerce_heuristically(values) + } + updates = {} + deletes = [] + data.each_pair do |key, values| + if Array(values).empty? + deletes << key + else + updates[key] = values + end + end + new_record.add_metadata_to!(updates, table) + new_record.simpledb_attributes = updates + new_record.deletable_attributes = deletes + new_record + end + + def add_metadata_to!(hash, table_name) + hash.merge!({ + STORAGE_NAME_KEY => [table_name], + METADATA_KEY => [version_token, Table.token_for(table_name)] + }) + end + + protected + + attr_writer :item_name + attr_writer :simpledb_attributes + attr_writer :deletable_attributes + + private + + def app_data + transform_hash(simpledb_attributes) {|h,k,v| + h[k] = v unless META_KEYS.include?(k) + } + end + + def extract_attributes(resource) + attributes = resource.attributes(:property) + attributes = attributes.to_a.map {|a| [a.first.name.to_s, a.last]}.to_hash + attributes = adjust_to_sdb_attributes(attributes) + updates, deletes = attributes.partition{|name,value| + !Array(value).empty? + } + attrs_to_update = updates.inject({}){|h, (k,v)| h[k] = v; h} + table = Table.new(resource.model) + if resource.new? + add_metadata_to!(attrs_to_update, table.simpledb_type) + end + attrs_to_delete = deletes.inject({}){|h, (k,v)| h[k] = v; h}.keys + [attrs_to_update, attrs_to_delete] + end + + # hack for converting and storing strings longer than 1024 one thing to + # note if you use string longer than 1019 chars you will loose the ability + # to do full text matching on queries as the string can be broken at any + # place during chunking + def adjust_to_sdb_attributes(attrs) + attrs = transform_hash(attrs) do |result, key, value| + if primitive_value_of(value.class) <= String + result[key] = ChunkedString.new(value).to_a + elsif value.class == Object # This is for SdbArray + result[key] = value.to_ary + elsif primitive_value_of(value.class) <= Array + result[key] = value + elsif value.nil? + result[key] = nil + else + result[key] = [value.to_s] + end + end + # Stringify keys + transform_hash(attrs) {|h, k, v| h[k.to_s] = v} + end + + def primitive_value_of(type) + if type < DataMapper::Type + type.primitive + else + type + end + end + + # Creates an item name for a resource + def item_name_for_resource(resource) + table = Table.new(resource.model) + sdb_type = table.simpledb_type + + item_name = "#{sdb_type}+" + keys = table.keys_for_model + item_name += keys.map do |property| + property.get(resource) + end.join('-') + + Digest::SHA1.hexdigest(item_name) + end + + def coerce_heuristically(values) + if values + case values.size + when 0 + values + when 1 + value = coerce_to_type(values, String) + value.nil? ? [] : [value] + else + if ChunkedString.valid?(values) + string = ChunkedString.new(values) + coerced_string = coerce_to_type([string], Array).first + ChunkedString.new(coerced_string).to_a + else + coerce_to_type(values, Array) + end + end + else + [] + end + end + + end + + # Version 0 records are records that have no associated version + # metadata. Any records created by versions of the DataMapper/SimplDB adapter + # prior to 1.1.0 are considered to be version 0. + # + # Version 0 records have a few distinguishing characteristics: + # * An attribute with the token "nil" as its sole member is treated as a + # null/empty attribute. + # * The token "[[[NEWLINE]]]" inside of String attributes is replaced with \n + class RecordV0 < Record + version "00.00.00" + + def coerce_to_type(values, type) + values = values.map{|v| replace_newline_placeholders(v)} + result = super(values, type) + + if result == "nil" + nil + elsif result == ["nil"] + [] + elsif result && type <= String + # TODO redundant + replace_newline_placeholders(result) + else + result + end + end + + private + + def replace_newline_placeholders(value) + value.gsub("[[[NEWLINE]]]", "\n") + end + end + + class RecordV1_1 < Record + version "01.01.00" + end +end diff --git a/lib/simpledb_adapter/sdb_array.rb b/lib/simpledb/sdb_array.rb similarity index 100% rename from lib/simpledb_adapter/sdb_array.rb rename to lib/simpledb/sdb_array.rb diff --git a/lib/simpledb/table.rb b/lib/simpledb/table.rb new file mode 100644 index 0000000..06b1b01 --- /dev/null +++ b/lib/simpledb/table.rb @@ -0,0 +1,40 @@ +module SimpleDB + class Table + + def self.name_from_metadata(metadata) + Array(metadata).grep(/^table:(.*)$/) do |match| + return $1 + end + nil + end + + def self.token_for(name) + "table:#{name}" + end + + attr_reader :model + + def initialize(model) + @model = model + end + + # Returns a string so we know what type of + def simpledb_type + model.storage_name(repository_name) + end + + def repository_name + # TODO this should probably take into account the adapter + model.repository.name + end + + # Returns the keys for model sorted in alphabetical order + def keys_for_model + model.key(repository_name).sort {|a,b| a.name.to_s <=> b.name.to_s } + end + + def token + self.class.token_for(simpledb_type) + end + end +end diff --git a/lib/simpledb/utils.rb b/lib/simpledb/utils.rb new file mode 100644 index 0000000..c771b2f --- /dev/null +++ b/lib/simpledb/utils.rb @@ -0,0 +1,15 @@ +module SimpleDB + module Utils + def transform_hash(original, options={}, &block) + original.inject({}){|result, (key,value)| + value = if (options[:deep] && Hash === value) + transform_hash(value, options, &block) + else + value + end + block.call(result,key,value) + result + } + end + end +end diff --git a/lib/simpledb_adapter.rb b/lib/simpledb_adapter.rb index bd59833..9bd74e6 100644 --- a/lib/simpledb_adapter.rb +++ b/lib/simpledb_adapter.rb @@ -1,10 +1,20 @@ -require 'rubygems' +gem 'dm-migrations', '~> 0.10.0' +gem 'dm-types', '~> 0.10.0' +gem 'dm-aggregates', '~> 0.10.0' +gem 'dm-core', '~> 0.10.0' + + require 'dm-core' -require 'digest/sha1' require 'dm-aggregates' +require 'digest/sha1' require 'right_aws' require 'uuidtools' -require File.expand_path('simpledb_adapter/sdb_array', File.dirname(__FILE__)) + +require 'simpledb/sdb_array' +require 'simpledb/utils' +require 'simpledb/record' +require 'simpledb/table' + module DataMapper @@ -53,6 +63,7 @@ def destroy_model_storage(model) module Adapters class SimpleDBAdapter < AbstractAdapter + include SimpleDB::Utils attr_reader :sdb_options @@ -75,8 +86,17 @@ def initialize(name, normalised_options) @sdb_options[:domain] = options.fetch(:domain) { options[:path].to_s.gsub(%r{(^/+)|(/+$)},"") # remove slashes } + # We do not expect to be saving any nils in future, because now we + # represent null values by removing the attributes. The representation + # here is chosen on the basis of it being unlikely to match any strings + # found in real-world records, as well as being eye-catching in case any + # nils DO manage to sneak in. It would be preferable if we could disable + # RightAWS's nil-token replacement altogether, but that does not appear + # to be an option. + @sdb_options[:nil_representation] = "<[<[]>]>" @consistency_policy = normalised_options.fetch(:wait_for_consistency) { false } + @sdb = options.fetch(:sdb_interface) { nil } end def create(resources) @@ -85,11 +105,10 @@ def create(resources) resources.each do |resource| uuid = UUIDTools::UUID.timestamp_create initialize_serial(resource, uuid.to_i) - item_name = item_name_for_resource(resource) - sdb_type = simpledb_type(resource.model) - attributes = resource.attributes.merge(:simpledb_type => sdb_type) - attributes = adjust_to_sdb_attributes(attributes) - attributes.reject!{|name, value| value.nil?} + + record = SimpleDB::Record.from_resource(resource) + attributes = record.writable_attributes + item_name = record.item_name sdb.put_attributes(domain, item_name, attributes) created += 1 end @@ -103,7 +122,8 @@ def delete(collection) deleted = 0 time = Benchmark.realtime do collection.each do |resource| - item_name = item_name_for_resource(resource) + record = SimpleDB::Record.from_resource(resource) + item_name = record.item_name sdb.delete_attributes(domain, item_name) deleted += 1 end @@ -115,33 +135,17 @@ def delete(collection) def read(query) maybe_wait_for_consistency - sdb_type = simpledb_type(query.model) - + table = SimpleDB::Table.new(query.model) conditions, order, unsupported_conditions = - set_conditions_and_sort_order(query, sdb_type) + set_conditions_and_sort_order(query, table.simpledb_type) results = get_results(query, conditions, order) - proto_resources = results.map do |result| - name, attributes = *result.to_a.first - proto_resource = query.fields.inject({}) do |proto_resource, property| - value = attributes[property.field.to_s] - if value != nil - if value.size > 1 - if property.type == String - value = chunks_to_string(value) - else - value = value.map {|v| property.typecast(v) } - end - else - value = property.typecast(value.first) - end - else - value = property.typecast(nil) - end - proto_resource[property.name.to_s] = value - proto_resource - end - proto_resource - end + records = results.map{|result| + SimpleDB::Record.from_simpledb_hash(result) + } + + proto_resources = records.map{|record| + record.to_resource_hash(query.fields) + } query.conditions.operands.reject!{ |op| !unsupported_conditions.include?(op) } @@ -152,10 +156,14 @@ def read(query) def update(attributes, collection) updated = 0 - attrs_to_update, attrs_to_delete = prepare_attributes(attributes) time = Benchmark.realtime do collection.each do |resource| - item_name = item_name_for_resource(resource) + updated_resource = resource.dup + updated_resource.attributes = attributes + record = SimpleDB::Record.from_resource(updated_resource) + attrs_to_update = record.writable_attributes + attrs_to_delete = record.deletable_attributes + item_name = record.item_name unless attrs_to_update.empty? sdb.put_attributes(domain, item_name, attrs_to_update, :replace) end @@ -177,7 +185,8 @@ def query(query_call, query_limit = 999999999) def aggregate(query) raise ArgumentError.new("Only count is supported") unless (query.fields.first.operator == :count) - sdb_type = simpledb_type(query.model) + table = SimpleDB::Table.new(query.model) + sdb_type = table.simpledb_type conditions, order, unsupported_conditions = set_conditions_and_sort_order(query, sdb_type) query_call = "SELECT count(*) FROM #{domain} " @@ -200,53 +209,6 @@ def wait_for_consistency end private - - # hack for converting and storing strings longer than 1024 one thing to - # note if you use string longer than 1019 chars you will loose the ability - # to do full text matching on queries as the string can be broken at any - # place during chunking - def adjust_to_sdb_attributes(attrs) - attrs.each_pair do |key, value| - if value.kind_of?(String) - # Strings need to be inside arrays in order to prevent RightAws from - # inadvertantly splitting them on newlines when it calls - # Array(value). - attrs[key] = [value] - end - if value.is_a?(String) && value.length > 1019 - chunked = string_to_chunks(value) - attrs[key] = chunked - end - end - attrs - end - - def string_to_chunks(value) - chunks = value.to_s.scan(%r/.{1,1019}/) # 1024 - '1024:'.size - i = -1 - fmt = '%04d:' - chunks.map!{|chunk| [(fmt % (i += 1)), chunk].join} - raise ArgumentError, 'that is just too big yo!' if chunks.size >= 256 - chunks - end - - def chunks_to_string(value) - begin - chunks = - Array(value).flatten.map do |chunk| - index, text = chunk.split(%r/:/, 2) - [Float(index).to_i, text] - end - chunks.replace chunks.sort_by{|index, text| index} - string_result = chunks.map!{|index, text| text}.join - string_result - rescue ArgumentError, TypeError - #return original value, they could have put strings in the system not using the adapter or previous versions - #that are larger than chunk size, but less than 1024 - value - end - end - # Returns the domain for the model def domain @sdb_options[:domain] @@ -333,7 +295,9 @@ def select(query_call, query_limit) #gets all results or proper number of results depending on the :limit def get_results(query, conditions, order) - output_list = query.fields.map{|f| f.field}.join(', ') + fields_to_request = query.fields.map{|f| f.field} + fields_to_request << SimpleDB::Record::METADATA_KEY + output_list = fields_to_request.join(', ') query_call = "SELECT #{output_list} FROM #{domain} " query_call << "WHERE #{conditions.compact.join(' AND ')}" if conditions.length > 0 query_call << " #{order}" @@ -361,24 +325,6 @@ def item_name_for_query(query) Digest::SHA1.hexdigest(item_name) end - # Creates an item name for a resource - def item_name_for_resource(resource) - sdb_type = simpledb_type(resource.model) - - item_name = "#{sdb_type}+" - keys = keys_for_model(resource.model) - item_name += keys.map do |property| - property.get(resource) - end.join('-') - - Digest::SHA1.hexdigest(item_name) - end - - # Returns the keys for model sorted in alphabetical order - def keys_for_model(model) - model.key(self.name).sort {|a,b| a.name.to_s <=> b.name.to_s } - end - def not_eql_query?(query) # Curosity check to make sure we are only dealing with a delete conditions = query.conditions.map {|c| c.slug }.uniq @@ -394,26 +340,10 @@ def sdb @sdb end - # Returns a string so we know what type of - def simpledb_type(model) - model.storage_name(model.repository.name) - end - def format_log_entry(query, ms = 0) 'SDB (%.1fs) %s' % [ms, query.squeeze(' ')] end - def prepare_attributes(attributes) - attributes = attributes.to_a.map {|a| [a.first.name.to_s, a.last]}.to_hash - attributes = adjust_to_sdb_attributes(attributes) - updates, deletes = attributes.partition{|name,value| - !value.nil? && !(value.respond_to?(:to_ary) && value.to_ary.empty?) - } - attrs_to_update = Hash[updates] - attrs_to_delete = Hash[deletes].keys - [attrs_to_update, attrs_to_delete] - end - def update_consistency_token @current_consistency_token = UUIDTools::UUID.timestamp_create.to_s sdb.put_attributes( @@ -457,6 +387,7 @@ def modified! end end # class SimpleDBAdapter + # Required naming scheme. SimpledbAdapter = SimpleDBAdapter diff --git a/spec/associations_spec.rb b/spec/integration/associations_spec.rb similarity index 100% rename from spec/associations_spec.rb rename to spec/integration/associations_spec.rb diff --git a/spec/compliance_spec.rb b/spec/integration/compliance_spec.rb similarity index 100% rename from spec/compliance_spec.rb rename to spec/integration/compliance_spec.rb diff --git a/spec/date_spec.rb b/spec/integration/date_spec.rb similarity index 100% rename from spec/date_spec.rb rename to spec/integration/date_spec.rb diff --git a/spec/limit_and_order_spec.rb b/spec/integration/limit_and_order_spec.rb similarity index 100% rename from spec/limit_and_order_spec.rb rename to spec/integration/limit_and_order_spec.rb diff --git a/spec/migrations_spec.rb b/spec/integration/migrations_spec.rb similarity index 100% rename from spec/migrations_spec.rb rename to spec/integration/migrations_spec.rb diff --git a/spec/multiple_records_spec.rb b/spec/integration/multiple_records_spec.rb similarity index 100% rename from spec/multiple_records_spec.rb rename to spec/integration/multiple_records_spec.rb diff --git a/spec/nils_spec.rb b/spec/integration/nils_spec.rb similarity index 100% rename from spec/nils_spec.rb rename to spec/integration/nils_spec.rb diff --git a/spec/sdb_array_spec.rb b/spec/integration/sdb_array_spec.rb similarity index 89% rename from spec/sdb_array_spec.rb rename to spec/integration/sdb_array_spec.rb index c1086b9..4510d68 100644 --- a/spec/sdb_array_spec.rb +++ b/spec/integration/sdb_array_spec.rb @@ -1,14 +1,13 @@ require 'pathname' require Pathname(__FILE__).dirname.expand_path + 'spec_helper' -require Pathname(__FILE__).dirname.expand_path + '../lib/simpledb_adapter/sdb_array' -require 'spec/autorun' +require 'simpledb/sdb_array' describe 'with multiple records saved' do class Hobbyist include DataMapper::Resource property :name, String, :key => true - property :hobbies, SdbArray + property :hobbies, SdbArray end before(:each) do @@ -35,7 +34,7 @@ class Hobbyist person.save @adapter.wait_for_consistency lego_person = Hobbyist.first(:name => 'Jeremy Boles') - lego_person.hobbies.should == "lego" + lego_person.hobbies.should == ["lego"] end it 'should allow deletion of array' do @@ -44,7 +43,7 @@ class Hobbyist person.save @adapter.wait_for_consistency lego_person = Hobbyist.first(:name => 'Jeremy Boles') - lego_person.hobbies.should == nil + lego_person.hobbies.should == [] end it 'should find all records with diving hobby' do diff --git a/spec/simpledb_adapter_spec.rb b/spec/integration/simpledb_adapter_spec.rb similarity index 72% rename from spec/simpledb_adapter_spec.rb rename to spec/integration/simpledb_adapter_spec.rb index 5a0a7c1..d446a5a 100644 --- a/spec/simpledb_adapter_spec.rb +++ b/spec/integration/simpledb_adapter_spec.rb @@ -28,6 +28,15 @@ class Network describe DataMapper::Adapters::SimpleDBAdapter do + class Project + include DataMapper::Resource + property :id, Integer, :key => true + property :project_repo, String + property :repo_user, String + property :description, String + end + + LONG_VALUE =<<-EOF #!/bin/sh @@ -159,4 +168,60 @@ class Network end end end + + context "given a pre-existing v0 record" do + before :each do + @record_name = "33d9e5a6fcbd746dc40904a6766d4166e14305fe" + record_attributes = { + "simpledb_type" => ["projects"], + "project_repo" => ["git://github.com/TwP/servolux.git"], + "files_complete" => ["nil"], + "repo_user" => ["nil"], + "id" => ["1077338529"], + "description" => [ + "0002:line 2[[[NEWLINE]]]line 3[[[NEW", + "0001:line 1[[[NEWLINE]]]", + "0003:LINE]]]line 4" + ] + } + @sdb.put_attributes(@domain, @record_name, record_attributes) + sleep 0.4 + @record = Project.get(1077338529) + end + + it "should interpret legacy nil values correctly" do + @record.repo_user.should be_nil + end + + it "should interpret legacy strings correctly" do + @record.description.should == + "line 1\nline 2\nline 3\nline 4" + end + + it "should save legacy records without adding new metadata" do + @record.repo_user = "steve" + @record.save + sleep 0.4 + attributes = @sdb.get_attributes(@domain, @record_name)[:attributes] + attributes.should_not include("__dm_metadata") + end + end + + describe "given a brand-new record" do + before :each do + @record = Project.new( + :repo_user => "steve", + :id => 123, + :project_repo => "git://example.org/foo") + end + + it "should add metadata to the record on save" do + @record.save + sleep 0.4 + items = @sdb.select("select * from #{@domain} where id = '123'")[:items] + attributes = items.first.values.first + attributes["__dm_metadata"].should include("v01.01.00") + attributes["__dm_metadata"].should include("table:projects") + end + end end diff --git a/spec/spec_helper.rb b/spec/integration/spec_helper.rb similarity index 85% rename from spec/spec_helper.rb rename to spec/integration/spec_helper.rb index 17cc58e..a9e1d54 100644 --- a/spec/spec_helper.rb +++ b/spec/integration/spec_helper.rb @@ -1,8 +1,11 @@ require 'pathname' -require Pathname(__FILE__).dirname.parent.expand_path + 'lib/simpledb_adapter' +ROOT = File.expand_path('../..', File.dirname(__FILE__)) +$LOAD_PATH.unshift(File.join(ROOT,'lib')) +require 'simpledb_adapter' require 'logger' require 'fileutils' require 'spec' +require 'spec/autorun' DOMAIN_FILE_MESSAGE = < test_domain) diff --git a/spec/unit/record_spec.rb b/spec/unit/record_spec.rb new file mode 100644 index 0000000..2fdb233 --- /dev/null +++ b/spec/unit/record_spec.rb @@ -0,0 +1,346 @@ +require File.expand_path('unit_spec_helper', File.dirname(__FILE__)) +require 'simpledb/record' +require 'simpledb/sdb_array' + +describe SimpleDB::Record do + + + context "given a record from SimpleDB" do + before :each do + @thing_class = Class.new do + include DataMapper::Resource + + property :foo, Integer + end + @it = SimpleDB::Record.from_simpledb_hash( + {"KEY" => { + "foo" => ["123"], + "baz" => ["456"], + "simpledb_type" => ["thingies"] + } + }) + end + + it "should return nil when asked for a non-existant attribute as String" do + @it["bar", String].should be_nil + end + + it "should return nil when asked for a non-existant attribute as Integer" do + @it["bar", Integer].should be_nil + end + + it "should return [] when asked for a non-existant attribute as Array" do + @it["bar", Array].should == [] + end + + it "should be able to coerce based on a property" do + @it["foo", @thing_class.properties[:foo]].should == 123 + end + + it "should be able to coerce based on a simple type" do + @it["foo", Integer].should == "123" + end + + context "converted to a resource hash" do + before :each do + @hash = @it.to_resource_hash(@thing_class.properties) + end + + it "should only include properties specified in the field set" do + @hash.should_not include(:bar) + end + end + + end + + + context "given a record with no version info" do + before :each do + @resource_class = Class.new do + include DataMapper::Resource + + property :foo, Integer, :key => true + end + + @it = SimpleDB::Record.from_simpledb_hash( + {"KEY" => { + "foo" => ["123"], + "text" => [ + "0001:line 1[[[NEWLINE]]]line 2", + "0002:[[[NEWLINE]]]line 3[[[NEW", + "0003:LINE]]]line 4" + ], + "short_text" => ["foo[[[NEWLINE]]]bar"], + "simpledb_type" => ["thingies"], + "null_field" => ["nil"], + "array" => ["foo[[[NEWLINE]]]bar", "baz"] + } + }) + end + + it "should identify the record as version 0" do + @it.version.should == "00.00.00" + end + + it "should be able to convert the record to a DM-friendly hash" do + @it.to_resource_hash(@resource_class.properties).should == { + "foo" => 123, + } + end + + it "should be able to extract the storage name" do + @it.storage_name.should == "thingies" + end + + it "should subtitute newlines for newline placeholders" do + @it["text", String].should == + "line 1\nline 2\nline 3\nline 4" + end + + it "should identify the table from the simpledb_type attributes" do + @it.table.should == "thingies" + end + + it "should not write any V1.1 metadata" do + @it.writable_attributes.should_not include("__dm_metadata") + end + + it "should interpret ['nil'] as the null value" do + @it["null_field", String].should == nil + @it["null_field", Array].should == [] + end + + describe "migrated to latest version" do + before :each do + @it = @it.migrate + end + + it "should be the latest version" do + @it.version.should == "01.01.00" + end + + it "should mark nil attributes as deletable" do + @it.writable_attributes.should_not include("null_field") + @it.deletable_attributes.should include("null_field") + end + + it "should contain valued attributes" do + @it.writable_attributes["foo"].should == ["123"] + @it.writable_attributes["text"].should == + ["line 1\nline 2\nline 3\nline 4"] + @it.writable_attributes["short_text"].should == + ["foo\nbar"] + @it.writable_attributes["array"].should == + ["foo\nbar", "baz"] + end + + + it "should have writable metadata attributes" do + @it.writable_attributes["__dm_metadata"].should include("v01.01.00") + @it.writable_attributes["__dm_metadata"].should include("table:thingies") + @it.writable_attributes["simpledb_type"].should == ["thingies"] + end + end + end + + context "given a version 1.1 record" do + before :each do + @resource_class = Class.new do + include DataMapper::Resource + + property :bar, Integer + end + + @it = SimpleDB::Record.from_simpledb_hash( + {"KEY" => { + "__dm_metadata" => ["v01.01.00", "table:mystuff"], + "bar" => ["456"], + "text" => [ + "0001:line 1[[[NEWLINE]]]line 2", + "0002:line 3[[[NEW", + "0003:LINE]]]line 4" + ], + } + }) + end + + it "should be a V1 record" do + @it.should be_a_kind_of(SimpleDB::RecordV1_1) + end + + it "should identify the record as version 1" do + @it.version.should == "01.01.00" + end + + it "should be able to convert the record to a DM-friendly hash" do + @it.to_resource_hash(@resource_class.properties).should == { + "bar" => 456 + } + end + + it "should not substitute newline tokens" do + @it["text", String].should == + "line 1[[[NEWLINE]]]line 2line 3[[[NEWLINE]]]line 4" + end + + it "should find the table given in the metadata" do + @it.table.should == "mystuff" + end + end + + context "given a V1.1 record with a chunked string" do + class Poem + include ::DataMapper::Resource + property :text, String + end + + before :each do + @it = SimpleDB::Record.from_simpledb_hash( + {"KEY" => { + "__dm_metadata" => ["v01.01.00"], + "text" => [ + "0002:did gyre and gimbal in the wabe", + "0001:twas brillig and the slithy toves\n", + ] + } + }) + end + + it "should unchunk the text when asked to read it as a String" do + @it["text",String].should == "twas brillig and the slithy toves\n" + + "did gyre and gimbal in the wabe" + end + + it "should return the chunks when asked to read it as an Array" do + @it["text",Array].should == [ + "0002:did gyre and gimbal in the wabe", + "0001:twas brillig and the slithy toves\n", + ] + end + + it "should return the first chunk when asked to read it as anything else" do + @it["text", Integer].should == "0002:did gyre and gimbal in the wabe" + end + + it "should be able to construct a resource hash" do + @it.to_resource_hash(Poem.properties).should == { + "text" => "twas brillig and the slithy toves\ndid gyre and gimbal in the wabe" + } + end + + end + + describe "given an unsaved (new) datamapper resource" do + before :each do + @resource_class = Class.new do + include DataMapper::Resource + storage_names[:default] = "books" + + property :author, String, :key => true + property :date, Date + property :text, DataMapper::Types::Text + property :tags, DataMapper::Types::SdbArray + property :isbn, String + end + @text = "lorem ipsum\n" * 100 + @date = Date.new(2001,1,1) + @author = "Cicero" + @resource = @resource_class.new( + :text => @text, + :date => @date, + :author => @author, + :tags => ['latin', 'classic'], + :isbn => nil) + + @it = SimpleDB::Record.from_resource(@resource) + end + + it "should be able to generate an item name" do + @it.item_name.should == + Digest::SHA1.hexdigest("books+Cicero") + end + + context "as a SimpleDB hash" do + before :each do + @hash = @it.writable_attributes + @deletes = @it.deletable_attributes + end + + it "should translate primitives successfully" do + @hash["author"].should == ["Cicero"] + @hash["date"].should == ["2001-01-01"] + end + + it "should chunk large text sections" do + @hash["text"].should have(2).chunks + end + + it "should be able to round-trip the text it chunks" do + SimpleDB::Record.from_simpledb_hash({"NAME" => @hash})["text", String].should == + @text + end + + it "should translate arrays properly" do + @hash["tags"].should == ['latin', 'classic'] + end + + it "should be able to round-trip arrays" do + SimpleDB::Record.from_simpledb_hash({"NAME" => @hash})["tags", DataMapper::Types::SdbArray].should == + ['latin', 'classic'] + end + + it "should not include nil values in writable attributes" do + @hash.should_not include("isbn") + end + + it "should include resource type in writable attributes" do + @hash["simpledb_type"].should == ["books"] + end + + it "should include nil values in deleteable attributes" do + @deletes.should include("isbn") + end + + it "should include version in writable attributes" do + @hash["__dm_metadata"].should include("v01.01.00") + end + + it "should include type in writable attributes" do + @hash["__dm_metadata"].should include("table:books") + end + + end + end + + describe "given a saved datamapper resource" do + before :each do + @resource_class = Class.new do + include DataMapper::Resource + storage_names[:default] = "books" + + property :author, String, :key => true + property :date, Date + property :text, DataMapper::Types::Text + property :tags, DataMapper::Types::SdbArray + property :isbn, String + end + @date = Date.new(2001,1,1) + @author = "Cicero" + @resource = @resource_class.new( + :text => "", + :date => @date, + :author => @author, + :tags => ['latin', 'classic'], + :isbn => nil) + @resource.stub!(:saved? => true) + @resource.stub!(:new? => false) + + @it = SimpleDB::Record.from_resource(@resource) + end + + it "should not include metadata in writable attributes" do + @it.writable_attributes.should_not include("__dm_metadata") + end + end + +end diff --git a/spec/unit/simpledb_adapter_spec.rb b/spec/unit/simpledb_adapter_spec.rb new file mode 100644 index 0000000..2f45726 --- /dev/null +++ b/spec/unit/simpledb_adapter_spec.rb @@ -0,0 +1,92 @@ +require File.expand_path('unit_spec_helper', File.dirname(__FILE__)) +require 'simpledb_adapter' + +describe DataMapper::Adapters::SimpleDBAdapter do + class Product + include DataMapper::Resource + + property :id, Serial + property :name, String + property :stock, Integer + end + + describe "given a record" do + before :each do + @record = Product.new(:name => "War and Peace", :stock => 3) + end + + it "should be able to save the record" do + @sdb.should_receive(:put_attributes).with( + anything, + anything, + hash_including( + 'simpledb_type' => ["products"], + 'stock' => ["3"], + 'name' => ["War and Peace"])) + @record.save + end + end + + describe "given an existing record" do + before :each do + @sdb.stub(:select). + and_return(:items => [ + {"HANDLE" => { + 'id' => ['12345'], + 'name' => ['War and Peace'], + 'stock' => ['3']}} + ]) + @record = Product.first + end + + it "should be able to update the record" do + @record.stock = 5 + @sdb.should_receive(:put_attributes).with( + anything, + anything, + hash_including('stock' => ["5"]), + :replace) + @record.save + end + end + + describe "given a record exists in the DB" do + before :each do + @sdb.stub(:select). + and_return(:items => [ + {"HANDLE" => { + 'id' => ['12345'], + 'name' => ['War and Peace'], + 'stock' => ['3'], + '__dm_metadata' => ['v01.01.00', 'foobar']}} + ]) + end + + it "should request metadata for the record" do + @sdb.should_receive(:select). + with(/select.*__dm_metadata.*from/i, anything). + and_return(:items => [ + {"HANDLE" => { + 'id' => ['12345'], + 'name' => ['War and Peace'], + 'stock' => ['3'], + '__dm_metadata' => ['v01.01.00', 'foobar']}} + ]) + @record = Product.first + end + + it "should write correct metadata back to the record on update" do + @record = Product.first + @record.stock = 5 + @sdb.should_receive(:put_attributes).with( + anything, + anything, + hash_including( + 'stock' => ["5"], + '__dm_metadata' => ['v01.01.00', 'foobar']), + :replace) + @record.save + end + end + +end diff --git a/spec/unit/unit_spec_helper.rb b/spec/unit/unit_spec_helper.rb new file mode 100644 index 0000000..dbd3540 --- /dev/null +++ b/spec/unit/unit_spec_helper.rb @@ -0,0 +1,26 @@ +require 'spec' +ROOT = File.expand_path('../..', File.dirname(__FILE__)) +$LOAD_PATH.unshift(File.join(ROOT,'lib')) +require 'simpledb_adapter' + +Spec::Runner.configure do |config| + config.before :each do + @sdb = stub("RightAWS::SdbInterface").as_null_object + @log = stub("Log").as_null_object + + # Using Abstract adapter as a null DB + DataMapper.setup(:default, + :adapter => 'simpledb', + :access_key => "ACCESS_KEY", + :secret_key => "SECRET_KEY", + :domain => "DOMAIN", + :logger => @log, + :sdb_interface => @sdb + ) + end + + config.after :each do + DataMapper::Repository.adapters.delete(:default) + end + +end