diff --git a/CHANGELOG.rdoc b/CHANGELOG.rdoc index 4931a11..f960b0c 100644 --- a/CHANGELOG.rdoc +++ b/CHANGELOG.rdoc @@ -5,7 +5,10 @@ Below is a complete listing of changes for each revision of Oklahoma Mixer. == 0.4.0 * Added the read_only?() method -* Added error handling for table queries +* Added error handling for Table Databases queries +* Added support for Table Databases queries on read only databases +* Added support for iteration blocks to Table Databases searches +* Modified Table Databases blocks to yield tuples consistent with the iterators == 0.3.0 diff --git a/TODO.rdoc b/TODO.rdoc index 8351b22..0fbcaea 100644 --- a/TODO.rdoc +++ b/TODO.rdoc @@ -4,8 +4,6 @@ The following is a list of planned expansions for Oklahoma Mixer in the order I intend to address them. 1. Resolve minor issues - * Support a read only-version of all() without a block - * Make all() yield() like Hash#each (a key-value tuple) * Speed up counts (fetch keys only to prevent HashMap conversion) * Push :select down into all() call to speed up paginate() 2. Include some higher level abstractions like mixed tables, queues, and shards diff --git a/lib/oklahoma_mixer/array_list.rb b/lib/oklahoma_mixer/array_list.rb index 6e287a3..6b890b1 100644 --- a/lib/oklahoma_mixer/array_list.rb +++ b/lib/oklahoma_mixer/array_list.rb @@ -11,17 +11,12 @@ def initialize(pointer_or_size) attr_reader :pointer - def shift - value = C.read_from_func(:shift, @pointer) - block_given? ? yield(value) : value - end + include Enumerable - def map - values = [ ] - while value = shift - values << yield(value) + def each + (0...C.num(pointer)).each do |i| + yield C.read_from_func(:val, :no_free, @pointer, i) end - values end def push(*values) diff --git a/lib/oklahoma_mixer/array_list/c.rb b/lib/oklahoma_mixer/array_list/c.rb index 8b65e24..fec4dfa 100644 --- a/lib/oklahoma_mixer/array_list/c.rb +++ b/lib/oklahoma_mixer/array_list/c.rb @@ -12,10 +12,14 @@ module C # :nodoc: :returns => :pointer func :name => :del, :args => :pointer - - func :name => :shift, - :args => [:pointer, :pointer], + + func :name => :num, + :args => :pointer, + :returns => :int + func :name => :val, + :args => [:pointer, :int, :pointer], :returns => :pointer + func :name => :push, :args => [:pointer, :pointer, :int] end diff --git a/lib/oklahoma_mixer/table_database.rb b/lib/oklahoma_mixer/table_database.rb index 3dcde74..a8be5c0 100644 --- a/lib/oklahoma_mixer/table_database.rb +++ b/lib/oklahoma_mixer/table_database.rb @@ -114,7 +114,7 @@ def each def all(options = { }, &iterator) query(options) do |q| mode = results_mode(options) - if block_given? + if not iterator.nil? and not read_only? results = self callback = lambda { |key_pointer, key_size, doc_map, _| if mode != :docs @@ -125,9 +125,10 @@ def all(options = { }, &iterator) doc = map.to_hash { |string| cast_to_encoded_string(string) } end flags = case mode - when :keys then yield(key) - when :docs then yield(doc) - else yield(key, doc) + when :keys then iterator[key] + when :docs then iterator[doc] + when :aoh then iterator[doc.merge!(:primary_key => key)] + else iterator[[key, doc]] end Array(flags).inject(0) { |returned_flags, flag| returned_flags | case flag.to_s @@ -144,34 +145,16 @@ def all(options = { }, &iterator) end } } + unless lib.qryproc(q.pointer, callback, nil) + error_code = lib.ecode(@db) + error_message = lib.errmsg(error_code) + fail Error::QueryError, + "#{error_message} (error code #{error_code})" + end + results else - results = mode != :hoh ? [ ] : { } - callback = lambda { |key_pointer, key_size, doc_map, _| - if mode == :docs - results << cast_value_out(doc_map, :no_free) - else - key = cast_key_out(key_pointer.get_bytes(0, key_size)) - case mode - when :keys - results << key - when :hoh - results[key] = cast_value_out(doc_map, :no_free) - when :aoh - results << cast_value_out(doc_map, :no_free). - merge(:primary_key => key) - else - results << [key, cast_value_out(doc_map, :no_free)] - end - end - 0 - } + query_results(lib.qrysearch(q.pointer), mode, &iterator) end - unless lib.qryproc(q.pointer, callback, nil) - error_code = lib.ecode(@db) - error_message = lib.errmsg(error_code) - fail Error::QueryError, "#{error_message} (error code #{error_code})" - end - results end end @@ -196,7 +179,8 @@ def paginate(options) fail Error::QueryError, ":per_page must be >= 1" if results.per_page < 1 results.total_entries = 0 all( options.merge( :select => :keys_and_docs, - :limit => nil ) ) { |key, value| + :return => :aoa, + :limit => nil ) ) do |key, value| if results.total_entries >= results.offset and results.size < results.per_page case mode @@ -213,21 +197,21 @@ def paginate(options) end end results.total_entries += 1 - } + end results end - def union(q, *queries) - search([q] + queries, lib::SEARCHES[:TDBMSUNION]) + def union(q, *queries, &iterator) + search([q] + queries, lib::SEARCHES[:TDBMSUNION], &iterator) end - def intersection(q, *queries) - search([q] + queries, lib::SEARCHES[:TDBMSISECT]) + def intersection(q, *queries, &iterator) + search([q] + queries, lib::SEARCHES[:TDBMSISECT], &iterator) end alias_method :isect, :intersection - def difference(q, *queries) - search([q] + queries, lib::SEARCHES[:TDBMSDIFF]) + def difference(q, *queries, &iterator) + search([q] + queries, lib::SEARCHES[:TDBMSDIFF], &iterator) end alias_method :diff, :difference @@ -365,34 +349,61 @@ def results_mode(options) end end - def search(queries, operation) - mode = results_mode(queries.first) - qs = queries.map { |q| query(q) } - keys = ArrayList.new( Utilities.temp_pointer(qs.size) do |pointer| - pointer.write_array_of_pointer(qs.map { |q| q.pointer }) - lib.metasearch(pointer, qs.size, operation) - end ) - case mode - when :keys - keys.map { |key| cast_key_out(key) } - when :docs - keys.map { |key| self[cast_key_out(key)] } - when :hoh - results = { } - while key = keys.shift { |k| cast_key_out(k) } - results[key] = self[key] + def query_results(results, mode, &iterator) + keys = ArrayList.new(results) + if iterator.nil? + results = mode == :hoh ? { } : [ ] + iterator = lambda do |key_and_value| + if mode == :hoh + results[key_and_value.first] = key_and_value.last + else + results << key_and_value + end end - results - when :aoh - keys.map { |key| - key = cast_key_out(key) - self[key].merge(:primary_key => key) - } else - keys.map { |key| - key = cast_key_out(key) - [key, self[key]] - } + results = self + end + keys.each do |key| + flags = Array( case mode + when :keys + iterator[cast_key_out(key)] + when :docs + iterator[self[cast_key_out(key)]] + when :aoh + k = cast_key_out(key) + iterator[self[k].merge!(:primary_key => k)] + else + k = cast_key_out(key) + v = self[k] + iterator[[k, v]] + end ).map { |flag| flag.to_s } + if flags.include? "delete" + if read_only? + warn "attempted delete from a read only query" + else + delete(key) + end + elsif v and flags.include? "update" + if read_only? + warn "attempted update from a read only query" + else + self[k] = v + end + end + break if flags.include? "break" + end + results + ensure + keys.free if keys + end + + def search(queries, operation, &iterator) + qs = queries.map { |q| query(q) } + Utilities.temp_pointer(qs.size) do |pointer| + pointer.write_array_of_pointer(qs.map { |q| q.pointer }) + query_results( lib.metasearch(pointer, qs.size, operation), + results_mode(queries.first), + &iterator) end ensure if qs @@ -400,7 +411,6 @@ def search(queries, operation) q.free end end - keys.free if keys end end end diff --git a/lib/oklahoma_mixer/table_database/c.rb b/lib/oklahoma_mixer/table_database/c.rb index e257552..ea53b6a 100644 --- a/lib/oklahoma_mixer/table_database/c.rb +++ b/lib/oklahoma_mixer/table_database/c.rb @@ -90,6 +90,9 @@ module C # :nodoc: :args => [:pointer, :string, INDEXES], :returns => :bool + func :name => :qrysearch, + :args => :pointer, + :returns => :pointer call :name => :TDBQRYPROC, :args => [:pointer, :int, :pointer, :pointer], :returns => :int diff --git a/test/table_database/query_test.rb b/test/table_database/query_test.rb index 74490d5..f95741d 100644 --- a/test/table_database/query_test.rb +++ b/test/table_database/query_test.rb @@ -114,6 +114,16 @@ def test_all_can_pass_documents_only_to_a_passed_block_and_return_self results.sort_by { |doc| doc.size } ) end + def test_all_can_pass_key_in_document_to_a_passed_block_and_return_self + load_simple_data + results = [ ] + assert_equal(@db, @db.all(:return => :aoh) { |kv| results << kv }) + assert_equal( [ {:primary_key => "pk2"}, + { :primary_key => "pk1", + "a" => "1", "b" => "2", "c" => "3" } ], + results.sort_by { |doc| doc.size } ) + end + def test_all_with_a_block_does_not_modify_records_by_default load_simple_data assert_equal(@db, @db.all { }) @@ -159,6 +169,49 @@ def test_all_with_a_block_can_combine_flags assert_match((%w[pk1 pk2] - results).first, @db.keys.first) end + def test_all_methods_can_control_what_is_passed_to_the_block + load_simple_data + [ [{:select => :keys}, "pk1"], + [{:select => :docs}, {"a" => "1", "b" => "2", "c" => "3"}], + [ {:return => :aoa}, [ "pk1", + {"a" => "1", "b" => "2", "c" => "3"} ] ], + [ {:return => :hoh}, [ "pk1", + {"a" => "1", "b" => "2", "c" => "3"} ] ], + [ {:return => :aoh}, { :primary_key => "pk1", + "a" => "1", + "b" => "2", + "c" => "3" } ] ].each do |query, results| + args = [ ] + @db.all(query.merge(:conditions => [:a, :==, 1])) do |kv| + args << kv + end + assert_equal([results], args) + end + end + + def test_all_yields_key_value_tuples + load_simple_data + [ [ {:return => :aoa}, [ "pk1", + {"a" => "1", "b" => "2", "c" => "3"} ] ], + [ {:return => :hoh}, [ "pk1", + { "a" => "1", + "b" => "2", + "c" => "3" } ] ] ].each do |query, tuple| + yielded = nil + @db.all(query.merge(:conditions => [:a, :==, 1])) do |kv| + yielded = kv + end + assert_equal(tuple, yielded) + key, value = nil, nil + @db.all(query.merge(:conditions => [:a, :==, 1])) do |k, v| + key = k + value = v + end + assert_equal(tuple.first, key) + assert_equal(tuple.last, value) + end + end + def test_all_fails_with_an_error_for_malformed_conditions assert_raise(OKMixer::Error::QueryError) do @db.all(:conditions => :first) # not column, operator, and expression @@ -757,6 +810,82 @@ def test_union_respects_select_and_return_on_the_first_query :order => :first }, {:conditions => [:age, :==, 34]} ) ) end + + def test_union_can_be_passed_a_block_to_iterate_over_the_results + load_condition_data + results = [ ] + assert_equal( @db, + @db.union( { :select => :keys, + :conditions => [:first, :ends_with?, "es"], + :order => :first }, + {:conditions => [:age, :==, 34]} ) { |k| + results << k + } ) + assert_equal(%w[dana james], results) + end + + def test_union_with_a_block_can_update_records + load_condition_data + assert_equal( @db, + @db.union( { :conditions => [:first, :ends_with?, "es"], + :order => :first }, + {:conditions => [:age, :==, 34]} ) { |k, v| + if k == "dana" + v["salutation"] = "Mrs." # add + v["middle"] = "AL" # update + v.delete("age") # delete + :update + end + } ) + assert_equal( { "salutation" => "Mrs.", + "first" => "Dana", + "middle" => "AL", + "last" => "Gray" }, @db["dana"] ) + end + + def test_union_with_a_block_can_delete_records + load_condition_data + results = [ ] + assert_equal( @db, + @db.union( { :select => :keys, + :conditions => [:first, :ends_with?, "es"], + :order => :first }, + {:conditions => [:age, :==, 34]} ) { |k| + :delete + } ) + assert_equal(1, @db.size) + assert_equal(%w[jim], @db.keys) + end + + def test_union_with_a_block_can_end_the_query + load_condition_data + results = [ ] + assert_equal( @db, + @db.union( { :select => :keys, + :conditions => [:first, :ends_with?, "es"], + :order => :first }, + {:conditions => [:age, :==, 34]} ) { |k| + results << k + :break + } ) + assert_equal(%w[dana], results) + end + + def test_union_with_a_block_can_combine_flags + load_condition_data + results = [ ] + assert_equal( @db, + @db.union( { :select => :keys, + :conditions => [:first, :ends_with?, "es"], + :order => :first }, + {:conditions => [:age, :==, 34]} ) { |k| + results << k + %w[delete break] + } ) + assert_equal(%w[dana], results) + assert_nil(@db["dana"]) + assert_equal(2, @db.size) + end def test_intersection_returns_the_set_intersection_of_multiple_queries load_condition_data @@ -798,6 +927,86 @@ def test_intersection_respects_select_and_return_on_the_first_query :order => :first }, {:conditions => [:last, :==, "Gray"]} ) ) end + + def test_intersection_can_be_passed_a_block_to_iterate_over_the_results + load_condition_data + results = [ ] + assert_equal( @db, + @db.isect( { :select => :keys, + :return => :hoh, + :conditions => [:first, :include?, "a"], + :order => :first }, + {:conditions => [:last, :==, "Gray"]} ) { |k| + results << k + } ) + assert_equal(%w[dana james], results) + end + + def test_intersection_with_a_block_can_update_records + load_condition_data + assert_equal( @db, + @db.isect( { :return => :hoh, + :conditions => [:first, :include?, "a"], + :order => :first }, + {:conditions => [:last, :==, "Gray"]} ) { |k, v| + if k == "dana" + v["salutation"] = "Mrs." # add + v["middle"] = "AL" # update + v.delete("age") # delete + :update + end + } ) + assert_equal( { "salutation" => "Mrs.", + "first" => "Dana", + "middle" => "AL", + "last" => "Gray" }, @db["dana"] ) + end + + def test_intersection_with_a_block_can_delete_records + load_condition_data + assert_equal( @db, + @db.isect( { :select => :keys, + :return => :hoh, + :conditions => [:first, :include?, "a"], + :order => :first }, + {:conditions => [:last, :==, "Gray"]} ) { |k| + :delete + } ) + assert_equal(1, @db.size) + assert_equal(%w[jim], @db.keys) + end + + def test_intersection_with_a_block_can_end_the_query + load_condition_data + results = [ ] + assert_equal( @db, + @db.isect( { :select => :keys, + :return => :hoh, + :conditions => [:first, :include?, "a"], + :order => :first }, + {:conditions => [:last, :==, "Gray"]} ) { |k| + results << k + :break + } ) + assert_equal(%w[dana], results) + end + + def test_intersection_with_a_block_can_combine_flags + load_condition_data + results = [ ] + assert_equal( @db, + @db.isect( { :select => :keys, + :return => :hoh, + :conditions => [:first, :include?, "a"], + :order => :first }, + {:conditions => [:last, :==, "Gray"]} ) { |k| + results << k + %w[delete break] + } ) + assert_equal(%w[dana], results) + assert_nil(@db["dana"]) + assert_equal(2, @db.size) + end def test_difference_returns_the_set_difference_of_multiple_queries load_condition_data @@ -836,6 +1045,145 @@ def test_difference_respects_select_and_return_on_the_first_query {:conditions => [:first, :==, "Jim"]} ) ) end + def test_difference_can_be_passed_a_block_to_iterate_over_the_results + load_condition_data + results = [ ] + assert_equal( @db, + @db.diff( { :select => :keys, + :conditions => [:last, :==, "Gray"], + :order => :first }, + {:conditions => [:first, :==, "Jim"]} ) { |k| + results << k + } ) + assert_equal(%w[dana james], results) + end + + def test_difference_with_a_block_can_update_records + load_condition_data + assert_equal( @db, + @db.diff( { :conditions => [:last, :==, "Gray"], + :order => :first }, + {:conditions => [:first, :==, "Jim"]} ) { |k, v| + if k == "dana" + v["salutation"] = "Mrs." # add + v["middle"] = "AL" # update + v.delete("age") # delete + :update + end + } ) + assert_equal( { "salutation" => "Mrs.", + "first" => "Dana", + "middle" => "AL", + "last" => "Gray" }, @db["dana"] ) + end + + def test_difference_with_a_block_can_delete_records + load_condition_data + assert_equal( @db, + @db.diff( { :select => :keys, + :conditions => [:last, :==, "Gray"], + :order => :first }, + {:conditions => [:first, :==, "Jim"]} ) { |k| + :delete + } ) + assert_equal(1, @db.size) + assert_equal(%w[jim], @db.keys) + end + + def test_difference_with_a_block_can_end_the_query + load_condition_data + results = [ ] + assert_equal( @db, + @db.diff( { :select => :keys, + :conditions => [:last, :==, "Gray"], + :order => :first }, + {:conditions => [:first, :==, "Jim"]} ) { |k| + results << k + :break + } ) + assert_equal(%w[dana], results) + end + + def test_difference_with_a_block_can_combine_flags + load_condition_data + results = [ ] + assert_equal( @db, + @db.diff( { :select => :keys, + :conditions => [:last, :==, "Gray"], + :order => :first }, + {:conditions => [:first, :==, "Jim"]} ) { |k| + results << k + %w[delete break] + } ) + assert_equal(%w[dana], results) + assert_nil(@db["dana"]) + assert_equal(2, @db.size) + end + + def test_search_methods_can_control_what_is_passed_to_the_block + load_condition_data + [ [{:select => :keys}, "dana"], + [ {:select => :docs}, { "first" => "Dana", + "middle" => "Ann Leslie", + "last" => "Gray", + "age" => "34" } ], + [ {:return => :aoa}, [ "dana", + { "first" => "Dana", + "middle" => "Ann Leslie", + "last" => "Gray", + "age" => "34" } ] ], + [ {:return => :hoh}, [ "dana", + { "first" => "Dana", + "middle" => "Ann Leslie", + "last" => "Gray", + "age" => "34" } ] ], + [ {:return => :aoh}, { :primary_key => "dana", + "first" => "Dana", + "middle" => "Ann Leslie", + "last" => "Gray", + "age" => "34" } ] ].each do |query, results| + %w[union intersection difference].each do |search| + args = [ ] + @db.send( search, + query.merge(:conditions => [:first, :==, "Dana"]) ) do |kv| + args << kv + end + assert_equal([results], args) + end + end + end + + def test_search_methods_yields_key_value_tuples + load_condition_data + [ [ {:return => :aoa}, [ "dana", + { "first" => "Dana", + "middle" => "Ann Leslie", + "last" => "Gray", + "age" => "34" } ] ], + [ {:return => :hoh}, [ "dana", + { "first" => "Dana", + "middle" => "Ann Leslie", + "last" => "Gray", + "age" => "34" } ] ] ].each do |query, tuple| + %w[union intersection difference].each do |search| + yielded = nil + @db.send( search, + query.merge(:conditions => [:first, :==, "Dana"]) ) do |kv| + yielded = kv + end + assert_equal(tuple, yielded) + key, value = nil, nil + @db.send( search, + query.merge(:conditions => [:first, :==, "Dana"]) ) do |k, v| + key = k + value = v + end + assert_equal(tuple.first, key) + assert_equal(tuple.last, value) + end + end + end + private def load_simple_data diff --git a/test/table_database/read_only_query_test.rb b/test/table_database/read_only_query_test.rb index b8954c3..52f94ae 100644 --- a/test/table_database/read_only_query_test.rb +++ b/test/table_database/read_only_query_test.rb @@ -2,7 +2,11 @@ class TestReadOnlyQuery < Test::Unit::TestCase def setup - tdb { } # create the database file + # create the database and load some data + tdb do |db| + db[:pk1] = {:a => 1, :b => 2, :c => 3} + db[:pk2] = { } + end @db = tdb("r") end @@ -11,9 +15,141 @@ def teardown remove_db_files end - def test_using_a_block_fails_with_an_error - assert_raise(OKMixer::Error::QueryError) do - @db.all { } + def test_all_with_a_block_can_end_the_query + results = [ ] + assert_equal(@db, @db.all { |k, _| results << k; :break }) + assert_equal(1, results.size) + assert_match(/\Apk[12]\z/, results.first) + end + + def test_all_with_block_delete_is_ignored_and_triggers_a_warning + warning = capture_stderr do + assert_equal(@db, @db.all { :delete }) + end + assert_equal(2, @db.size) + assert( !warning.empty?, + "A warning was not issued for :delete in a read only query" ) + end + + def test_all_with_block_update_is_ignored_and_triggers_a_warning + warning = capture_stderr do + assert_equal( @db, @db.all { |k, v| + if k == "pk1" + v["a"] = "1.1" # change + v.delete("c") # remove + v[:d] = 4 # add + :update + end + } ) + end + assert_equal({"a" => "1", "b" => "2", "c" => "3"}, @db[:pk1]) + assert( !warning.empty?, + "A warning was not issued for :update in a read only query" ) + end + + def test_all_methods_can_control_what_is_passed_to_the_block + [ [{:select => :keys}, "pk1"], + [{:select => :docs}, {"a" => "1", "b" => "2", "c" => "3"}], + [ {:return => :aoa}, [ "pk1", + {"a" => "1", "b" => "2", "c" => "3"} ] ], + [ {:return => :hoh}, [ "pk1", + {"a" => "1", "b" => "2", "c" => "3"} ] ], + [ {:return => :aoh}, { :primary_key => "pk1", + "a" => "1", + "b" => "2", + "c" => "3" } ] ].each do |query, results| + args = [ ] + @db.all(query.merge(:conditions => [:a, :==, 1])) do |kv| + args << kv + end + assert_equal([results], args) + end + end + + def test_all_yields_key_value_tuples + [ [ {:return => :aoa}, [ "pk1", + {"a" => "1", "b" => "2", "c" => "3"} ] ], + [ {:return => :hoh}, [ "pk1", + { "a" => "1", + "b" => "2", + "c" => "3" } ] ] ].each do |query, tuple| + yielded = nil + @db.all(query.merge(:conditions => [:a, :==, 1])) do |kv| + yielded = kv + end + assert_equal(tuple, yielded) + key, value = nil, nil + @db.all(query.merge(:conditions => [:a, :==, 1])) do |k, v| + key = k + value = v + end + assert_equal(tuple.first, key) + assert_equal(tuple.last, value) + end + end + + def test_count_works_on_a_read_only_database + assert_equal(@db.size, @db.count) + end + + def test_paginate_works_on_a_read_only_database + assert_equal( %w[pk1], @db.paginate( :select => :keys, + :order => :primary_key, + :per_page => 1, + :page => 1 ) ) + assert_equal( %w[pk2], @db.paginate( :select => :keys, + :order => :primary_key, + :per_page => 1, + :page => 2 ) ) + assert_equal( [ ], @db.paginate( :select => :keys, + :order => :primary_key, + :per_page => 1, + :page => 3 ) ) + end + + def test_search_with_a_block_can_end_the_query + each_search do |search| + results = [ ] + assert_equal(@db, @db.send(search, :order => :primary_key) { |k, _| + results << k; :break + }) + assert_equal(1, results.size) + assert_equal("pk1", results.first) + end + end + + def test_search_with_block_delete_is_ignored_and_triggers_a_warning + each_search do |search| + warning = capture_stderr do + assert_equal(@db, @db.send(search, :order => :primary_key) { :delete }) + end + assert_equal(2, @db.size) + assert( !warning.empty?, + "A warning was not issued for :delete in a read only search" ) + end + end + + def test_search_with_block_update_is_ignored_and_triggers_a_warning + each_search do |search| + warning = capture_stderr do + assert_equal( @db, @db.send(search, :order => :primary_key) { |k, v| + if k == "pk1" + v["a"] = "1.1" # change + v.delete("c") # remove + v[:d] = 4 # add + :update + end + } ) + end + assert_equal({"a" => "1", "b" => "2", "c" => "3"}, @db[:pk1]) + assert( !warning.empty?, + "A warning was not issued for :update in a read only search" ) end end + + private + + def each_search(&test) + %w[union intersection difference].each(&test) + end end