diff --git a/API.md b/API.md index 5f1731a..8e127d2 100644 --- a/API.md +++ b/API.md @@ -1,5 +1,5 @@ # Supported API -## Implemented 7% (20/259) +## Implemented 8% (22/259) ### Database (2 / 24) |Command |impl|test|note| @@ -47,21 +47,21 @@ |`rocksdb_multi_get` | | | | |`rocksdb_multi_get_cf`| | | | -### Iteration (8 / 12) +### Iteration (10 / 12) |Command |impl|test|note| |----------------------------|:--:|:--:|----| -|`rocksdb_create_iterator` | ✓ | | | +|`rocksdb_create_iterator` | ✓ | ✓ | | |`rocksdb_create_iterator_cf`| | | | -|`rocksdb_iter_destroy` | ✓ | | | -|`rocksdb_iter_valid` | ✓ | | | -|`rocksdb_iter_seek_to_first`| ✓ | | | -|`rocksdb_iter_seek_to_last` | ✓ | | | -|`rocksdb_iter_seek` | | | | -|`rocksdb_iter_next` | ✓ | | | -|`rocksdb_iter_prev` | ✓ | | | -|`rocksdb_iter_key` | ✓ | | | -|`rocksdb_iter_value` | | | | +|`rocksdb_iter_destroy` | ✓ | ✓ | | +|`rocksdb_iter_valid` | ✓ | ✓ | | +|`rocksdb_iter_seek_to_first`| ✓ | ✓ | | +|`rocksdb_iter_seek_to_last` | ✓ | ✓ | | +|`rocksdb_iter_seek` | ✓ | ✓ | | +|`rocksdb_iter_next` | ✓ | ✓ | | +|`rocksdb_iter_prev` | ✓ | ✓ | | +|`rocksdb_iter_key` | ✓ | ✓ | | +|`rocksdb_iter_value` | ✓ | ✓ | | |`rocksdb_iter_get_error` | | | | ### Snapshots (0 / 15) diff --git a/Makefile b/Makefile index f80dd24..62ea07f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ API_IMPLS := src/rocksdb/api.cr OPT_IMPLS := src/rocksdb/options.cr -CMD_TESTS := $(shell find spec/commands -name '*.cr') +CMD_TESTS := $(shell find spec -name '*_spec.cr') API_FILES := $(shell find doc/api -name '*.*') .PHONY : all diff --git a/README.md b/README.md index 7aede6c..c78b7ec 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,44 @@ db.get("foo") # => "" db.get?("foo") # => nil db.get!("foo") # raise RocksDB::NotFound.new("foo") -db.keys # => ["k1","k2","k3",...] -db.keys(2) # => ["k1","k2"] - db.close ``` +### Iterations + +```shell +3.times{|i| db.put("k#{i}", i) } +db.keys # => ["k0","k1","k2"] +db.keys(2) # => ["k0","k1"] + +db.each do |(k,v)| +``` + +### Iterator + +```shell +iter = db.new_iterator +iter.key # => "k0" +iter.next +iter.value # => "1" +iter.first +iter.key # => "k0" +iter.seek("k2") +iter.next +iter.valid? # => false +``` + +- same as `each` + +```shell +iter = db.new_iterator +iter.first +while (iter.valid?) + # yield {iter.key, iter.value} + iter.next +end +``` + ### binary data Although all data are stored as Binary in RocksDB, @@ -57,8 +89,12 @@ db.get("\u{0}") # => "\t" #### 0.3.0 -- [x] `keys` -- [ ] `each` +- [x] `keys` (String) +- [x] `each` (String) + +#### 0.4.0 + +- [ ] Iterations for Binary ## Testing diff --git a/doc/api/impl b/doc/api/impl index 2462f85..428c5d9 100644 --- a/doc/api/impl +++ b/doc/api/impl @@ -6,9 +6,11 @@ rocksdb_iter_destroy rocksdb_iter_key rocksdb_iter_next rocksdb_iter_prev +rocksdb_iter_seek rocksdb_iter_seek_to_first rocksdb_iter_seek_to_last rocksdb_iter_valid +rocksdb_iter_value rocksdb_open rocksdb_options_create rocksdb_options_destroy diff --git a/doc/api/test b/doc/api/test index 41f7527..b14ab1a 100644 --- a/doc/api/test +++ b/doc/api/test @@ -1,7 +1,18 @@ each keys +new_iterator rocksdb_close +rocksdb_create_iterator rocksdb_delete rocksdb_get +rocksdb_iter_destroy +rocksdb_iter_key +rocksdb_iter_next +rocksdb_iter_prev +rocksdb_iter_seek +rocksdb_iter_seek_to_first +rocksdb_iter_seek_to_last +rocksdb_iter_valid +rocksdb_iter_value rocksdb_open rocksdb_put diff --git a/spec/commands/iteration_spec.cr b/spec/commands/iteration_spec.cr index 5fd8397..5fa05f4 100644 --- a/spec/commands/iteration_spec.cr +++ b/spec/commands/iteration_spec.cr @@ -1,17 +1,17 @@ require "../spec_helper" describe "Iterations" do - path = "tmp/db_iter" + path = "tmp/db_iteration" it "(setup)" do db = RocksDB::DB.new(path) - 5.times{|i| db.put("k#{i}", i) } + 3.times{|i| db.put("k#{i}", i) } db.close end it "#keys" do db = RocksDB::DB.new(path) - db.keys.should eq(%w( k0 k1 k2 k3 k4 )) + db.keys.should eq(%w( k0 k1 k2 )) db.close end @@ -23,12 +23,19 @@ describe "Iterations" do it "#each" do db = RocksDB::DB.new(path) + items = [] of Tuple(String, String) + db.each do |k,v| + items << {k, v} + end + items.should eq([{"k0", "0"}, {"k1", "1"}, {"k2", "2"}]) + db.close + end -# items = [] of Tuple(String, String) -# db.each do |k,v| -# items << {k, v} -# end - + it "#new_iterator" do + db = RocksDB::DB.new(path) + iter = db.new_iterator + iter.should be_a(RocksDB::Iterator) + iter.close db.close end end diff --git a/spec/iterator_spec.cr b/spec/iterator_spec.cr new file mode 100644 index 0000000..b166f40 --- /dev/null +++ b/spec/iterator_spec.cr @@ -0,0 +1,61 @@ +require "./spec_helper" + +describe "Iterator" do + path = "tmp/db_iteration" + + it "work" do + db = RocksDB::DB.new(path) + 3.times{|i| db.put("k#{i}", i) } + iter = RocksDB::Iterator.new(db) + + iter.valid?.should be_false + + iter.first + iter.valid?.should be_true + iter.key.should eq("k0") + iter.value.should eq("0") + + iter.next + iter.key.should eq("k1") + iter.value.should eq("1") + + iter.next + iter.key.should eq("k2") + iter.value.should eq("2") + + iter.prev + iter.key.should eq("k1") + iter.value.should eq("1") + + iter.first + iter.key.should eq("k0") + iter.value.should eq("0") + + iter.last + iter.key.should eq("k2") + iter.value.should eq("2") + + iter.next + iter.valid?.should be_false + + iter.seek("k1") + iter.valid?.should be_true + iter.key.should eq("k1") + + iter.seek("xx") + iter.valid?.should be_false + + db.close + end + + it "#rocksdb_create_iterator" do ; end + it "#rocksdb_iter_destroy" do ; end + it "#rocksdb_iter_seek_to_first" do ; end + it "#rocksdb_iter_seek_to_last" do ; end + it "#rocksdb_iter_seek" do; end + it "#rocksdb_iter_prev" do; end + it "#rocksdb_iter_next" do; end + it "#rocksdb_iter_key" do; end + it "#rocksdb_iter_value" do ; end + it "#rocksdb_iter_valid" do ; end +end diff --git a/src/rocksdb/api.cr b/src/rocksdb/api.cr index b4279f3..e01d116 100644 --- a/src/rocksdb/api.cr +++ b/src/rocksdb/api.cr @@ -35,11 +35,12 @@ module RocksDB::Api api rocksdb_iter_seek_to_last api rocksdb_iter_prev api rocksdb_iter_next + api rocksdb_iter_seek api rocksdb_iter_key + api rocksdb_iter_value api rocksdb_iter_valid # fun rocksdb_iter_seek(x0 : RocksdbIteratorT, k : LibC::Char*, klen : LibC::SizeT) -# fun rocksdb_iter_value(x0 : RocksdbIteratorT, vlen : LibC::SizeT*) : LibC::Char* # fun rocksdb_iter_get_error(x0 : RocksdbIteratorT, errptr : LibC::Char**) diff --git a/src/rocksdb/commands/iteration.cr b/src/rocksdb/commands/iteration.cr index 0b21f70..d9f566f 100644 --- a/src/rocksdb/commands/iteration.cr +++ b/src/rocksdb/commands/iteration.cr @@ -1,20 +1,32 @@ module RocksDB::Commands include Api + + def new_iterator + Iterator.new(self) + end def each(&block) - iter = Iterator.new(self) +# iter = StringIterator.new(self) + iter = new_iterator + iter.first + while iter.valid? + yield({iter.key, iter.value}) + iter.next + end + ensure + iter.try(&.close) end def keys(limit = Int32::MAX) - iter = Iterator.new(self) - iter.first! + iter = new_iterator + iter.first array = [] of String i = 0 while (key = iter.key?) i += 1 break if i > limit array << key - iter.next! + iter.next end return array ensure diff --git a/src/rocksdb/db.cr b/src/rocksdb/db.cr index bb092dd..0c06faa 100644 --- a/src/rocksdb/db.cr +++ b/src/rocksdb/db.cr @@ -1,8 +1,8 @@ class RocksDB::DB - include Value(LibRocksDB::RocksdbT) + include RawMemory(LibRocksDB::RocksdbT) include Commands - alias RawValue = Options | ReadOptions | WriteOptions + alias Closable = Options | ReadOptions | WriteOptions @options : Options @read_options : ReadOptions @@ -12,11 +12,11 @@ class RocksDB::DB getter! write_options def initialize(@path : String, options : Options? = nil, read_options : ReadOptions? = nil, write_options : WriteOptions? = nil) - @raw_values = [] of RawValue + @closables = [] of Closable - @options = options || raw_value(Options.new.set_create_if_missing(1)) - @read_options = read_options || raw_value(ReadOptions.new) - @write_options = write_options || raw_value(WriteOptions.new) + @options = options || mark_closable(Options.new.set_create_if_missing(1)) + @read_options = read_options || mark_closable(ReadOptions.new) + @write_options = write_options || mark_closable(WriteOptions.new) @len = Pointer(UInt64).malloc(1_u64) @err = Pointer(Pointer(UInt8)).malloc(1_u64) @@ -25,8 +25,8 @@ class RocksDB::DB end protected def free - @raw_values.each(&.close) - @raw_values.clear + @closables.each(&.close) + @closables.clear rocksdb_close(@raw.not_nil!) end @@ -34,8 +34,8 @@ class RocksDB::DB @path end - protected def raw_value(value) - @raw_values << value + protected def mark_closable(value) + @closables << value value end end diff --git a/src/rocksdb/dsl.cr b/src/rocksdb/dsl.cr index 5869801..123377a 100644 --- a/src/rocksdb/dsl.cr +++ b/src/rocksdb/dsl.cr @@ -3,7 +3,7 @@ macro option_class(klass) {% method = klass.stringify.downcase %} class RocksDB::{{ klass.id }} include Api - include Value({{ underly.id }}) + include RawMemory({{ underly.id }}) def initialize @raw = rocksdb_{{ method.id }}_create diff --git a/src/rocksdb/iterator.cr b/src/rocksdb/iterator.cr index 26b4281..9b41efe 100644 --- a/src/rocksdb/iterator.cr +++ b/src/rocksdb/iterator.cr @@ -1,44 +1,63 @@ +#abstract class RocksDB::Iterator(T) class RocksDB::Iterator include Api - include Value(LibRocksDB::RocksdbIteratorT) + include RawMemory(LibRocksDB::RocksdbIteratorT) - @read_options : ReadOptions +# protected abstract def zero : T +# protected abstract def underlying(ptr, size) : T + protected def zero + "" + end + + protected def underlying(ptr, size) : String + String.new(ptr, size) + end def initialize(@db : DB, read_options : ReadOptions? = nil) - @read_options = (read_options || @db.read_options).not_nil! - @raw = rocksdb_create_iterator(@db.raw, @read_options.raw) + opts = (read_options || @db.read_options).not_nil! + @raw = rocksdb_create_iterator(@db.raw, opts.raw) @len = Pointer(UInt64).malloc(1_u64) @opened = true end - def first! + def first rocksdb_iter_seek_to_first(raw) end - def last! + def last rocksdb_iter_seek_to_last(raw) end - def next! + def next rocksdb_iter_next(raw) end - - def prev! + + def prev rocksdb_iter_prev(raw) end - + + def seek(key) + rocksdb_iter_seek(raw, key, key.bytesize) + end + def valid? rocksdb_iter_valid(raw) == 1 end - def key? - return nil if !valid? + def key ptr = rocksdb_iter_key(raw, @len) - @len.value == 0 ? nil : String.new(ptr, @len.value) + underlying(ptr, @len.value) end - def key - key? || "" + def value + ptr = rocksdb_iter_value(raw, @len) + underlying(ptr, @len.value) + end + + def key? + return nil if !valid? + ptr = rocksdb_iter_key(raw, @len) + @len.value == 0 ? nil : underlying(ptr, @len.value) end protected def free @@ -49,9 +68,21 @@ class RocksDB::Iterator "iter" end -# fun rocksdb_iter_valid(x0 : RocksdbIteratorT) : UInt8 # fun rocksdb_iter_seek(x0 : RocksdbIteratorT, k : LibC::Char*, klen : LibC::SizeT) -# fun rocksdb_iter_value(x0 : RocksdbIteratorT, vlen : LibC::SizeT*) : LibC::Char* # fun rocksdb_iter_get_error(x0 : RocksdbIteratorT, errptr : LibC::Char**) end + +{% if flag?(:fixed_issue3635) %} +module RocksDB + class StringIterator < Iterator(String) + protected def zero + "" + end + + protected def underlying(ptr, size) : String + String.new(ptr, size) + end + end +end +{% end %} diff --git a/src/rocksdb/types.cr b/src/rocksdb/types.cr index 975f8c8..57efeab 100644 --- a/src/rocksdb/types.cr +++ b/src/rocksdb/types.cr @@ -14,7 +14,7 @@ module RocksDB end end -module RocksDB::Value(T) +module RocksDB::RawMemory(T) @opened : Bool @raw : T?