Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

support db options

  • Loading branch information...
commit 2beabbdb00f2c2455b344197c74f16a6a4695633 1 parent edd22c2
@byplayer byplayer authored committed
Showing with 417 additions and 13 deletions.
  1. +215 −8 ext/leveldb/leveldb.cc
  2. +28 −5 lib/leveldb.rb
  3. +174 −0 test/db_options_test.rb
View
223 ext/leveldb/leveldb.cc
@@ -1,9 +1,13 @@
#include <ruby.h>
+#include <memory>
#include "leveldb/db.h"
+#include "leveldb/cache.h"
#include "leveldb/slice.h"
#include "leveldb/write_batch.h"
+using namespace std;
+
static VALUE m_leveldb;
static VALUE c_db;
static VALUE c_iter;
@@ -20,6 +24,17 @@ static VALUE k_name;
static ID to_s;
static leveldb::ReadOptions uncached_read_options;
+static VALUE c_db_options;
+static VALUE k_create_if_missing;
+static VALUE k_error_if_exists;
+static VALUE k_paranoid_checks;
+static VALUE k_write_buffer_size;
+static VALUE k_block_cache_size;
+static VALUE k_block_size;
+static VALUE k_block_restart_interval;
+static VALUE k_compression;
+static VALUE k_max_open_files;
+
// support 1.9 and 1.8
#ifndef RSTRING_PTR
#define RSTRING_PTR(v) RSTRING(v)->ptr
@@ -45,21 +60,201 @@ static void db_free(bound_db* db) {
delete db;
}
-static VALUE db_make(VALUE klass, VALUE v_pathname, VALUE v_create_if_necessary, VALUE v_break_if_exists) {
+static void set_val(VALUE opts, VALUE key, VALUE db_options, bool* pOptionVal) {
+ VALUE v = rb_hash_aref(opts, key);
+ VALUE set_v;
+ if(NIL_P(v) || v == Qfalse) {
+ *pOptionVal = false;
+ set_v = Qfalse;
+ } else if(v == Qtrue){
+ *pOptionVal = true;
+ set_v = Qtrue;
+ } else {
+ rb_raise(rb_eTypeError, "invalid type for %s", rb_id2name(SYM2ID(key)));
+ }
+
+ string param("@");
+ param += rb_id2name(SYM2ID(key));
+ rb_iv_set(db_options, param.c_str(), set_v);
+}
+
+static void set_val(VALUE opts, VALUE key, VALUE db_options, size_t* pOptionVal) {
+ VALUE v = rb_hash_aref(opts, key);
+ VALUE set_v;
+ if(NIL_P(v)) {
+ set_v = UINT2NUM(*pOptionVal);
+ } else if(FIXNUM_P(v)) {
+ *pOptionVal = NUM2UINT(v);
+ set_v = v;
+ } else {
+ rb_raise(rb_eTypeError, "invalid type for %s", rb_id2name(SYM2ID(key)));
+ }
+
+ string param("@");
+ param += rb_id2name(SYM2ID(key));
+ rb_iv_set(db_options, param.c_str(), set_v);
+}
+
+static void set_val(VALUE opts, VALUE key, VALUE db_options, int* pOptionVal) {
+ VALUE v = rb_hash_aref(opts, key);
+ VALUE set_v;
+ if(NIL_P(v)) {
+ set_v = INT2NUM(*pOptionVal);
+ } else if(FIXNUM_P(v)) {
+ *pOptionVal = NUM2INT(v);
+ set_v = v;
+ } else {
+ rb_raise(rb_eTypeError, "invalid type for %s", rb_id2name(SYM2ID(key)));
+ }
+
+ string param("@");
+ param += rb_id2name(SYM2ID(key));
+ rb_iv_set(db_options, param.c_str(), set_v);
+}
+
+static void set_db_option(VALUE o_options, VALUE opts, leveldb::Options* options) {
+ if(!NIL_P(o_options)) {
+ Check_Type(opts, T_HASH);
+
+ set_val(opts, k_create_if_missing, o_options, &(options->create_if_missing));
+ set_val(opts, k_error_if_exists, o_options, &(options->error_if_exists));
+ set_val(opts, k_paranoid_checks, o_options, &(options->paranoid_checks));
+ set_val(opts, k_write_buffer_size, o_options, &(options->write_buffer_size));
+ set_val(opts, k_max_open_files, o_options, &(options->max_open_files));
+ set_val(opts, k_block_size, o_options, &(options->block_size));
+ set_val(opts, k_block_restart_interval, o_options, &(options->block_restart_interval));
+
+ VALUE v;
+ v = rb_hash_aref(opts, k_block_cache_size);
+ if(!NIL_P(v)) {
+ if(FIXNUM_P(v)) {
+ options->block_cache = leveldb::NewLRUCache(NUM2INT(v));
+ rb_iv_set(o_options, "@block_cache_size", v);
+ } else {
+ rb_raise(rb_eTypeError, "invalid type for %s", rb_id2name(SYM2ID(k_block_cache_size)));
+ }
+ }
+
+ v = rb_hash_aref(opts, k_compression);
+ rb_iv_set(o_options, "@compression", UINT2NUM(options->compression));
+ if(!NIL_P(v)) {
+ if(FIXNUM_P(v)) {
+ switch(NUM2INT(v)) {
+ case leveldb::kNoCompression:
+ options->compression = leveldb::kNoCompression;
+ rb_iv_set(o_options, "@compression", v);
+ break;
+
+ case leveldb::kSnappyCompression:
+ options->compression = leveldb::kSnappyCompression;
+ rb_iv_set(o_options, "@compression", v);
+ break;
+
+ default:
+ rb_raise(rb_eTypeError, "invalid type for %s", rb_id2name(SYM2ID(k_compression)));
+ break;
+ }
+ } else {
+ rb_raise(rb_eTypeError, "invalid type for %s", rb_id2name(SYM2ID(k_compression)));
+ }
+ }
+ }
+}
+
+/*
+ * call-seq:
+ * make(pathname, options)
+ *
+ * open level-db database
+ *
+ * pathname path for database
+ *
+ * [options[ :create_if_missing ]] create if database doesn't exit
+ *
+ * [options[ :error_if_exists ]] raise error if database exists
+ *
+ * [options[ :paranoid_checks ]] If true, the implementation will do aggressive checking of the
+ * data it is processing and will stop early if it detects any
+ * errors. This may have unforeseen ramifications: for example, a
+ * corruption of one DB entry may cause a large number of entries to
+ * become unreadable or for the entire DB to become unopenable.
+ *
+ * Default: false
+ * [options[ :write_buffer_size ]] Amount of data to build up in memory (backed by an unsorted log
+ * on disk) before converting to a sorted on-disk file.
+ *
+ * Larger values increase performance, especially during bulk
+ * loads.
+ * Up to two write buffers may be held in memory at the same time,
+ * so you may wish to adjust this parameter to control memory
+ * usage.
+ * Also, a larger write buffer will result in a longer recovery
+ * time the next time the database is opened.
+ *
+ * Default: 4MB
+ * [options[ :max_open_files ]] Number of open files that can be used by the DB. You may need to
+ * increase this if your database has a large working set (budget
+ * one open file per 2MB of working set).
+ *
+ * Default: 1000
+ * [options[ :block_cache_size ]] Control over blocks (user data is stored in a set of blocks,
+ * and a block is the unit of reading from disk).
+ *
+ * If non nil, use the specified cache size.
+ * If nil, leveldb will automatically create and use an 8MB
+ * internal cache.
+ *
+ * Default: nil
+ * [options[ :block_size ]] Approximate size of user data packed per block. Note that the
+ * block size specified here corresponds to uncompressed data. The
+ * actual size of the unit read from disk may be smaller if
+ * compression is enabled. This parameter can be changed dynamically.
+ *
+ * Default: 4K
+ * [options[ :block_restart_interval ]] Number of keys between restart points for delta
+ * encoding of keys.
+ * This parameter can be changed dynamically.
+ * Most clients should leave this parameter alone.
+ *
+ * Default: 16
+ * [options[ :compression ]] LevelDB::CompressionType::SnappyCompression or
+ * LevelDB::CompressionType::NoCompression.
+ *
+ * Compress blocks using the specified compression algorithm.
+ * This parameter can be changed dynamically.
+ *
+ * Default: LevelDB::CompressionType::SnappyCompression,
+ * which gives lightweight but fast compression.
+ *
+ * Typical speeds of SnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
+ * ~200-500MB/s compression
+ * ~400-800MB/s decompression
+ * Note that these speeds are significantly faster than most
+ * persistent storage speeds, and therefore it is typically never
+ * worth switching to NoCompression. Even if the input data is
+ * incompressible, the SnappyCompression implementation will
+ * efficiently detect that and will switch to uncompressed mode.
+ * [return] LevelDB::DB instance
+ */
+static VALUE db_make(int argc, VALUE* argv, VALUE self) {
+ VALUE v_pathname, v_options;
+ rb_scan_args(argc, argv, "11", &v_pathname, &v_options);
Check_Type(v_pathname, T_STRING);
- bound_db* db = new bound_db;
+ auto_ptr<bound_db> db(new bound_db);
std::string pathname = std::string((char*)RSTRING_PTR(v_pathname));
leveldb::Options options;
- if(RTEST(v_create_if_necessary)) options.create_if_missing = true;
- if(RTEST(v_break_if_exists)) options.error_if_exists = true;
+ VALUE o_options = rb_class_new_instance(0, NULL, c_db_options);
+ set_db_option(o_options, v_options, &options);
+
leveldb::Status status = leveldb::DB::Open(options, pathname, &db->db);
+ VALUE o_db = Data_Wrap_Struct(self, NULL, db_free, db.release());
RAISE_ON_ERROR(status);
- VALUE o_db = Data_Wrap_Struct(klass, NULL, db_free, db);
- VALUE argv[1] = { v_pathname };
- rb_obj_call_init(o_db, 1, argv);
+ rb_iv_set(o_db, "@options", o_options);
+ VALUE init_argv[1] = { v_pathname };
+ rb_obj_call_init(o_db, 1, init_argv);
return o_db;
}
@@ -494,6 +689,16 @@ void Init_leveldb() {
k_reversed = ID2SYM(rb_intern("reversed"));
k_class = rb_intern("class");
k_name = rb_intern("name");
+ k_create_if_missing = ID2SYM(rb_intern("create_if_missing"));
+ k_error_if_exists = ID2SYM(rb_intern("error_if_exists"));
+ k_paranoid_checks = ID2SYM(rb_intern("paranoid_checks"));
+ k_write_buffer_size = ID2SYM(rb_intern("write_buffer_size"));
+ k_block_cache_size = ID2SYM(rb_intern("block_cache_size"));
+ k_block_size = ID2SYM(rb_intern("block_size"));
+ k_block_restart_interval = ID2SYM(rb_intern("block_restart_interval"));
+ k_compression = ID2SYM(rb_intern("compression"));
+ k_max_open_files = ID2SYM(rb_intern("max_open_files"));
+
to_s = rb_intern("to_s");
uncached_read_options = leveldb::ReadOptions();
uncached_read_options.fill_cache = false;
@@ -501,7 +706,7 @@ void Init_leveldb() {
m_leveldb = rb_define_module("LevelDB");
c_db = rb_define_class_under(m_leveldb, "DB", rb_cObject);
- rb_define_singleton_method(c_db, "make", RUBY_METHOD_FUNC(db_make), 3);
+ rb_define_singleton_method(c_db, "make", RUBY_METHOD_FUNC(db_make), -1);
rb_define_method(c_db, "initialize", RUBY_METHOD_FUNC(db_init), 1);
rb_define_method(c_db, "get", RUBY_METHOD_FUNC(db_get), -1);
rb_define_method(c_db, "delete", RUBY_METHOD_FUNC(db_delete), -1);
@@ -525,6 +730,8 @@ void Init_leveldb() {
rb_define_method(c_batch, "put", RUBY_METHOD_FUNC(batch_put), 2);
rb_define_method(c_batch, "delete", RUBY_METHOD_FUNC(batch_delete), 1);
+ c_db_options = rb_define_class_under(m_leveldb, "Options", rb_cObject);
+
c_error = rb_define_class_under(m_leveldb, "Error", rb_eStandardError);
}
}
View
33 lib/leveldb.rb
@@ -6,20 +6,29 @@ class DB
class << self
## Loads or creates a LevelDB database as necessary, stored on disk at
## +pathname+.
- def new pathname
- make path_string(pathname), true, false
+ def new(pathname, options = {})
+ options ||= {}
+
+ make(path_string(pathname),
+ options.merge(:create_if_missing => true,
+ :error_if_exists => false))
end
## Creates a new LevelDB database stored on disk at +pathname+. Throws an
## exception if the database already exists.
- def create pathname
- make path_string(pathname), true, true
+ def create(pathname, options = {})
+ options ||= {}
+
+ make(path_string(pathname),
+ options.merge(:create_if_missing => true,
+ :error_if_exists => true))
end
## Loads a LevelDB database stored on disk at +pathname+. Throws an
## exception unless the database already exists.
def load pathname
- make path_string(pathname), false, false
+ make(path_string(pathname),
+ { :create_if_missing => false, :error_if_exists => false })
end
private
@@ -31,6 +40,7 @@ def path_string pathname
end
attr_reader :pathname
+ attr_reader :options
alias :includes? :exists?
alias :contains? :exists?
@@ -73,4 +83,17 @@ class << self
private :new
end
end
+
+class Options
+ attr_reader :create_if_missing, :error_if_exists,
+ :block_cache_size, :paranoid_checks,
+ :write_buffer_size, :max_open_files,
+ :block_size, :block_restart_interval,
+ :compression
+end
+
+module CompressionType
+ NoCompression = 0x0
+ SnappyCompression = 0x1
+end
end # module LevelDB
View
174 test/db_options_test.rb
@@ -0,0 +1,174 @@
+require 'test/unit'
+require File.expand_path("../../lib/leveldb", __FILE__)
+require 'fileutils'
+
+class DBOptionsTest < Test::Unit::TestCase
+ def setup
+ @path = File.expand_path(File.join('..', 'db_test.db'), __FILE__)
+ end
+
+ def assert_raise_type_error(msg)
+ begin
+ yield
+ flunk("don't raise TypeError")
+ rescue TypeError => e
+ assert_equal(msg, e.to_s)
+ end
+ end
+
+ def test_create_if_missing_default
+ db = LevelDB::DB.make(@path, {})
+ assert_equal db.options.create_if_missing, false
+ end
+
+ def test_create_if_missing
+ db = LevelDB::DB.make(@path, :create_if_missing => true)
+ assert_equal db.options.create_if_missing, true
+ end
+
+ def test_create_if_missing_invalid
+ assert_raise_type_error "invalid type for create_if_missing" do
+ db = LevelDB::DB.make(@path, :create_if_missing => "true")
+ end
+ end
+
+ def test_error_if_exists_default
+ db = LevelDB::DB.make(@path, {})
+ assert_equal db.options.error_if_exists, false
+ end
+
+ def test_error_if_exists
+ FileUtils.rm_rf @path
+ db = LevelDB::DB.make(@path, :error_if_exists => true, :create_if_missing => true)
+ assert_equal db.options.error_if_exists, true
+ end
+
+ def test_error_if_exists_invalid
+ assert_raise_type_error "invalid type for error_if_exists" do
+ LevelDB::DB.make(@path, :error_if_exists => 1)
+ end
+ end
+
+ def test_paranoid_check_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.paranoid_checks, false
+ end
+
+ def test_paranoid_check_on
+ db = LevelDB::DB.new(@path, :paranoid_checks => true)
+ assert_equal db.options.paranoid_checks, true
+ end
+
+ def test_paranoid_check_off
+ db = LevelDB::DB.new(@path, :paranoid_checks => false)
+ assert_equal db.options.paranoid_checks, false
+ end
+
+ def test_paranoid_check_invalid
+ assert_raise_type_error "invalid type for paranoid_checks" do
+ LevelDB::DB.new(@path, :paranoid_checks => "on")
+ end
+ end
+
+ def test_write_buffer_size_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.write_buffer_size, (4 * 1024 * 1024)
+ end
+
+ def test_write_buffer_size
+ db = LevelDB::DB.new(@path, :write_buffer_size => 10 * 1042)
+ assert_equal db.options.write_buffer_size, (10 * 1042)
+ end
+
+ def test_write_buffer_size_raise
+ assert_raise_type_error "invalid type for write_buffer_size" do
+ db = LevelDB::DB.new(@path, :write_buffer_size => "1234")
+ end
+ end
+
+ def test_max_open_files_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.max_open_files, 1000
+ end
+
+ def test_max_open_files
+ db = LevelDB::DB.new(@path, :max_open_files => 2000)
+ assert_equal db.options.max_open_files, 2000
+ end
+
+ def test_max_open_files_invalid
+ assert_raise_type_error "invalid type for max_open_files" do
+ LevelDB::DB.new(@path, :max_open_files => "2000")
+ end
+ end
+
+ def test_cache_size_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.block_cache_size, nil
+ end
+
+ def test_cache_size
+ db = LevelDB::DB.new(@path, :block_cache_size => 10 * 1024 * 1024)
+ assert_equal db.options.block_cache_size, (10 * 1024 * 1024)
+ end
+
+ def test_cache_size_invalid
+ assert_raise_type_error "invalid type for block_cache_size" do
+ db = LevelDB::DB.new(@path, :block_cache_size => false)
+ end
+ end
+
+ def test_block_size_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.block_size, (4 * 1024)
+ end
+
+ def test_block_size
+ db = LevelDB::DB.new(@path, :block_size => (2 * 1024))
+ assert_equal db.options.block_size, (2 * 1024)
+ end
+
+ def test_block_size_invalid
+ assert_raise_type_error "invalid type for block_size" do
+ LevelDB::DB.new(@path, :block_size => true)
+ end
+ end
+
+ def test_block_restart_interval_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.block_restart_interval, 16
+ end
+
+ def test_block_restart_interval
+ db = LevelDB::DB.new(@path, {:block_restart_interval => 32})
+ assert_equal db.options.block_restart_interval, 32
+ end
+
+ def test_block_restart_interval_invalid
+ assert_raise_type_error "invalid type for block_restart_interval" do
+ LevelDB::DB.new(@path, {:block_restart_interval => "abc"})
+ end
+ end
+
+ def test_compression_default
+ db = LevelDB::DB.new(@path)
+ assert_equal db.options.compression, LevelDB::CompressionType::SnappyCompression
+ end
+
+ def test_compression
+ db = LevelDB::DB.new(@path, :compression => LevelDB::CompressionType::NoCompression)
+ assert_equal db.options.compression, LevelDB::CompressionType::NoCompression
+ end
+
+ def test_compression_invalid_type
+ assert_raise_type_error "invalid type for compression" do
+ LevelDB::DB.new(@path, :compression => "1234")
+ end
+ end
+
+ def test_compression_invalid_range
+ assert_raise_type_error "invalid type for compression" do
+ LevelDB::DB.new(@path, :compression => 999)
+ end
+ end
+end

0 comments on commit 2beabbd

Please sign in to comment.
Something went wrong with that request. Please try again.