Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Add snappy support

  • Loading branch information...
commit 5e1a4c578e7caf84e80417b59b50a97dba26f645 1 parent f0b9692
@dizzyd dizzyd authored
View
2  .gitignore
@@ -5,3 +5,5 @@ priv/*
*.beam
ebin
c_src/leveldb/
+c_src/snappy*/
+c_src/system
View
15 c_src/build_deps.sh
@@ -1,6 +1,7 @@
#!/bin/bash
-VSN="c8c5866a86c8d4a3e80d8708d14a06776fb683d1"
+LEVELDB_VSN="c8c5866a86c8d4a3e80d8708d14a06776fb683d1"
+SNAPPY_VSN="1.0.4"
set -e
@@ -12,10 +13,20 @@ BASEDIR="$PWD"
case "$1" in
clean)
- rm -rf leveldb
+ rm -rf leveldb system snappy-$SNAPPY_VSN
;;
*)
+ if [ ! -d snappy-$SNAPPY_VSN ]; then
+ tar -xjf snappy-$SNAPPY_VSN.tar.gz
+ (cd snappy-$SNAPPY_VSN && ./configure --prefix=$BASEDIR/system)
+ fi
+
+ (cd snappy-$SNAPPY_VSN && make && make install)
+
+ export CFLAGS="$CFLAGS -I $BASEDIR/system/include"
+ export LDFLAGS="$LDFLAGS -L $BASEDIR/system/lib"
+
if [ ! -d leveldb ]; then
tar -xjf leveldb.tar.bz2
(cd leveldb && git checkout $VSN)
View
15 c_src/eleveldb.cc
@@ -77,7 +77,8 @@ static ERL_NIF_TERM ATOM_INVALID_ITERATOR;
static ERL_NIF_TERM ATOM_CACHE_SIZE;
static ERL_NIF_TERM ATOM_PARANOID_CHECKS;
static ERL_NIF_TERM ATOM_ERROR_DB_DESTROY;
-static ERL_NIF_TERM ATOM_KEYS_ONLY;
+static ERL_NIF_TERM ATOM_KEYS_ONLY;
+static ERL_NIF_TERM ATOM_COMPRESSION;
static ErlNifFunc nif_funcs[] =
{
@@ -137,6 +138,17 @@ ERL_NIF_TERM parse_open_option(ErlNifEnv* env, ERL_NIF_TERM item, leveldb::Optio
if (cache_sz != 0)
opts.block_cache = leveldb::NewLRUCache(cache_sz);
}
+ else if (option[0] == ATOM_COMPRESSION)
+ {
+ if (option[1] == ATOM_TRUE)
+ {
+ opts.compression = leveldb::kSnappyCompression;
+ }
+ else
+ {
+ opts.compression = leveldb::kNoCompression;
+ }
+ }
}
return ATOM_OK;
@@ -647,6 +659,7 @@ static int on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
ATOM(ATOM_PARANOID_CHECKS, "paranoid_checks");
ATOM(ATOM_ERROR_DB_DESTROY, "error_db_destroy");
ATOM(ATOM_KEYS_ONLY, "keys_only");
+ ATOM(ATOM_COMPRESSION, "compression");
return 0;
}
View
BIN  c_src/snappy-1.0.4.tar.gz
Binary file not shown
View
2  rebar.config
@@ -6,7 +6,7 @@
%% Make sure to set -fPIC when compiling leveldb
{"CFLAGS", "$CFLAGS -fPIC"},
{"DRV_CFLAGS", "$DRV_CFLAGS -Werror -I c_src/leveldb/include"},
- {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/leveldb/libleveldb.a -lstdc++"}
+ {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/leveldb/libleveldb.a c_src/system/lib/libsnappy.a -lstdc++"}
]}.
{pre_hooks, [{compile, "c_src/build_deps.sh"}]}.
View
29 src/eleveldb.erl
@@ -69,8 +69,9 @@ init() ->
{max_open_files, pos_integer()} |
{block_size, pos_integer()} |
{block_restart_interval, pos_integer()} |
- {cache_size, pos_integer()} |
- {paranoid_checks, boolean()}].
+ {cache_size, pos_integer()} |
+ {paranoid_checks, boolean()} |
+ {compression, boolean()}].
-type read_options() :: [{verify_checksums, boolean()} |
{fill_cache, boolean()}].
@@ -231,6 +232,30 @@ destroy_test() ->
ok = ?MODULE:destroy("/tmp/eleveldb.destroy.test", []),
{error, {db_open, _}} = open("/tmp/eleveldb.destroy.test", [{error_if_exists, true}]).
+compression_test() ->
+ CompressibleData = list_to_binary([0 || _X <- lists:seq(1,50000)]),
+ os:cmd("rm -rf /tmp/eleveldb.compress.0 /tmp/eleveldb.compress.1"),
+ {ok, Ref0} = open("/tmp/eleveldb.compress.0", [{write_buffer_size, 5},
+ {create_if_missing, true},
+ {compression, false}]),
+ [ok = ?MODULE:put(Ref0, <<I:64/unsigned>>, CompressibleData, [{sync, true}]) ||
+ I <- lists:seq(1,10)],
+ {ok, Ref1} = open("/tmp/eleveldb.compress.1", [{write_buffer_size, 5},
+ {create_if_missing, true},
+ {compression, true}]),
+ [ok = ?MODULE:put(Ref1, <<I:64/unsigned>>, CompressibleData, [{sync, true}]) ||
+ I <- lists:seq(1,10)],
+ %% Sum up the sizes of the .sst files in each directory -- compressed dir should
+ %% be less than uncompressed
+ Size = fun(Dir) ->
+ lists:sum([element(2, element(2, file:read_file_info(F))) ||
+ F <- filelib:wildcard(filename:join(Dir, "*.sst"))])
+ end,
+ UncompressedSize = Size("/tmp/eleveldb.compress.0"),
+ CompressedSize = Size("/tmp/eleveldb.compress.1"),
+ ?assert(UncompressedSize > CompressedSize).
+
+
-ifdef(EQC).
qc(P) ->
Please sign in to comment.
Something went wrong with that request. Please try again.