Permalink
Browse files

Add configurable file compression (snappy, deflate or none)

Not only this makes database and view index files smaller it also increases
database read/write performance, view index generation (specially for large
documents and/or documents with nested JSON structures) and compaction.
Closes COUCHDB-1120.




git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1098558 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent 5ee419e commit e2e8554d4de18c4ad3765fb105491c77997aff39 @fdmanana fdmanana committed May 2, 2011
Showing with 3,208 additions and 80 deletions.
  1. +7 −0 .gitignore
  2. +30 −0 LICENSE
  3. +8 −0 NOTICE
  4. +64 −0 configure.ac
  5. +8 −0 etc/couchdb/default.ini.tpl.in
  6. +1 −0 license.skip
  7. +1 −1 src/Makefile.am
  8. +2 −0 src/couchdb/Makefile.am
  9. +2 −0 src/couchdb/couch_api_wrap.erl
  10. +16 −13 src/couchdb/couch_btree.erl
  11. +2 −2 src/couchdb/couch_changes.erl
  12. +73 −0 src/couchdb/couch_compress.erl
  13. +41 −9 src/couchdb/couch_db.erl
  14. +6 −2 src/couchdb/couch_db.hrl
  15. +62 −26 src/couchdb/couch_db_updater.erl
  16. +17 −1 src/couchdb/couch_doc.erl
  17. +25 −11 src/couchdb/couch_file.erl
  18. +5 −0 src/couchdb/couch_httpd.erl
  19. +1 −1 src/couchdb/couch_httpd_db.erl
  20. +2 −1 src/couchdb/couch_query_servers.erl
  21. +1 −1 src/couchdb/couch_replication_manager.erl
  22. +1 −1 src/couchdb/couch_replicator.erl
  23. +2 −2 src/couchdb/couch_replicator_utils.erl
  24. +23 −0 src/couchdb/couch_server.erl
  25. +5 −3 src/couchdb/couch_view_group.erl
  26. +74 −0 src/snappy/Makefile.am
  27. +121 −0 src/snappy/erl_nif_compat.h
  28. +1 −0 src/snappy/google-snappy/AUTHORS
  29. +28 −0 src/snappy/google-snappy/COPYING
  30. +130 −0 src/snappy/google-snappy/config.h.in
  31. +150 −0 src/snappy/google-snappy/snappy-internal.h
  32. +72 −0 src/snappy/google-snappy/snappy-sinksource.cc
  33. +136 −0 src/snappy/google-snappy/snappy-sinksource.h
  34. +42 −0 src/snappy/google-snappy/snappy-stubs-internal.cc
  35. +477 −0 src/snappy/google-snappy/snappy-stubs-internal.h
  36. +85 −0 src/snappy/google-snappy/snappy-stubs-public.h.in
  37. +1,015 −0 src/snappy/google-snappy/snappy.cc
  38. +155 −0 src/snappy/google-snappy/snappy.h
  39. +12 −0 src/snappy/snappy.app.in
  40. +57 −0 src/snappy/snappy.erl
  41. +231 −0 src/snappy/snappy_nif.cc
  42. +1 −1 test/etap/010-file-basics.t
  43. +10 −2 test/etap/020-btree-basics.t
  44. +2 −1 test/etap/200-view-group-no-db-leaks.t
  45. +2 −1 test/etap/test_util.erl.in
  46. +2 −1 utils/Makefile.am
View
@@ -69,6 +69,13 @@ src/ejson/.deps/
src/ejson/.libs/
src/ejson/priv
src/mochiweb/mochiweb.app
+src/snappy/.deps/
+src/snappy/.libs/
+src/snappy/priv
+src/snappy/snappy.app
+src/snappy/google-snappy/snappy-stubs-public.h
+src/snappy/google-snappy/stamp-h2
+src/snappy/google-snappy/.deps/
test/local.ini
test/etap/.deps/
test/etap/run
View
30 LICENSE
@@ -448,3 +448,33 @@ For the src/ejson/erl_nif_compat.h file
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
+
+For the src/snappy/google-snappy component
+
+ Copyright 2005 and onwards Google Inc.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
View
8 NOTICE
@@ -57,3 +57,11 @@ This product also includes the following third-party components:
* yajl (http://lloyd.github.com/yajl/)
Copyright 2010, Lloyd Hilaiel
+
+ * snappy (http://code.google.com/p/snappy/)
+
+ Copyright 2005 and onwards Google Inc.
+
+ * snappy-erlang-nif (https://github.com/fdmanana/snappy-erlang-nif)
+
+ Copyright 2011, Filipe David Manana <fdmanana@apache.org>
View
@@ -19,6 +19,7 @@ AC_CONFIG_AUX_DIR([build-aux])
AC_CONFIG_MACRO_DIR([m4])
AM_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([src/snappy/google-snappy/config.h])
AM_INIT_AUTOMAKE([1.6.3 foreign])
@@ -30,6 +31,67 @@ AC_PROG_CC
AC_PROG_LIBTOOL
AC_PROG_LN_S
+dnl Config for google snappy
+m4_define([snappy_major], [1])
+m4_define([snappy_minor], [0])
+m4_define([snappy_patchlevel], [1])
+
+AC_PROG_CXX
+AC_LANG([C++])
+AC_C_BIGENDIAN
+AC_CHECK_HEADERS([stdint.h stddef.h sys/mman.h sys/resource.h])
+AC_CHECK_FUNC([mmap])
+
+AC_MSG_CHECKING([if the compiler supports __builtin_expect])
+
+AC_TRY_COMPILE(, [
+ return __builtin_expect(1, 1) ? 1 : 0
+], [
+ snappy_have_builtin_expect=yes
+ AC_MSG_RESULT([yes])
+], [
+ snappy_have_builtin_expect=no
+ AC_MSG_RESULT([no])
+])
+if test x$snappy_have_builtin_expect = xyes ; then
+ AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.])
+fi
+
+AC_MSG_CHECKING([if the compiler supports __builtin_ctzll])
+
+AC_TRY_COMPILE(, [
+ return (__builtin_ctzll(0x100000000LL) == 32) ? 1 : 0
+], [
+ snappy_have_builtin_ctz=yes
+ AC_MSG_RESULT([yes])
+], [
+ snappy_have_builtin_ctz=no
+ AC_MSG_RESULT([no])
+])
+if test x$snappy_have_builtin_ctz = xyes ; then
+ AC_DEFINE([HAVE_BUILTIN_CTZ], [1], [Define to 1 if the compiler supports __builtin_ctz and friends.])
+fi
+
+if test "$ac_cv_header_stdint_h" = "yes"; then
+ AC_SUBST([ac_cv_have_stdint_h], [1])
+else
+ AC_SUBST([ac_cv_have_stdint_h], [0])
+fi
+if test "$ac_cv_header_stddef_h" = "yes"; then
+ AC_SUBST([ac_cv_have_stddef_h], [1])
+else
+ AC_SUBST([ac_cv_have_stddef_h], [0])
+fi
+
+SNAPPY_MAJOR="snappy_major"
+SNAPPY_MINOR="snappy_minor"
+SNAPPY_PATCHLEVEL="snappy_patchlevel"
+
+AC_SUBST([SNAPPY_MAJOR])
+AC_SUBST([SNAPPY_MINOR])
+AC_SUBST([SNAPPY_PATCHLEVEL])
+dnl End of google snappy specific config
+
AC_MSG_CHECKING([for pthread_create in -lpthread])
original_LIBS="$LIBS"
@@ -423,6 +485,8 @@ AC_CONFIG_FILES([src/erlang-oauth/Makefile])
AC_CONFIG_FILES([src/etap/Makefile])
AC_CONFIG_FILES([src/ibrowse/Makefile])
AC_CONFIG_FILES([src/mochiweb/Makefile])
+AC_CONFIG_FILES([src/snappy/Makefile])
+AC_CONFIG_FILES([src/snappy/google-snappy/snappy-stubs-public.h])
AC_CONFIG_FILES([src/ejson/Makefile])
AC_CONFIG_FILES([test/Makefile])
AC_CONFIG_FILES([test/bench/Makefile])
@@ -11,6 +11,14 @@ os_process_timeout = 5000 ; 5 seconds. for view and external servers.
max_dbs_open = 100
delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned
uri_file = %localstaterundir%/couch.uri
+; Method used to compress everything that is appended to database and view index files, except
+; for attachments (see the attachments section). Available methods are:
+;
+; none - no compression
+; snappy - use google snappy, a very fast compressor/decompressor
+; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest,
+; lowest compression ratio) to 9 (slowest, highest compression ratio)
+file_compression = snappy
[httpd]
port = 5984
View
@@ -88,6 +88,7 @@
^src/etap/*
^src/ibrowse/*
^src/mochiweb/*
+^src/snappy/*
^stamp-h1
^test/Makefile
^test/Makefile.in
View
@@ -10,4 +10,4 @@
## License for the specific language governing permissions and limitations under
## the License.
-SUBDIRS = couchdb ejson erlang-oauth etap ibrowse mochiweb
+SUBDIRS = couchdb ejson erlang-oauth etap ibrowse mochiweb snappy
View
@@ -34,6 +34,7 @@ source_files = \
couch_auth_cache.erl \
couch_btree.erl \
couch_changes.erl \
+ couch_compress.erl \
couch_config.erl \
couch_config_writer.erl \
couch_db.erl \
@@ -102,6 +103,7 @@ compiled_files = \
couch_auth_cache.beam \
couch_btree.beam \
couch_changes.beam \
+ couch_compress.beam \
couch_config.beam \
couch_config_writer.beam \
couch_db.beam \
@@ -420,6 +420,8 @@ options_to_query_args(HttpDb, Path, Options) ->
options_to_query_args([], Acc) ->
lists:reverse(Acc);
+options_to_query_args([ejson_body | Rest], Acc) ->
+ options_to_query_args(Rest, Acc);
options_to_query_args([delay_commit | Rest], Acc) ->
options_to_query_args(Rest, Acc);
options_to_query_args([revs | Rest], Acc) ->
@@ -41,7 +41,9 @@ set_options(Bt, [{join, Assemble}|Rest]) ->
set_options(Bt, [{less, Less}|Rest]) ->
set_options(Bt#btree{less=Less}, Rest);
set_options(Bt, [{reduce, Reduce}|Rest]) ->
- set_options(Bt#btree{reduce=Reduce}, Rest).
+ set_options(Bt#btree{reduce=Reduce}, Rest);
+set_options(Bt, [{compression, Comp}|Rest]) ->
+ set_options(Bt#btree{compression=Comp}, Rest).
open(State, Fd, Options) ->
{ok, set_options(#btree{root=State, fd=Fd}, Options)}.
@@ -274,26 +276,26 @@ complete_root(Bt, KPs) ->
% written. Plus with the "case byte_size(term_to_binary(InList)) of" code
% it's probably really inefficient.
-chunkify(InList) ->
- case byte_size(term_to_binary(InList)) of
+chunkify(#btree{compression = Comp} = Bt, InList) ->
+ case byte_size(couch_compress:compress(InList, Comp)) of
Size when Size > ?CHUNK_THRESHOLD ->
NumberOfChunksLikely = ((Size div ?CHUNK_THRESHOLD) + 1),
ChunkThreshold = Size div NumberOfChunksLikely,
- chunkify(InList, ChunkThreshold, [], 0, []);
+ chunkify(Bt, InList, ChunkThreshold, [], 0, []);
_Else ->
[InList]
end.
-chunkify([], _ChunkThreshold, [], 0, OutputChunks) ->
+chunkify(_Bt, [], _ChunkThreshold, [], 0, OutputChunks) ->
lists:reverse(OutputChunks);
-chunkify([], _ChunkThreshold, OutList, _OutListSize, OutputChunks) ->
+chunkify(_Bt, [], _ChunkThreshold, OutList, _OutListSize, OutputChunks) ->
lists:reverse([lists:reverse(OutList) | OutputChunks]);
-chunkify([InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputChunks) ->
- case byte_size(term_to_binary(InElement)) of
+chunkify(Bt, [InElement | RestInList], ChunkThreshold, OutList, OutListSize, OutputChunks) ->
+ case byte_size(couch_compress:compress(InElement, Bt#btree.compression)) of
Size when (Size + OutListSize) > ChunkThreshold andalso OutList /= [] ->
- chunkify(RestInList, ChunkThreshold, [], 0, [lists:reverse([InElement | OutList]) | OutputChunks]);
+ chunkify(Bt, RestInList, ChunkThreshold, [], 0, [lists:reverse([InElement | OutList]) | OutputChunks]);
Size ->
- chunkify(RestInList, ChunkThreshold, [InElement | OutList], OutListSize + Size, OutputChunks)
+ chunkify(Bt, RestInList, ChunkThreshold, [InElement | OutList], OutListSize + Size, OutputChunks)
end.
modify_node(Bt, RootPointerInfo, Actions, QueryOutput) ->
@@ -346,13 +348,14 @@ get_node(#btree{fd = Fd}, NodePos) ->
{ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos),
{NodeType, NodeList}.
-write_node(Bt, NodeType, NodeList) ->
+write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) ->
% split up nodes into smaller sizes
- NodeListList = chunkify(NodeList),
+ NodeListList = chunkify(Bt, NodeList),
% now write out each chunk and return the KeyPointer pairs for those nodes
ResultList = [
begin
- {ok, Pointer, Size} = couch_file:append_term(Bt#btree.fd, {NodeType, ANodeList}),
+ {ok, Pointer, Size} = couch_file:append_term(
+ Fd, {NodeType, ANodeList}, [{compression, Comp}]),
{LastKey, _} = lists:last(ANodeList),
SubTreeSize = reduce_tree_size(NodeType, Size, ANodeList),
{LastKey, {Pointer, reduce_node(Bt, NodeType, ANodeList), SubTreeSize}}
@@ -105,7 +105,7 @@ os_filter_fun(FilterName, Style, Req, Db) ->
end;
[DName, FName] ->
DesignId = <<"_design/", DName/binary>>,
- DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
+ DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, [ejson_body]),
% validate that the ddoc has the filter fun
#doc{body={Props}} = DDoc,
couch_util:get_nested_json_value({Props}, [<<"filters">>, FName]),
@@ -178,7 +178,7 @@ filter_view(ViewName, Style, Db) ->
throw({bad_request, "Invalid `view` parameter."});
[DName, VName] ->
DesignId = <<"_design/", DName/binary>>,
- DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, []),
+ DDoc = couch_httpd_db:couch_doc_open(Db, DesignId, nil, [ejson_body]),
% validate that the ddoc has the filter fun
#doc{body={Props}} = DDoc,
couch_util:get_nested_json_value({Props}, [<<"views">>, VName]),
@@ -0,0 +1,73 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_compress).
+
+-export([compress/2, decompress/1, is_compressed/1]).
+-export([get_compression_method/0]).
+
+-include("couch_db.hrl").
+
+% binaries compressed with snappy have their first byte set to this value
+-define(SNAPPY_PREFIX, 1).
+% binaries that are a result of an erlang:term_to_binary/1,2 call have this
+% value as their first byte
+-define(TERM_PREFIX, 131).
+
+
+get_compression_method() ->
+ case couch_config:get("couchdb", "file_compression") of
+ undefined ->
+ ?DEFAULT_COMPRESSION;
+ Method1 ->
+ case string:tokens(Method1, "_") of
+ [Method] ->
+ list_to_existing_atom(Method);
+ [Method, Level] ->
+ {list_to_existing_atom(Method), list_to_integer(Level)}
+ end
+ end.
+
+
+compress(Term, none) ->
+ ?term_to_bin(Term);
+compress(Term, {deflate, Level}) ->
+ term_to_binary(Term, [{minor_version, 1}, {compressed, Level}]);
+compress(Term, snappy) ->
+ Bin = ?term_to_bin(Term),
+ try
+ {ok, CompressedBin} = snappy:compress(Bin),
+ case byte_size(CompressedBin) < byte_size(Bin) of
+ true ->
+ <<?SNAPPY_PREFIX, CompressedBin/binary>>;
+ false ->
+ Bin
+ end
+ catch exit:snappy_nif_not_loaded ->
+ Bin
+ end.
+
+
+decompress(<<?SNAPPY_PREFIX, Rest/binary>>) ->
+ {ok, TermBin} = snappy:decompress(Rest),
+ binary_to_term(TermBin);
+decompress(<<?TERM_PREFIX, _/binary>> = Bin) ->
+ binary_to_term(Bin).
+
+
+is_compressed(<<?SNAPPY_PREFIX, _/binary>>) ->
+ true;
+is_compressed(<<?TERM_PREFIX, _/binary>>) ->
+ true;
+is_compressed(Term) when not is_binary(Term) ->
+ false.
+
Oops, something went wrong.

0 comments on commit e2e8554

Please sign in to comment.