Skip to content
Browse files

Merge indexer branch

Conflicts:
	Makefile.am
	include/libcouchstore/couch_db.h
	src/btree_read.c
	src/couch_db.c
	src/couch_save.c
	src/db_compact.c
	src/dbdump.c
	src/internal.h
	src/node_types.h
	tests/testapp.c

Change-Id: I6e154a2b9fe9e815a6a167902f357fbd065e972a
  • Loading branch information...
2 parents cec5c09 + 8db9bae commit 0cd305422a94b835f60f67d03b19004ff5aaba36 @apage43 apage43 committed
View
1 .gitignore
@@ -35,6 +35,7 @@
/couch_compact
/couch_dbdump
/couch_dbinfo
+/couch_viewgen
/couchscript
/docs/doxy
/libcouchstore*.changes
View
24 Makefile.am
@@ -15,11 +15,14 @@
# limitations under the License.
ACLOCAL_AMFLAGS = -I m4 --force
-bin_PROGRAMS = couch_dbdump couch_dbinfo couch_compact
+bin_PROGRAMS = couch_dbdump couch_dbinfo couch_compact couch_viewgen
EXTRA_DIST = python LICENSE README.md
+ICU_LOCAL_LIBS=-licuuc -licudata -licui18n
+
pkginclude_HEADERS = \
include/libcouchstore/couch_db.h \
+ include/libcouchstore/couch_index.h \
include/libcouchstore/couch_common.h \
include/libcouchstore/error.h \
include/libcouchstore/file_ops.h \
@@ -47,6 +50,8 @@ libcouchstore_la_SOURCES = \
src/bitfield.h \
src/btree_modify.c \
src/btree_read.c \
+ src/collate_json.c \
+ src/collate_json.h \
src/couch_btree.h \
src/couch_db.c \
src/couch_save.c \
@@ -59,7 +64,11 @@ libcouchstore_la_SOURCES = \
src/internal.h \
src/iobuffer.c \
src/iobuffer.h \
+ src/json_reduce.c \
+ src/json_reduce.h \
src/llmsort.c \
+ src/tree_writer.c \
+ src/tree_writer.h \
src/mergesort.c \
src/mergesort.h \
src/node_types.c \
@@ -68,7 +77,8 @@ libcouchstore_la_SOURCES = \
src/reduces.h \
src/strerror.c \
src/util.c \
- src/util.h
+ src/util.h \
+ src/couch_index.c
libcouchstore_la_LDFLAGS = $(AM_LDFLAGS) $(ICU_LOCAL_LDFLAGS) -version-info $(LIBCOUCHSTORE_API_CURRENT):$(LIBCOUCHSTORE_API_REVISION):$(LIBCOUCHSTORE_API_AGE) -no-undefined -lsnappy -lpthread
@@ -80,7 +90,7 @@ else
libcouchstore_la_SOURCES += src/os.c
endif
-libcouchstore_la_CFLAGS = $(AM_CFLAGS) $(ICU_LOCAL_CFLAGS) -DLIBCOUCHSTORE_INTERNAL=1 -Wstrict-aliasing=2
+libcouchstore_la_CFLAGS = $(AM_CFLAGS) $(ICU_LOCAL_CFLAGS) -DLIBCOUCHSTORE_INTERNAL=1 -Wstrict-aliasing=2 -pedantic
libcouchstore_la_LIBADD = librfc1321.la libbyteswap.la $(ICU_LOCAL_LIBS)
couch_dbdump_SOURCES = src/dbdump.c
@@ -89,6 +99,7 @@ couch_dbdump_CFLAGS = $(AM_CFLAGS) -D__STDC_FORMAT_MACROS
couch_dbdump_LDADD = libcouchstore.la libbyteswap.la -lsnappy
if WINDOWS
+couch_dbdump_SOURCES += win32/win32.c
couch_dbdump_LDADD += -lws2_32
endif
@@ -102,6 +113,11 @@ couch_compact_DEPENDENCIES = libcouchstore.la
couch_compact_CFLAGS = $(AM_CFLAGS) -D__STDC_FORMAT_MACROS
couch_compact_LDADD = libcouchstore.la libbyteswap.la -lsnappy
+couch_viewgen_SOURCES = src/viewgen.c
+couch_viewgen_DEPENDENCIES = libcouchstore.la
+couch_viewgen_CFLAGS = $(AM_CFLAGS) -D__STDC_FORMAT_MACROS
+couch_viewgen_LDADD = libcouchstore.la libbyteswap.la -lsnappy
+
extra_tests=
slow_tests=
@@ -147,7 +163,7 @@ CLEANFILES = test.couch
check_PROGRAMS = testapp
TESTS = ${check_PROGRAMS}
-testapp_SOURCES = tests/testapp.c src/util.c tests/macros.h
+testapp_SOURCES = tests/testapp.c src/util.c tests/macros.h tests/collate_json_test.c tests/indexer_test.c
testapp_CFLAGS = $(AM_CFLAGS)
testapp_DEPENDENCIES = libcouchstore.la libbyteswap.la
testapp_LDADD = libcouchstore.la libbyteswap.la
View
22 configure.ac
@@ -108,6 +108,28 @@ AS_IF([test "x$ac_cv_header_snappy_c_h" != "xyes"],
AS_IF([test "x${ac_cv_have_libsnappy}" != "xyes"],
[AC_MSG_ERROR(Failed to locate libsnappy)])
+AC_ARG_WITH([win32-icu-binaries], [AC_HELP_STRING([--with-win32-icu-binaries=PATH],
+ [set PATH to the Win32 native ICU binaries directory])], [
+ ICU_CONFIG="" # supposed to be a command to query options...
+ ICU_INCLUDE="$withval/include"
+ ICU_LOCAL_CFLAGS="-I$ICU_INCLUDE"
+ ICU_LIB_DIR="$withval/lib"
+ ICU_LOCAL_LDFLAGS="-L$ICU_LIB_DIR"
+ ICU_LOCAL_BIN=$withval/bin
+ AC_SUBST(ICU_INCLUDE)
+ AC_SUBST(ICU_LIB_DIR)
+], [
+ AC_CHECK_ICU([3.4.1])
+ ICU_LOCAL_CFLAGS=`$ICU_CONFIG --cppflags-searchpath`
+ ICU_LOCAL_LDFLAGS=`$ICU_CONFIG --ldflags-searchpath`
+ ICU_LOCAL_BIN=
+])
+
+AC_SUBST(ICU_CONFIG)
+AC_SUBST(ICU_LOCAL_CFLAGS)
+AC_SUBST(ICU_LOCAL_LDFLAGS)
+AC_SUBST(ICU_LOCAL_BIN)
+
AH_TOP([
#ifndef CONFIG_H
#define CONFIG_H
View
201 couchstore.xcodeproj/project.pbxproj
@@ -25,6 +25,7 @@
/* End PBXAggregateTarget section */
/* Begin PBXBuildFile section */
+ 270770E615A221090058085F /* tree_writer.c in Sources */ = {isa = PBXBuildFile; fileRef = 270770E515A221090058085F /* tree_writer.c */; };
27093EF61537441A0087804E /* iobuffer.c in Sources */ = {isa = PBXBuildFile; fileRef = 27093EF51537441A0087804E /* iobuffer.c */; };
274F97BD152E1D1400247C46 /* bitfield.h in Headers */ = {isa = PBXBuildFile; fileRef = 274F9782152E1D1400247C46 /* bitfield.h */; };
274F97BE152E1D1400247C46 /* btree_modify.c in Sources */ = {isa = PBXBuildFile; fileRef = 274F9783152E1D1400247C46 /* btree_modify.c */; };
@@ -60,15 +61,27 @@
2794AD0E153395940047643C /* testapp.c in Sources */ = {isa = PBXBuildFile; fileRef = 2794ACFF15338E640047643C /* testapp.c */; };
27A3A93215A3612800E1BDBC /* macros.h in Headers */ = {isa = PBXBuildFile; fileRef = 27A3A93115A3612800E1BDBC /* macros.h */; };
27AA1B101538DE0700D03755 /* libcouchstore.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 274F9758152E1CCA00247C46 /* libcouchstore.dylib */; };
- 27AA1B171538DE2500D03755 /* couchscript.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274F978F152E1D1400247C46 /* couchscript.cc */; };
27AA1B191538DE3900D03755 /* liblua.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 27AA1B181538DE3900D03755 /* liblua.dylib */; };
27AA1B1D1538EC4300D03755 /* arena.c in Sources */ = {isa = PBXBuildFile; fileRef = 27AA1B1C1538EC4300D03755 /* arena.c */; };
+ 27B07E77159E75BD0084E054 /* libcouchstore.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 274F9758152E1CCA00247C46 /* libcouchstore.dylib */; };
+ 27B07E80159E75EE0084E054 /* viewgen.c in Sources */ = {isa = PBXBuildFile; fileRef = 27B07E7F159E75EE0084E054 /* viewgen.c */; };
+ 27D3316215DEC20500CDD3C2 /* node_types.c in Sources */ = {isa = PBXBuildFile; fileRef = 27D3316015DEC20500CDD3C2 /* node_types.c */; };
+ 27D3316315DEC20500CDD3C2 /* node_types.h in Headers */ = {isa = PBXBuildFile; fileRef = 27D3316115DEC20500CDD3C2 /* node_types.h */; };
27D37EC71547292B00D3F0C8 /* libcouchstore.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 274F9758152E1CCA00247C46 /* libcouchstore.dylib */; };
27D37ED3154729F400D3F0C8 /* db_compact.c in Sources */ = {isa = PBXBuildFile; fileRef = 27D37ECF154729F400D3F0C8 /* db_compact.c */; };
27D37ED4154729F400D3F0C8 /* llmsort.c in Sources */ = {isa = PBXBuildFile; fileRef = 27D37ED0154729F400D3F0C8 /* llmsort.c */; };
27D37ED6154729F400D3F0C8 /* mergesort.c in Sources */ = {isa = PBXBuildFile; fileRef = 27D37ED2154729F400D3F0C8 /* mergesort.c */; };
27D37ED815472A3300D3F0C8 /* compactor.c in Sources */ = {isa = PBXBuildFile; fileRef = 27D37ED715472A3300D3F0C8 /* compactor.c */; };
27E65A001544F2F0000A7658 /* couch_save.c in Sources */ = {isa = PBXBuildFile; fileRef = 27E659FF1544F2F0000A7658 /* couch_save.c */; };
+ 27F00F1215B9D0AA008CDA68 /* couch_index.c in Sources */ = {isa = PBXBuildFile; fileRef = 27F00F1115B9D0AA008CDA68 /* couch_index.c */; };
+ 27F08CD315AB8FBC003C3E2B /* collate_json.c in Sources */ = {isa = PBXBuildFile; fileRef = 27F08CD215AB8FBC003C3E2B /* collate_json.c */; };
+ 27F08CD715ABA25A003C3E2B /* collate_json_test.c in Sources */ = {isa = PBXBuildFile; fileRef = 27F08CD615ABA25A003C3E2B /* collate_json_test.c */; };
+ 27F08CDF15ABAF8C003C3E2B /* libicui18n.44.1.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 27F08CDE15ABAF8C003C3E2B /* libicui18n.44.1.dylib */; };
+ 27F08CE015ABAFA9003C3E2B /* libicuuc.44.1.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 27F08CDA15ABAF55003C3E2B /* libicuuc.44.1.dylib */; };
+ 27F28E9615C861AD007A4B95 /* couchscript.cc in Sources */ = {isa = PBXBuildFile; fileRef = 274F978F152E1D1400247C46 /* couchscript.cc */; };
+ 27F28EED15C9B5AC007A4B95 /* indexer_test.c in Sources */ = {isa = PBXBuildFile; fileRef = 27F28EEC15C9B5AC007A4B95 /* indexer_test.c */; };
+ 27F28F1115C9F7B1007A4B95 /* json_reduce.h in Headers */ = {isa = PBXBuildFile; fileRef = 27F28F1015C9F7B1007A4B95 /* json_reduce.h */; };
+ 27F28F1415C9F92F007A4B95 /* json_reduce.c in Sources */ = {isa = PBXBuildFile; fileRef = 27F28F1315C9F92F007A4B95 /* json_reduce.c */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@@ -146,6 +159,15 @@
);
runOnlyForDeploymentPostprocessing = 1;
};
+ 27B07E78159E75BD0084E054 /* CopyFiles */ = {
+ isa = PBXCopyFilesBuildPhase;
+ buildActionMask = 2147483647;
+ dstPath = /usr/share/man/man1/;
+ dstSubfolderSpec = 0;
+ files = (
+ );
+ runOnlyForDeploymentPostprocessing = 1;
+ };
27D37EC81547292B00D3F0C8 /* CopyFiles */ = {
isa = PBXCopyFilesBuildPhase;
buildActionMask = 2147483647;
@@ -158,6 +180,8 @@
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFileReference section */
+ 270770E515A221090058085F /* tree_writer.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = tree_writer.c; sourceTree = "<group>"; };
+ 270770E815A2211F0058085F /* tree_writer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tree_writer.h; sourceTree = "<group>"; };
27093EF51537441A0087804E /* iobuffer.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = iobuffer.c; sourceTree = "<group>"; };
27093EF81537442E0087804E /* iobuffer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = iobuffer.h; sourceTree = "<group>"; };
274F9758152E1CCA00247C46 /* libcouchstore.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libcouchstore.dylib; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -192,8 +216,8 @@
274F97E2152E1D5100247C46 /* visibility.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = visibility.h; sourceTree = "<group>"; };
274F97E3152E1D9200247C46 /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = SOURCE_ROOT; };
274F97E5152E1E3E00247C46 /* libsnappy.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libsnappy.dylib; path = /usr/local/lib/libsnappy.dylib; sourceTree = "<absolute>"; };
- 274F97ED152E22E500247C46 /* dbdump */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = dbdump; sourceTree = BUILT_PRODUCTS_DIR; };
- 274F9804152E235600247C46 /* dbinfo */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = dbinfo; sourceTree = BUILT_PRODUCTS_DIR; };
+ 274F97ED152E22E500247C46 /* couch_dbdump */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = couch_dbdump; sourceTree = BUILT_PRODUCTS_DIR; };
+ 274F9804152E235600247C46 /* couch_dbinfo */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = couch_dbinfo; sourceTree = BUILT_PRODUCTS_DIR; };
2794ACFF15338E640047643C /* testapp.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; name = testapp.c; path = tests/testapp.c; sourceTree = SOURCE_ROOT; };
2794AD011533901D0047643C /* config_static.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = config_static.h; sourceTree = SOURCE_ROOT; };
2794AD0C153395780047643C /* testapp */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = testapp; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -202,12 +226,27 @@
27AA1B181538DE3900D03755 /* liblua.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = liblua.dylib; path = usr/local/lib/liblua.dylib; sourceTree = SDKROOT; };
27AA1B1C1538EC4300D03755 /* arena.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = arena.c; sourceTree = "<group>"; };
27AA1B1E1538EC5600D03755 /* arena.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = arena.h; sourceTree = "<group>"; };
+ 27B07E7C159E75BD0084E054 /* couch_viewgen */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = couch_viewgen; sourceTree = BUILT_PRODUCTS_DIR; };
+ 27B07E7F159E75EE0084E054 /* viewgen.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = viewgen.c; sourceTree = "<group>"; };
+ 27D3316015DEC20500CDD3C2 /* node_types.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = node_types.c; sourceTree = "<group>"; };
+ 27D3316115DEC20500CDD3C2 /* node_types.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = node_types.h; sourceTree = "<group>"; };
27D37ECC1547292B00D3F0C8 /* couch_compact */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = couch_compact; sourceTree = BUILT_PRODUCTS_DIR; };
27D37ECF154729F400D3F0C8 /* db_compact.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = db_compact.c; sourceTree = "<group>"; };
27D37ED0154729F400D3F0C8 /* llmsort.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = llmsort.c; sourceTree = "<group>"; };
27D37ED2154729F400D3F0C8 /* mergesort.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mergesort.c; sourceTree = "<group>"; };
27D37ED715472A3300D3F0C8 /* compactor.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = compactor.c; sourceTree = "<group>"; };
27E659FF1544F2F0000A7658 /* couch_save.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = couch_save.c; sourceTree = "<group>"; };
+ 27F00F0F15B9CFA6008CDA68 /* view_format.md */ = {isa = PBXFileReference; lastKnownFileType = text; path = view_format.md; sourceTree = SOURCE_ROOT; };
+ 27F00F1115B9D0AA008CDA68 /* couch_index.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = couch_index.c; sourceTree = "<group>"; };
+ 27F08CD215AB8FBC003C3E2B /* collate_json.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = collate_json.c; sourceTree = "<group>"; };
+ 27F08CD515AB8FFE003C3E2B /* collate_json.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = collate_json.h; sourceTree = "<group>"; };
+ 27F08CD615ABA25A003C3E2B /* collate_json_test.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = collate_json_test.c; path = ../tests/collate_json_test.c; sourceTree = "<group>"; };
+ 27F08CDA15ABAF55003C3E2B /* libicuuc.44.1.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libicuuc.44.1.dylib; path = /usr/local/Cellar/icu4c/4.4.1/lib/libicuuc.44.1.dylib; sourceTree = "<absolute>"; };
+ 27F08CDE15ABAF8C003C3E2B /* libicui18n.44.1.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libicui18n.44.1.dylib; path = /usr/local/Cellar/icu4c/4.4.1/lib/libicui18n.44.1.dylib; sourceTree = "<absolute>"; };
+ 27F28EC415C88867007A4B95 /* couch_index.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = couch_index.h; sourceTree = "<group>"; };
+ 27F28EEC15C9B5AC007A4B95 /* indexer_test.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = indexer_test.c; path = ../tests/indexer_test.c; sourceTree = "<group>"; };
+ 27F28F1015C9F7B1007A4B95 /* json_reduce.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = json_reduce.h; sourceTree = "<group>"; };
+ 27F28F1315C9F92F007A4B95 /* json_reduce.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = json_reduce.c; sourceTree = "<group>"; };
D960D28F1547399900DDDF1E /* mergesort.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = mergesort.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
@@ -216,6 +255,8 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
+ 27F08CE015ABAFA9003C3E2B /* libicuuc.44.1.dylib in Frameworks */,
+ 27F08CDF15ABAF8C003C3E2B /* libicui18n.44.1.dylib in Frameworks */,
274F97E6152E1E3E00247C46 /* libsnappy.dylib in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
@@ -256,6 +297,14 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
+ 27B07E76159E75BD0084E054 /* Frameworks */ = {
+ isa = PBXFrameworksBuildPhase;
+ buildActionMask = 2147483647;
+ files = (
+ 27B07E77159E75BD0084E054 /* libcouchstore.dylib in Frameworks */,
+ );
+ runOnlyForDeploymentPostprocessing = 0;
+ };
27D37EC51547292B00D3F0C8 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
@@ -282,11 +331,12 @@
isa = PBXGroup;
children = (
274F9758152E1CCA00247C46 /* libcouchstore.dylib */,
- 274F97ED152E22E500247C46 /* dbdump */,
- 274F9804152E235600247C46 /* dbinfo */,
+ 274F97ED152E22E500247C46 /* couch_dbdump */,
+ 274F9804152E235600247C46 /* couch_dbinfo */,
2794AD0C153395780047643C /* testapp */,
27AA1B151538DE0700D03755 /* couchscript */,
27D37ECC1547292B00D3F0C8 /* couch_compact */,
+ 27B07E7C159E75BD0084E054 /* couch_viewgen */,
);
name = Products;
sourceTree = "<group>";
@@ -306,14 +356,12 @@
274F978D152E1D1400247C46 /* couch_file_write.c */,
274F9790152E1D1400247C46 /* crc32.c */,
274F9791152E1D1400247C46 /* crc32.h */,
- 27D37ECF154729F400D3F0C8 /* db_compact.c */,
274F9794152E1D1400247C46 /* fatbuf.h */,
274F9795152E1D1400247C46 /* internal.h */,
+ 27D3316015DEC20500CDD3C2 /* node_types.c */,
+ 27D3316115DEC20500CDD3C2 /* node_types.h */,
27093EF51537441A0087804E /* iobuffer.c */,
27093EF81537442E0087804E /* iobuffer.h */,
- 27D37ED0154729F400D3F0C8 /* llmsort.c */,
- 27D37ED2154729F400D3F0C8 /* mergesort.c */,
- D960D28F1547399900DDDF1E /* mergesort.h */,
274F97A0152E1D1400247C46 /* os.c */,
274F97A1152E1D1400247C46 /* reduces.c */,
274F97A2152E1D1400247C46 /* reduces.h */,
@@ -323,6 +371,8 @@
274F97B1152E1D1400247C46 /* util.h */,
27AA1B1C1538EC4300D03755 /* arena.c */,
27AA1B1E1538EC5600D03755 /* arena.h */,
+ 27F00F1515B9E1A9008CDA68 /* compactor */,
+ 27F00F1415B9E189008CDA68 /* view indexer */,
);
path = src;
sourceTree = "<group>";
@@ -351,6 +401,7 @@
children = (
274F97DE152E1D5100247C46 /* couch_common.h */,
274F97DF152E1D5100247C46 /* couch_db.h */,
+ 27F28EC415C88867007A4B95 /* couch_index.h */,
274F97E0152E1D5100247C46 /* error.h */,
274F97E1152E1D5100247C46 /* file_ops.h */,
274F97E2152E1D5100247C46 /* visibility.h */,
@@ -365,8 +416,11 @@
274F9792152E1D1400247C46 /* dbdump.c */,
274F9793152E1D1400247C46 /* dbinfo.c */,
2794ACFF15338E640047643C /* testapp.c */,
+ 27F08CD615ABA25A003C3E2B /* collate_json_test.c */,
+ 27F28EEC15C9B5AC007A4B95 /* indexer_test.c */,
274F978F152E1D1400247C46 /* couchscript.cc */,
27A3A93115A3612800E1BDBC /* macros.h */,
+ 27B07E7F159E75EE0084E054 /* viewgen.c */,
);
name = tools;
path = src;
@@ -376,11 +430,39 @@
isa = PBXGroup;
children = (
274F97E5152E1E3E00247C46 /* libsnappy.dylib */,
+ 27F08CDE15ABAF8C003C3E2B /* libicui18n.44.1.dylib */,
+ 27F08CDA15ABAF55003C3E2B /* libicuuc.44.1.dylib */,
27AA1B181538DE3900D03755 /* liblua.dylib */,
);
name = Frameworks;
sourceTree = "<group>";
};
+ 27F00F1415B9E189008CDA68 /* view indexer */ = {
+ isa = PBXGroup;
+ children = (
+ 27F08CD215AB8FBC003C3E2B /* collate_json.c */,
+ 27F08CD515AB8FFE003C3E2B /* collate_json.h */,
+ 27F00F1115B9D0AA008CDA68 /* couch_index.c */,
+ 27F28F1315C9F92F007A4B95 /* json_reduce.c */,
+ 27F28F1015C9F7B1007A4B95 /* json_reduce.h */,
+ 27F00F0F15B9CFA6008CDA68 /* view_format.md */,
+ );
+ name = "view indexer";
+ sourceTree = "<group>";
+ };
+ 27F00F1515B9E1A9008CDA68 /* compactor */ = {
+ isa = PBXGroup;
+ children = (
+ 27D37ECF154729F400D3F0C8 /* db_compact.c */,
+ 27D37ED0154729F400D3F0C8 /* llmsort.c */,
+ 27D37ED2154729F400D3F0C8 /* mergesort.c */,
+ D960D28F1547399900DDDF1E /* mergesort.h */,
+ 270770E515A221090058085F /* tree_writer.c */,
+ 270770E815A2211F0058085F /* tree_writer.h */,
+ );
+ name = compactor;
+ sourceTree = "<group>";
+ };
/* End PBXGroup section */
/* Begin PBXHeadersBuildPhase section */
@@ -399,6 +481,8 @@
274F97DA152E1D1400247C46 /* util.h in Headers */,
274F97E4152E1D9200247C46 /* config.h in Headers */,
27A3A93215A3612800E1BDBC /* macros.h in Headers */,
+ 27F28F1115C9F7B1007A4B95 /* json_reduce.h in Headers */,
+ 27D3316315DEC20500CDD3C2 /* node_types.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -436,7 +520,7 @@
);
name = dbdump;
productName = dbdump;
- productReference = 274F97ED152E22E500247C46 /* dbdump */;
+ productReference = 274F97ED152E22E500247C46 /* couch_dbdump */;
productType = "com.apple.product-type.tool";
};
274F97FA152E235600247C46 /* dbinfo */ = {
@@ -453,7 +537,7 @@
);
name = dbinfo;
productName = dbdump;
- productReference = 274F9804152E235600247C46 /* dbinfo */;
+ productReference = 274F9804152E235600247C46 /* couch_dbinfo */;
productType = "com.apple.product-type.tool";
};
2794AD02153395780047643C /* testapp */ = {
@@ -490,6 +574,23 @@
productReference = 27AA1B151538DE0700D03755 /* couchscript */;
productType = "com.apple.product-type.tool";
};
+ 27B07E73159E75BD0084E054 /* viewgen */ = {
+ isa = PBXNativeTarget;
+ buildConfigurationList = 27B07E79159E75BD0084E054 /* Build configuration list for PBXNativeTarget "viewgen" */;
+ buildPhases = (
+ 27B07E74159E75BD0084E054 /* Sources */,
+ 27B07E76159E75BD0084E054 /* Frameworks */,
+ 27B07E78159E75BD0084E054 /* CopyFiles */,
+ );
+ buildRules = (
+ );
+ dependencies = (
+ );
+ name = viewgen;
+ productName = dbdump;
+ productReference = 27B07E7C159E75BD0084E054 /* couch_viewgen */;
+ productType = "com.apple.product-type.tool";
+ };
27D37EC21547292B00D3F0C8 /* couch_compact */ = {
isa = PBXNativeTarget;
buildConfigurationList = 27D37EC91547292B00D3F0C8 /* Build configuration list for PBXNativeTarget "couch_compact" */;
@@ -513,7 +614,7 @@
274F974F152E1CCA00247C46 /* Project object */ = {
isa = PBXProject;
attributes = {
- LastUpgradeCheck = 0430;
+ LastUpgradeCheck = 0440;
ORGANIZATIONNAME = "Couchbase, Inc.";
};
buildConfigurationList = 274F9752152E1CCA00247C46 /* Build configuration list for PBXProject "couchstore" */;
@@ -534,6 +635,7 @@
274F97FA152E235600247C46 /* dbinfo */,
2794AD02153395780047643C /* testapp */,
27AA1B0B1538DE0700D03755 /* couchscript */,
+ 27B07E73159E75BD0084E054 /* viewgen */,
27F0C71E15339B34004EBA89 /* All */,
);
};
@@ -562,6 +664,11 @@
27D37ED3154729F400D3F0C8 /* db_compact.c in Sources */,
27D37ED4154729F400D3F0C8 /* llmsort.c in Sources */,
27D37ED6154729F400D3F0C8 /* mergesort.c in Sources */,
+ 270770E615A221090058085F /* tree_writer.c in Sources */,
+ 27F08CD315AB8FBC003C3E2B /* collate_json.c in Sources */,
+ 27F00F1215B9D0AA008CDA68 /* couch_index.c in Sources */,
+ 27F28F1415C9F92F007A4B95 /* json_reduce.c in Sources */,
+ 27D3316215DEC20500CDD3C2 /* node_types.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -586,6 +693,8 @@
buildActionMask = 2147483647;
files = (
2794AD0E153395940047643C /* testapp.c in Sources */,
+ 27F08CD715ABA25A003C3E2B /* collate_json_test.c in Sources */,
+ 27F28EED15C9B5AC007A4B95 /* indexer_test.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -593,7 +702,15 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
- 27AA1B171538DE2500D03755 /* couchscript.cc in Sources */,
+ 27F28E9615C861AD007A4B95 /* couchscript.cc in Sources */,
+ );
+ runOnlyForDeploymentPostprocessing = 0;
+ };
+ 27B07E74159E75BD0084E054 /* Sources */ = {
+ isa = PBXSourcesBuildPhase;
+ buildActionMask = 2147483647;
+ files = (
+ 27B07E80159E75EE0084E054 /* viewgen.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -651,6 +768,7 @@
"DEBUG=1",
"$(inherited)",
);
+ GCC_STRICT_ALIASING = YES;
GCC_SYMBOLS_PRIVATE_EXTERN = NO;
GCC_TREAT_WARNINGS_AS_ERRORS = YES;
GCC_VERSION = com.apple.compilers.llvm.clang.1_0;
@@ -670,7 +788,10 @@
MACOSX_DEPLOYMENT_TARGET = 10.7;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = macosx;
- WARNING_CFLAGS = "-Wall";
+ WARNING_CFLAGS = (
+ "-Wstrict-aliasing=2",
+ "-Wall",
+ );
};
name = Debug;
};
@@ -684,6 +805,7 @@
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
GCC_C_LANGUAGE_STANDARD = c99;
GCC_ENABLE_OBJC_EXCEPTIONS = YES;
+ GCC_STRICT_ALIASING = YES;
GCC_TREAT_WARNINGS_AS_ERRORS = YES;
GCC_VERSION = com.apple.compilers.llvm.clang.1_0;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
@@ -701,14 +823,22 @@
LIBRARY_SEARCH_PATHS = /usr/local/lib;
MACOSX_DEPLOYMENT_TARGET = 10.7;
SDKROOT = macosx;
- WARNING_CFLAGS = "-Wall";
+ WARNING_CFLAGS = (
+ "-Wstrict-aliasing=2",
+ "-Wall",
+ );
};
name = Release;
};
274F975D152E1CCA00247C46 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
+ COMBINE_HIDPI_IMAGES = YES;
EXECUTABLE_PREFIX = lib;
+ LIBRARY_SEARCH_PATHS = (
+ "$(inherited)",
+ /usr/local/Cellar/icu4c/4.4.1/lib,
+ );
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Debug;
@@ -716,7 +846,12 @@
274F975E152E1CCA00247C46 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
+ COMBINE_HIDPI_IMAGES = YES;
EXECUTABLE_PREFIX = lib;
+ LIBRARY_SEARCH_PATHS = (
+ "$(inherited)",
+ /usr/local/Cellar/icu4c/4.4.1/lib,
+ );
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Release;
@@ -725,7 +860,7 @@
isa = XCBuildConfiguration;
buildSettings = {
GCC_C_LANGUAGE_STANDARD = gnu99;
- PRODUCT_NAME = "$(TARGET_NAME)";
+ PRODUCT_NAME = couch_dbdump;
};
name = Debug;
};
@@ -733,7 +868,7 @@
isa = XCBuildConfiguration;
buildSettings = {
GCC_C_LANGUAGE_STANDARD = gnu99;
- PRODUCT_NAME = "$(TARGET_NAME)";
+ PRODUCT_NAME = couch_dbdump;
};
name = Release;
};
@@ -741,7 +876,7 @@
isa = XCBuildConfiguration;
buildSettings = {
GCC_C_LANGUAGE_STANDARD = gnu99;
- PRODUCT_NAME = dbinfo;
+ PRODUCT_NAME = couch_dbinfo;
};
name = Debug;
};
@@ -749,7 +884,7 @@
isa = XCBuildConfiguration;
buildSettings = {
GCC_C_LANGUAGE_STANDARD = gnu99;
- PRODUCT_NAME = dbinfo;
+ PRODUCT_NAME = couch_dbinfo;
};
name = Release;
};
@@ -785,6 +920,23 @@
};
name = Release;
};
+ 27B07E7A159E75BD0084E054 /* Debug */ = {
+ isa = XCBuildConfiguration;
+ buildSettings = {
+ GCC_C_LANGUAGE_STANDARD = gnu99;
+ PRODUCT_NAME = couch_viewgen;
+ };
+ name = Debug;
+ };
+ 27B07E7B159E75BD0084E054 /* Release */ = {
+ isa = XCBuildConfiguration;
+ buildSettings = {
+ GCC_C_LANGUAGE_STANDARD = gnu99;
+ ONLY_ACTIVE_ARCH = YES;
+ PRODUCT_NAME = couch_viewgen;
+ };
+ name = Release;
+ };
27D37ECA1547292B00D3F0C8 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
@@ -805,6 +957,7 @@
27F0C72015339B34004EBA89 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
+ COMBINE_HIDPI_IMAGES = YES;
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Debug;
@@ -812,6 +965,7 @@
27F0C72115339B34004EBA89 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
+ COMBINE_HIDPI_IMAGES = YES;
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Release;
@@ -873,6 +1027,15 @@
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
+ 27B07E79159E75BD0084E054 /* Build configuration list for PBXNativeTarget "viewgen" */ = {
+ isa = XCConfigurationList;
+ buildConfigurations = (
+ 27B07E7A159E75BD0084E054 /* Debug */,
+ 27B07E7B159E75BD0084E054 /* Release */,
+ );
+ defaultConfigurationIsVisible = 0;
+ defaultConfigurationName = Release;
+ };
27D37EC91547292B00D3F0C8 /* Build configuration list for PBXNativeTarget "couch_compact" */ = {
isa = XCConfigurationList;
buildConfigurations = (
View
25 include/libcouchstore/couch_db.h
@@ -280,6 +280,22 @@ extern "C" {
/**
+ * Allocates a new DocInfo structure on the heap, plus optionally its id and rev_meta.
+ * If the id or rev_meta are given, their values will be copied into the allocated memory
+ * and the corresponding fields in the returned DocInfo will point there. Otherwise the
+ * DocInfo's id and/or rev_meta fields will be empty/null.
+ * @param id the document ID to copy into the DocInfo, or NULL to leave its ID NULL.
+ * @param rev_meta the revision metadata to copy into the DocInfo, or NULL to leave its
+ * rev_meta NULL.
+ * @return the allocated DocInfo, or NULL on an allocation failure. Must be freed by
+ * calling couchstore_free_docinfo.
+ */
+ LIBCOUCHSTORE_API
+ DocInfo* couchstore_alloc_docinfo(const sized_buf *id,
+ const sized_buf *rev_meta);
+
+
+ /**
* Free all allocated resources from a docinfo structure returned by
* couchstore_docinfo_by_id() or passed to a couchstore_changes_callback_fn.
*
@@ -536,7 +552,10 @@ extern "C" {
LIBCOUCHSTORE_API
void couchstore_free_local_document(LocalDoc *lDoc);
- /*
+
+ /*//////////////////// UTILITIES: */
+
+ /**
* Compact a database. This creates a new DB file with the same data as the
* source db, omitting data that is no longer needed.
* Will use default couch_file_ops to create and write the target db.
@@ -560,7 +579,7 @@ extern "C" {
COUCHSTORE_COMPACT_FLAG_DROP_DELETES = 1
};
- /*
+ /**
* Compact a database. This creates a new DB file with the same data as the
* source db, omitting data that is no longer needed.
* Will use specified couch_file_ops to create and write the target db.
@@ -575,6 +594,8 @@ extern "C" {
LIBCOUCHSTORE_API
couchstore_error_t couchstore_compact_db_ex(Db* source, const char* target_filename,
uint64_t flags, const couch_file_ops *ops);
+
+
/*//////////////////// MISC: */
/**
View
91 include/libcouchstore/couch_index.h
@@ -0,0 +1,91 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+#ifndef COUCHSTORE_COUCH_INDEX_H
+#define COUCHSTORE_COUCH_INDEX_H
+
+#include <libcouchstore/couch_db.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ /**
+ * Opaque reference to an open index file.
+ */
+ typedef struct _CouchStoreIndex CouchStoreIndex;
+
+ /*
+ * Types of indexes.
+ */
+ typedef uint64_t couchstore_index_type;
+ enum {
+ COUCHSTORE_VIEW_PRIMARY_INDEX = 0, /**< Primary index, maps emitted keys to values */
+ COUCHSTORE_VIEW_BACK_INDEX = 1, /**< Back-index, maps doc IDs to emitted keys */
+ };
+
+ /*
+ * Available built-in reduce functions to use on the JSON values in primary indexes.
+ * These are documented at <http://wiki.apache.org/couchdb/Built-In_Reduce_Functions>
+ */
+ typedef uint64_t couchstore_json_reducer;
+ enum {
+ COUCHSTORE_REDUCE_NONE = 0, /**< No reduction */
+ COUCHSTORE_REDUCE_COUNT = 1, /**< Count rows */
+ COUCHSTORE_REDUCE_SUM = 2, /**< Sum numeric values */
+ COUCHSTORE_REDUCE_STATS = 3, /**< Compute count, min, max, sum, sum of squares */
+ };
+
+
+ /**
+ * Create a new index file.
+ *
+ * The file should be closed with couchstore_close_index().
+ *
+ * @param filename The name of the file containing the index. Any existing file at this path
+ * will be deleted.
+ * @param index Pointer to where you want the handle to the index to be
+ * stored.
+ * @return COUCHSTORE_SUCCESS for success
+ */
+ LIBCOUCHSTORE_API
+ couchstore_error_t couchstore_create_index(const char *filename,
+ CouchStoreIndex** index);
+
+ /**
+ * Close an open index file.
+ *
+ * @param index Pointer to the index handle to free.
+ * @return COUCHSTORE_SUCCESS upon success
+ */
+ LIBCOUCHSTORE_API
+ couchstore_error_t couchstore_close_index(CouchStoreIndex* index);
+
+ /**
+ * Read an unsorted key-value file and add its contents to an index file.
+ * Each file added will create a new independent index within the file; they are not merged.
+ *
+ * The key-value file is a sequence of zero or more records, each of which consists of:
+ * key length (16 bits, big-endian)
+ * value length (32 bits, big-endian)
+ * key data
+ * value data
+ * The data formats inside the actual keys and values differ with the index types, and are
+ * documented in "The Binary (Termless) Format for Views" (view_format.md).
+ *
+ * @param inputPath The path to the key-value file
+ * @param index_type The type of index keys/values in the input file, and the type of index
+ * to generate in the index file. Note: CouchStore can currently add only one back index.
+ * @param reduce_function The type of JSON reduce function to apply to the data. Valid only
+ * for primary indexes; ignored in back-indexes.
+ * @param index The index file to write to
+ * @return COUCHSTORE_SUCCESS on success, else an error code
+ */
+ LIBCOUCHSTORE_API
+ couchstore_error_t couchstore_index_add(const char *inputPath,
+ couchstore_index_type index_type,
+ couchstore_json_reducer reduce_function,
+ CouchStoreIndex* index);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
View
74 m4/ac_check_icu.m4
@@ -0,0 +1,74 @@
+# ===========================================================================
+# http://autoconf-archive.cryp.to/ac_check_icu.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AC_CHECK_ICU(version, action-if, action-if-not)
+#
+# DESCRIPTION
+#
+# Defines ICU_LIBS, ICU_CFLAGS, ICU_CXXFLAGS. See icu-config(1) man page.
+#
+# LAST MODIFICATION
+#
+# 2008-04-12
+#
+# COPYLEFT
+#
+# Copyright (c) 2008 Akos Maroy <darkeye@tyrell.hu>
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved.
+
+AC_DEFUN([AC_CHECK_ICU], [
+ succeeded=no
+
+ if test -z "$ICU_CONFIG"; then
+ AC_PATH_PROG(ICU_CONFIG, icu-config, no)
+ fi
+
+ if test "$ICU_CONFIG" = "no" ; then
+ echo "*** The icu-config script could not be found. Make sure it is"
+ echo "*** in your path, and that taglib is properly installed."
+ echo "*** Or see http://ibm.com/software/globalization/icu/"
+ else
+ ICU_VERSION=`$ICU_CONFIG --version`
+ AC_MSG_CHECKING(for ICU >= $1)
+ VERSION_CHECK=`expr $ICU_VERSION \>\= $1`
+ if test "$VERSION_CHECK" = "1" ; then
+ AC_MSG_RESULT(yes)
+ succeeded=yes
+
+ AC_MSG_CHECKING(ICU_CFLAGS)
+ ICU_CFLAGS=`$ICU_CONFIG --cflags`
+ AC_MSG_RESULT($ICU_CFLAGS)
+
+ AC_MSG_CHECKING(ICU_CXXFLAGS)
+ ICU_CXXFLAGS=`$ICU_CONFIG --cxxflags`
+ AC_MSG_RESULT($ICU_CXXFLAGS)
+
+ AC_MSG_CHECKING(ICU_LIBS)
+ ICU_LIBS=`$ICU_CONFIG --ldflags`
+ AC_MSG_RESULT($ICU_LIBS)
+ else
+ ICU_CFLAGS=""
+ ICU_CXXFLAGS=""
+ ICU_LIBS=""
+ ## If we have a custom action on failure, don't print errors, but
+ ## do set a variable so people can do so.
+ ifelse([$3], ,echo "can't find ICU >= $1",)
+ fi
+
+ AC_SUBST(ICU_CFLAGS)
+ AC_SUBST(ICU_CXXFLAGS)
+ AC_SUBST(ICU_LIBS)
+ fi
+
+ if test $succeeded = yes; then
+ ifelse([$2], , :, [$2])
+ else
+ ifelse([$3], , AC_MSG_ERROR([Library requirements (ICU) not met.]), [$3])
+ fi
+])
View
5 src/arena.c
@@ -173,3 +173,8 @@ void arena_free_from_mark(arena *a, const arena_position *mark)
// TODO: Somehow roll back blocks_allocated and bytes_allocated (how?)
#endif
}
+
+void arena_free_all(arena *a)
+{
+ arena_free_from_mark(a, NULL);
+}
View
6 src/arena.h
@@ -70,4 +70,10 @@ const arena_position* arena_mark(arena *a);
*/
void arena_free_from_mark(arena *a, const arena_position *mark);
+/**
+ * Frees all blocks from the arena.
+ */
+void arena_free_all(arena *a);
+
+
#endif // COUCH_ARENA_H
View
15 src/btree_modify.c
@@ -1,5 +1,6 @@
/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include "config.h"
+#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
@@ -72,12 +73,12 @@ static couchfile_modify_result *make_modres(arena* a, couchfile_modify_request *
return res;
}
-couchfile_modify_result *new_btree_modres(arena *a, arena *transient_arena, Db* db, compare_info* cmp, reduce_fn reduce,
- reduce_fn rereduce)
+couchfile_modify_result *new_btree_modres(arena *a, arena *transient_arena, tree_file *file,
+ compare_info* cmp, reduce_fn reduce, reduce_fn rereduce)
{
couchfile_modify_request* rq = arena_alloc(a, sizeof(couchfile_modify_request));
rq->cmp = *cmp;
- rq->db = db;
+ rq->file = file;
rq->num_actions = 0;
rq->fetch_callback = NULL;
rq->reduce = reduce;
@@ -179,7 +180,7 @@ static couchstore_error_t flush_mr_partial(couchfile_modify_result *res, size_t
int itmcount = 0;
char *nodebuf = NULL;
sized_buf writebuf;
- char reducebuf[30];
+ char reducebuf[500]; // view_indexer.c's reduce fns need this much room
size_t reducesize = 0;
uint64_t subtreesize = 0;
off_t diskpos;
@@ -219,7 +220,7 @@ static couchstore_error_t flush_mr_partial(couchfile_modify_result *res, size_t
writebuf.size = dst - nodebuf;
- errcode = db_write_buf_compressed(res->rq->db, &writebuf, &diskpos, &disk_size);
+ errcode = db_write_buf_compressed(res->rq->file, &writebuf, &diskpos, &disk_size);
free(nodebuf); // here endeth the nodebuf.
if (errcode != COUCHSTORE_SUCCESS) {
return errcode;
@@ -227,10 +228,12 @@ static couchstore_error_t flush_mr_partial(couchfile_modify_result *res, size_t
if (res->node_type == KV_NODE && res->rq->reduce) {
res->rq->reduce(reducebuf, &reducesize, res->values->next, itmcount);
+ assert(reducesize <= sizeof(reducebuf));
}
if (res->node_type == KP_NODE && res->rq->rereduce) {
res->rq->rereduce(reducebuf, &reducesize, res->values->next, itmcount);
+ assert(reducesize <= sizeof(reducebuf));
}
node_pointer *ptr = (node_pointer *) arena_alloc(res->arena, sizeof(node_pointer) + final_key.size + reducesize);
@@ -314,7 +317,7 @@ static couchstore_error_t modify_node(couchfile_modify_request *rq,
}
if (nptr) {
- if ((nodebuflen = pread_compressed(rq->db, nptr->pointer, (char **) &nodebuf)) < 0) {
+ if ((nodebuflen = pread_compressed(rq->file, nptr->pointer, (char **) &nodebuf)) < 0) {
error_pass(COUCHSTORE_ERROR_READ);
}
}
View
2 src/btree_read.c
@@ -19,7 +19,7 @@ static couchstore_error_t btree_lookup_inner(couchfile_lookup_request *rq,
char *nodebuf = NULL;
- nodebuflen = pread_compressed(rq->db, diskpos, &nodebuf);
+ nodebuflen = pread_compressed(rq->file, diskpos, &nodebuf);
error_unless(nodebuflen >= 0, nodebuflen); // if negative, it's an error code
if (nodebuf[0] == 0) { //KP Node
View
341 src/collate_json.c
@@ -0,0 +1,341 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+
+// Reference: http://wiki.apache.org/couchdb/View_collation
+
+#include "config.h"
+#include "collate_json.h"
+#include <assert.h>
+#include <ctype.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unicode/ucol.h>
+#include <unicode/ucasemap.h>
+
+
+static int cmp(int n1, int n2)
+{
+ return n1 > n2 ? 1 : (n1 < n2 ? -1 : 0);
+}
+
+static int dcmp(double n1, double n2)
+{
+ return n1 > n2 ? 1 : (n1 < n2 ? -1 : 0);
+}
+
+
+// Types of values, ordered according to CouchDB collation order (see view_collation.js tests)
+typedef enum {
+ kEndArray,
+ kEndObject,
+ kComma,
+ kColon,
+ kNull,
+ kFalse,
+ kTrue,
+ kNumber,
+ kString,
+ kArray,
+ kObject,
+ kIllegal
+} ValueType;
+
+
+// "Raw" ordering is: 0:number, 1:false, 2:null, 3:true, 4:object, 5:array, 6:string
+// (according to view_collation_raw.js)
+static int8_t kRawOrderOfValueType[] = {
+ -4, -3, -2, -1,
+ 2, 1, 3, 0, 6, 5, 4,
+ 7
+};
+
+
+static ValueType valueTypeOf(char c)
+{
+ switch (c) {
+ case 'n': return kNull;
+ case 'f': return kFalse;
+ case 't': return kTrue;
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ case '-': return kNumber;
+ case '"': return kString;
+ case ']': return kEndArray;
+ case '}': return kEndObject;
+ case ',': return kComma;
+ case ':': return kColon;
+ case '[': return kArray;
+ case '{': return kObject;
+ default:
+ fprintf(stderr, "CouchStore CollateJSON: Unexpected character '%c' (0x%02x)\n", c, (unsigned char)c);
+ return kIllegal;
+ }
+}
+
+
+static int digitToInt(int c)
+{
+ if (isdigit(c))
+ return c - '0';
+ else if (isxdigit(c))
+ return 10 + tolower(c) - 'a';
+ else
+ return 0;
+}
+
+
+char ConvertJSONEscape(const char **in)
+{
+ char c = *++(*in);
+ switch (c) {
+ case 'u': {
+ // \u is a Unicode escape; 4 hex digits follow.
+ const char* digits = *in + 1;
+ *in += 4;
+ int uc = (digitToInt(digits[0]) << 12) | (digitToInt(digits[1]) << 8) |
+ (digitToInt(digits[2]) << 4) | (digitToInt(digits[3]));
+ if (uc > 127)
+ fprintf(stderr, "CouchStore CollateJSON: Can't correctly compare \\u%.4s\n", digits);
+ return (char)uc;
+ }
+ case 'b': return '\b';
+ case 'n': return '\n';
+ case 'r': return '\r';
+ case 't': return '\t';
+ default: return c;
+ }
+}
+
+
+static int compareStringsASCII(const char** in1, const char** in2)
+{
+ const char* str1 = *in1, *str2 = *in2;
+ while(true) {
+ char c1 = *++str1;
+ char c2 = *++str2;
+
+ // If one string ends, the other is greater; if both end, they're equal:
+ if (c1 == '"') {
+ if (c2 == '"')
+ break;
+ else
+ return -1;
+ } else if (c2 == '"')
+ return 1;
+
+ // Handle escape sequences:
+ if (c1 == '\\')
+ c1 = ConvertJSONEscape(&str1);
+ if (c2 == '\\')
+ c2 = ConvertJSONEscape(&str2);
+
+ // Compare the next characters:
+ int s = cmp(c1, c2);
+ if (s)
+ return s;
+ }
+
+ // Strings are equal, so update the positions:
+ *in1 = str1 + 1;
+ *in2 = str2 + 1;
+ return 0;
+}
+
+
+static const char* createStringFromJSON(const char** in, size_t *length, bool *freeWhenDone)
+{
+ // Scan the JSON string to find its length and whether it contains escapes:
+ const char* start = ++*in;
+ unsigned escapes = 0;
+ const char* str;
+ for (str = start; *str != '"'; ++str) {
+ if (*str == '\\') {
+ ++str;
+ if (*str == 'u') {
+ escapes += 5; // \uxxxx adds 5 bytes
+ str += 4;
+ } else
+ escapes += 1;
+ }
+ }
+ *in = str + 1;
+ *length = str - start;
+
+ *freeWhenDone = false;
+ if (escapes > 0) {
+ *length -= escapes;
+ char* buf = malloc(*length);
+ char* dst = buf;
+ char c;
+ for (str = start; (c = *str) != '"'; ++str) {
+ if (c == '\\')
+ c = ConvertJSONEscape(&str);
+ *dst++ = c;
+ }
+ assert(dst - buf == (int)*length);
+ start = buf;
+ *freeWhenDone = true;
+ }
+
+ return start;
+}
+
+
+static int compareUnicode(const char* str1, size_t len1,
+ const char* str2, size_t len2)
+{
+ static UCollator* coll = NULL;
+
+ UErrorCode status = U_ZERO_ERROR;
+ if (!coll) {
+ coll = ucol_open("", &status);
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "CouchStore CollateJSON: Couldn't initialize ICU (%d)\n", (int)status);
+ return -1;
+ }
+ }
+
+ UCharIterator iterA, iterB;
+ int result;
+
+ uiter_setUTF8(&iterA, str1, (int)len1);
+ uiter_setUTF8(&iterB, str2, (int)len2);
+
+ result = ucol_strcollIter(coll, &iterA, &iterB, &status);
+
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "CouchStore CollateJSON: ICU error %d\n", (int)status);
+ return -1;
+ }
+
+ if (result < 0) {
+ return -1;
+ } else if (result > 0) {
+ return 1;
+ }
+
+ return 0;
+}
+
+
+static int compareStringsUnicode(const char** in1, const char** in2)
+{
+ size_t len1, len2;
+ bool free1, free2;
+ const char* str1 = createStringFromJSON(in1, &len1, &free1);
+ const char* str2 = createStringFromJSON(in2, &len2, &free2);
+
+ int result = compareUnicode(str1, len1, str2, len2);
+
+ if (free1) {
+ free((char*)str1);
+ }
+ if (free2) {
+ free((char*)str2);
+ }
+ return result;
+}
+
+
+static double readNumber(const char* start, const char* end, char** endOfNumber) {
+ assert(end > start);
+ // First copy the string into a zero-terminated buffer so we can safely call strtod:
+ size_t len = end - start;
+ char buf[50];
+ char* str = (len < sizeof(buf)) ? buf : malloc(len + 1);
+ if (!str)
+ return 0.0;
+ memcpy(str, start, len);
+ str[len] = '\0';
+
+ char* endInStr;
+ double result = strtod(str, &endInStr);
+ *endOfNumber = (char*)start + (endInStr - str);
+ if (len >= sizeof(buf))
+ free(str);
+ return result;
+}
+
+
+int CollateJSON(sized_buf buf1,
+ sized_buf buf2,
+ CollateJSONMode mode)
+{
+ const char* str1 = buf1.buf;
+ const char* str2 = buf2.buf;
+ int depth = 0;
+
+ do {
+ // Get the types of the next token in each string:
+ ValueType type1 = valueTypeOf(*str1);
+ ValueType type2 = valueTypeOf(*str2);
+ // If types don't match, stop and return their relative ordering:
+ if (type1 != type2) {
+ if (mode != kCollateJSON_Raw)
+ return cmp(type1, type2);
+ else
+ return cmp(kRawOrderOfValueType[type1], kRawOrderOfValueType[type2]);
+
+ // If types match, compare the actual token values:
+ } else switch (type1) {
+ case kNull:
+ case kTrue:
+ str1 += 4;
+ str2 += 4;
+ break;
+ case kFalse:
+ str1 += 5;
+ str2 += 5;
+ break;
+ case kNumber: {
+ char* next1, *next2;
+ int diff;
+ if (depth == 0) {
+ // At depth 0, be careful not to fall off the end of the input, because there
+ // won't be any delimiters (']' or '}') after the number!
+ diff = dcmp( readNumber(str1, buf1.buf + buf1.size, &next1),
+ readNumber(str2, buf2.buf + buf2.size, &next2) );
+ } else {
+ diff = dcmp( strtod(str1, &next1), strtod(str2, &next2) );
+ }
+ if (diff)
+ return diff; // Numbers don't match
+ str1 = next1;
+ str2 = next2;
+ break;
+ }
+ case kString: {
+ int diff;
+ if (mode == kCollateJSON_Unicode)
+ diff = compareStringsUnicode(&str1, &str2);
+ else
+ diff = compareStringsASCII(&str1, &str2);
+ if (diff)
+ return diff; // Strings don't match
+ break;
+ }
+ case kArray:
+ case kObject:
+ ++str1;
+ ++str2;
+ ++depth;
+ break;
+ case kEndArray:
+ case kEndObject:
+ ++str1;
+ ++str2;
+ --depth;
+ break;
+ case kComma:
+ case kColon:
+ ++str1;
+ ++str2;
+ break;
+ case kIllegal:
+ return 0;
+ }
+ } while (depth > 0); // Keep going as long as we're inside an array or object
+ return 0;
+}
View
38 src/collate_json.h
@@ -0,0 +1,38 @@
+//
+// collate_json.h
+// couchstore
+//
+// Created by Jens Alfke on 7/9/12.
+// Copyright (c) 2012 Couchbase, Inc. All rights reserved.
+//
+
+#ifndef COUCH_COLLATE_JSON_H
+#define COUCH_COLLATE_JSON_H
+
+#include <libcouchstore/couch_common.h>
+#include <libcouchstore/visibility.h>
+#include <stdlib.h>
+
+
+typedef enum CollateJSONMode {
+ kCollateJSON_Unicode, //< Compare strings as Unicode (CouchDB's default)
+ kCollateJSON_Raw, //< CouchDB's "raw" collation rules
+ kCollateJSON_ASCII //< Like Unicode except strings are compared as binary UTF-8
+} CollateJSONMode;
+
+
+/**
+ * Compares two UTF-8 JSON strings using CouchDB's collation rules.
+ * CAREFUL: The two strings must be valid JSON, with no extraneous whitespace,
+ * otherwise this function will return wrong results or even crash.
+ */
+LIBCOUCHSTORE_API
+int CollateJSON(sized_buf buf1,
+ sized_buf buf2,
+ CollateJSONMode mode);
+
+// not part of the API -- exposed for testing only (see collate_json_test.c)
+LIBCOUCHSTORE_API
+char ConvertJSONEscape(const char **in);
+
+#endif
View
13 src/couch_btree.h
@@ -8,13 +8,15 @@
extern "C" {
#endif
+ typedef int (*compare_callback)(const sized_buf *k1, const sized_buf *k2);
+
typedef struct compare_info {
/* used by find_first_gteq */
int last_cmp_val;
sized_buf *last_cmp_key;
int list_pos;
/* Compare function */
- int (*compare)(const sized_buf *k1, const sized_buf *k2);
+ compare_callback compare;
void *arg;
} compare_info;
@@ -23,7 +25,7 @@ extern "C" {
typedef struct couchfile_lookup_request {
compare_info cmp;
- Db *db;
+ tree_file *file;
int num_keys;
/* If nonzero, calls fetch_callback for all keys between and including key 0 and key 1
in the keys array, or all keys after key 0 if it contains only one key.
@@ -68,7 +70,7 @@ extern "C" {
typedef struct couchfile_modify_request {
compare_info cmp;
- Db *db;
+ tree_file *file;
int num_actions;
couchfile_modify_action *actions;
void (*fetch_callback) (struct couchfile_modify_request *rq, sized_buf *k, sized_buf *v, void *arg);
@@ -108,8 +110,9 @@ extern "C" {
couchstore_error_t mr_push_item(sized_buf *k, sized_buf *v, couchfile_modify_result *dst);
- couchfile_modify_result* new_btree_modres(arena* a, arena* transient_arena, Db* db, compare_info* cmp, reduce_fn reduce,
- reduce_fn rereduce);
+ couchfile_modify_result* new_btree_modres(arena* a, arena* transient_arena, tree_file *file,
+ compare_info* cmp, reduce_fn reduce,
+ reduce_fn rereduce);
node_pointer* complete_new_btree(couchfile_modify_result* mr, couchstore_error_t *errcode);
View
170 src/couch_db.c
@@ -12,7 +12,6 @@
#include "bitfield.h"
#include "reduces.h"
#include "util.h"
-#include "iobuffer.h"
#define ROOT_BASE_SIZE 12
#define HEADER_BASE_SIZE 25
@@ -40,7 +39,7 @@ static couchstore_error_t find_header_at_pos(Db *db, off_t pos)
int errcode = COUCHSTORE_SUCCESS;
raw_file_header *header_buf = NULL;
uint8_t buf[2];
- ssize_t readsize = db->file_ops->pread(db->file_handle, buf, 2, pos);
+ ssize_t readsize = db->file.ops->pread(db->file.handle, buf, 2, pos);
error_unless(readsize == 2, COUCHSTORE_ERROR_READ);
if (buf[0] == 0) {
return COUCHSTORE_ERROR_NO_HEADER;
@@ -48,7 +47,7 @@ static couchstore_error_t find_header_at_pos(Db *db, off_t pos)
return COUCHSTORE_ERROR_CORRUPT;
}
- int header_len = pread_header(db, pos, (char**)&header_buf);
+ int header_len = pread_header(&db->file, pos, (char**)&header_buf);
if (header_len < 0) {
error_pass(header_len);
}
@@ -83,7 +82,7 @@ static couchstore_error_t find_header_at_pos(Db *db, off_t pos)
static couchstore_error_t find_header(Db *db)
{
couchstore_error_t last_header_errcode = COUCHSTORE_ERROR_NO_HEADER;
- int64_t pos = db->file_pos - 2;
+ int64_t pos = db->file.pos - 2;
pos -= pos % COUCH_BLOCK_SIZE;
for (; pos >= 0; pos -= COUCH_BLOCK_SIZE) {
couchstore_error_t errcode = find_header_at_pos(db, pos);
@@ -136,7 +135,7 @@ static couchstore_error_t write_header(Db *db)
root += idrootsize;
encode_root(root, db->header.local_docs_root);
off_t pos;
- couchstore_error_t errcode = db_write_header(db, &writebuf, &pos);
+ couchstore_error_t errcode = db_write_header(&db->file, &writebuf, &pos);
if (errcode == COUCHSTORE_SUCCESS) {
db->header.position = pos;
}
@@ -166,7 +165,7 @@ uint64_t couchstore_get_header_position(Db *db)
LIBCOUCHSTORE_API
couchstore_error_t couchstore_commit(Db *db)
{
- off_t curpos = db->file_pos;
+ off_t curpos = db->file.pos;
sized_buf zerobyte = {"\0", 1};
size_t seqrootsize = 0, idrootsize = 0, localrootsize = 0;
if (db->header.by_seq_root) {
@@ -178,20 +177,20 @@ couchstore_error_t couchstore_commit(Db *db)
if (db->header.local_docs_root) {
localrootsize = 12 + db->header.local_docs_root->reduce_value.size;
}
- db->file_pos += 25 + seqrootsize + idrootsize + localrootsize;
+ db->file.pos += 25 + seqrootsize + idrootsize + localrootsize;
//Extend file size to where end of header will land before we do first sync
- db_write_buf(db, &zerobyte, NULL, NULL);
+ db_write_buf(&db->file, &zerobyte, NULL, NULL);
- couchstore_error_t errcode = db->file_ops->sync(db->file_handle);
+ couchstore_error_t errcode = db->file.ops->sync(db->file.handle);
//Set the pos back to where it was when we started to write the real header.
- db->file_pos = curpos;
+ db->file.pos = curpos;
if (errcode == COUCHSTORE_SUCCESS) {
errcode = write_header(db);
}
if (errcode == COUCHSTORE_SUCCESS) {
- errcode = db->file_ops->sync(db->file_handle);
+ errcode = db->file.ops->sync(db->file.handle);
}
return errcode;
@@ -217,13 +216,8 @@ couchstore_error_t couchstore_open_db_ex(const char *filename,
int openflags;
/* Sanity check input parameters */
- if (filename == NULL || pDb == NULL || ops == NULL ||
- ops->version != 3 || ops->constructor == NULL || ops->open == NULL ||
- ops->close == NULL || ops->pread == NULL ||
- ops->pwrite == NULL || ops->goto_eof == NULL ||
- ops->sync == NULL || ops->destructor == NULL ||
- ((flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
- (flags & COUCHSTORE_OPEN_FLAG_CREATE))) {
+ if ((flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
+ (flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
}
@@ -240,16 +234,10 @@ couchstore_error_t couchstore_open_db_ex(const char *filename,
if (flags & COUCHSTORE_OPEN_FLAG_CREATE) {
openflags |= O_CREAT;
}
+
+ error_pass(tree_file_open(&db->file, filename, openflags, ops));
- db->filename = strdup(filename);
- error_unless(db->filename, COUCHSTORE_ERROR_ALLOC_FAIL);
-
- db->file_ops = couch_get_buffered_file_ops(ops, &db->file_handle);
- error_unless(db->file_ops, COUCHSTORE_ERROR_ALLOC_FAIL);
-
- error_pass(db->file_ops->open(&db->file_handle, filename, openflags));
-
- if ((db->file_pos = db->file_ops->goto_eof(db->file_handle)) == 0) {
+ if ((db->file.pos = db->file.ops->goto_eof(db->file.handle)) == 0) {
/* This is an empty file. Create a new fileheader unless the
* user wanted a read-only version of the file
*/
@@ -273,15 +261,7 @@ couchstore_error_t couchstore_open_db_ex(const char *filename,
LIBCOUCHSTORE_API
couchstore_error_t couchstore_close_db(Db *db)
{
- if(db == NULL) {
- return COUCHSTORE_SUCCESS;
- }
-
- if (db->file_ops) {
- db->file_ops->close(db->file_handle);
- db->file_ops->destructor(db->file_handle);
- }
- free((char*)db->filename);
+ tree_file_close(&db->file);
free(db->header.by_id_root);
free(db->header.by_seq_root);
@@ -295,10 +275,53 @@ couchstore_error_t couchstore_close_db(Db *db)
LIBCOUCHSTORE_API
const char* couchstore_get_db_filename(Db *db) {
- return db->filename;
+ return db->file.path;
}
-static int by_seq_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
+DocInfo* couchstore_alloc_docinfo(const sized_buf *id, const sized_buf *rev_meta) {
+ size_t size = sizeof(DocInfo);
+ if (id) {
+ size += id->size;
+ }
+ if (rev_meta) {
+ size += rev_meta->size;
+ }
+ DocInfo* docInfo = malloc(size);
+ if (!docInfo) {
+ return NULL;
+ }
+ memset(docInfo, 0, sizeof(DocInfo));
+ char *extra = (char *)docInfo + sizeof(DocInfo);
+ if (id) {
+ memcpy(extra, id->buf, id->size);
+ docInfo->id.buf = extra;
+ docInfo->id.size = id->size;
+ extra += id->size;
+ }
+ if (rev_meta) {
+ memcpy(extra, rev_meta->buf, rev_meta->size);
+ docInfo->rev_meta.buf = extra;
+ docInfo->rev_meta.size = rev_meta->size;
+ }
+ return docInfo;
+}
+
+LIBCOUCHSTORE_API
+void couchstore_free_docinfo(DocInfo *docinfo)
+{
+ free(docinfo);
+}
+
+LIBCOUCHSTORE_API
+void couchstore_free_document(Doc *doc)
+{
+ if (doc) {
+ char *offset = (char *) (&((fatbuf *) NULL)->buf);
+ fatbuf_free((fatbuf *) ((char *)doc - (char *)offset));
+ }
+}
+
+static couchstore_error_t by_seq_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
{
const raw_seq_index_value *raw = (const raw_seq_index_value*)v->buf;
ssize_t extraSize = v->size - sizeof(*raw);
@@ -315,27 +338,24 @@ static int by_seq_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
uint64_t rev_seq = decode_raw48(raw->rev_seq);
uint64_t db_seq = decode_sequence_key(k);
- DocInfo* docInfo = malloc(sizeof(DocInfo) + extraSize);
+ sized_buf id = {v->buf + sizeof(*raw), idsize};
+ sized_buf rev_meta = {id.buf + idsize, extraSize - id.size};
+ DocInfo* docInfo = couchstore_alloc_docinfo(&id, &rev_meta);
if (!docInfo) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- char *rbuf = (char *) docInfo;
- memcpy(rbuf + sizeof(DocInfo), v->buf + sizeof(*raw), extraSize);
- *pInfo = docInfo;
+
docInfo->db_seq = db_seq;
docInfo->rev_seq = rev_seq;
docInfo->deleted = deleted;
docInfo->bp = bp;
docInfo->size = datasize;
docInfo->content_meta = content_meta;
- docInfo->id.buf = rbuf + sizeof(DocInfo);
- docInfo->id.size = idsize;
- docInfo->rev_meta.buf = rbuf + sizeof(DocInfo) + idsize;
- docInfo->rev_meta.size = extraSize - idsize;
- return 0;
+ *pInfo = docInfo;
+ return COUCHSTORE_SUCCESS;
}
-static int by_id_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
+static couchstore_error_t by_id_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
{
const raw_id_index_value *raw = (const raw_id_index_value*)v->buf;
ssize_t revMetaSize = v->size - sizeof(*raw);
@@ -355,25 +375,20 @@ static int by_id_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
content_meta = decode_raw08(raw->content_meta);
revnum = decode_raw48(raw->rev_seq);
- DocInfo* docInfo = malloc(sizeof(DocInfo) + revMetaSize + k->size);
+ sized_buf rev_meta = {v->buf + sizeof(*raw), revMetaSize};
+ DocInfo* docInfo = couchstore_alloc_docinfo(k, &rev_meta);
if (!docInfo) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- char *rbuf = (char *) docInfo;
- memcpy(rbuf + sizeof(DocInfo), v->buf + sizeof(*raw), revMetaSize);
- *pInfo = docInfo;
+
docInfo->db_seq = seq;
docInfo->rev_seq = revnum;
docInfo->deleted = deleted;
docInfo->bp = bp;
docInfo->size = datasize;
docInfo->content_meta = content_meta;
- docInfo->rev_meta.buf = rbuf + sizeof(DocInfo);
- docInfo->rev_meta.size = revMetaSize;
- docInfo->id.buf = docInfo->rev_meta.buf + docInfo->rev_meta.size;
- docInfo->id.size = k->size;
- memcpy(docInfo->id.buf, k->buf, k->size);
- return 0;
+ *pInfo = docInfo;
+ return COUCHSTORE_SUCCESS;
}
//Fill in doc from reading file.
@@ -385,9 +400,9 @@ static couchstore_error_t bp_to_doc(Doc **pDoc, Db *db, off_t bp, couchstore_ope
fatbuf *docbuf = NULL;
if (options & DECOMPRESS_DOC_BODIES) {
- bodylen = pread_compressed(db, bp, &docbody);
+ bodylen = pread_compressed(&db->file, bp, &docbody);
} else {
- bodylen = pread_bin(db, bp, &docbody);
+ bodylen = pread_bin(&db->file, bp, &docbody);
}
error_unless(bodylen >= 0, bodylen); // if bodylen is negative it's an error code
@@ -459,7 +474,7 @@ couchstore_error_t couchstore_docinfo_by_id(Db *db,
rq.cmp.compare = ebin_cmp;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = 1;
rq.keys = &keylist;
rq.callback_ctx = pInfo;
@@ -497,7 +512,7 @@ couchstore_error_t couchstore_docinfo_by_sequence(Db *db,
rq.cmp.compare = seq_cmp;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = 1;
rq.keys = &keylist;
rq.callback_ctx = pInfo;
@@ -656,7 +671,7 @@ couchstore_error_t couchstore_changes_since(Db *db,
rq.cmp.compare = seq_cmp;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = 1;
rq.keys = &keylist;
rq.callback_ctx = &cbctx;
@@ -692,7 +707,7 @@ couchstore_error_t couchstore_all_docs(Db *db,
rq.cmp.compare = ebin_cmp;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = 1;
rq.keys = &keylist;
rq.callback_ctx = &cbctx;
@@ -759,10 +774,10 @@ couchstore_error_t couchstore_walk_tree(Db *db,
lookup_context lookup_ctx = {db, options, NULL, ctx, by_id, 1, callback};
sized_buf cmptmp;
couchfile_lookup_request rq;
-
+
rq.cmp.compare = compare;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = 1;
rq.keys = &keylist;
rq.callback_ctx = &lookup_ctx;
@@ -848,7 +863,7 @@ static couchstore_error_t iterate_docinfos(Db *db,
sized_buf cmptmp;
rq.cmp.compare = key_compare;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = numDocs;
rq.keys = (sized_buf**) keyptrs;
rq.callback_ctx = &cbctx;
@@ -914,7 +929,7 @@ couchstore_error_t couchstore_db_info(Db *db, DbInfo* dbinfo) {
const node_pointer *id_root = db->header.by_id_root;
const node_pointer *seq_root = db->header.by_seq_root;
const node_pointer *local_root = db->header.local_docs_root;
- dbinfo->filename = db->filename;
+ dbinfo->filename = db->file.path;
dbinfo->header_position = db->header.position;
dbinfo->last_sequence = db->header.update_seq;
dbinfo->deleted_count = dbinfo->doc_count = dbinfo->space_used = 0;
@@ -934,21 +949,6 @@ couchstore_error_t couchstore_db_info(Db *db, DbInfo* dbinfo) {
return COUCHSTORE_SUCCESS;
}
-LIBCOUCHSTORE_API
-void couchstore_free_document(Doc *doc)
-{
- if (doc) {
- char *offset = (char *) (&((fatbuf *) NULL)->buf);
- fatbuf_free((fatbuf *) ((char *)doc - (char *)offset));
- }
-}
-
-LIBCOUCHSTORE_API
-void couchstore_free_docinfo(DocInfo *docinfo)
-{
- free(docinfo);
-}
-
static couchstore_error_t local_doc_fetch(couchfile_lookup_request *rq,
void *k,
sized_buf *v)
@@ -1002,7 +1002,7 @@ couchstore_error_t couchstore_open_local_document(Db *db,
rq.cmp.compare = ebin_cmp;
rq.cmp.arg = &cmptmp;
- rq.db = db;
+ rq.file = &db->file;
rq.num_keys = 1;
rq.keys = &keylist;
rq.callback_ctx = pDoc;
@@ -1044,7 +1044,7 @@ couchstore_error_t couchstore_save_local_document(Db *db, LocalDoc *lDoc)
rq.fetch_callback = NULL;
rq.reduce = NULL;
rq.rereduce = NULL;
- rq.db = db;
+ rq.file = &db->file;
rq.compacting = 0;
nroot = modify_btree(&rq, db->header.local_docs_root, &errcode);
View
63 src/couch_file_read.c
@@ -1,5 +1,6 @@
/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include "config.h"
+#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
@@ -7,15 +8,55 @@
#include <snappy-c.h>
#include "internal.h"
+#include "iobuffer.h"
#include "bitfield.h"
#include "crc32.h"
#include "util.h"
#define MAX_HEADER_SIZE 1024 // Conservative estimate; just for sanity check
+couchstore_error_t tree_file_open(tree_file* file,
+ const char *filename,
+ int openflags,
+ const couch_file_ops *ops)
+{
+ couchstore_error_t errcode = COUCHSTORE_SUCCESS;
+
+ /* Sanity check input parameters */
+ if (filename == NULL || file == NULL || ops == NULL ||
+ ops->version != 3 || ops->constructor == NULL || ops->open == NULL ||
+ ops->close == NULL || ops->pread == NULL ||
+ ops->pwrite == NULL || ops->goto_eof == NULL ||
+ ops->sync == NULL || ops->destructor == NULL) {
+ return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
+ }
+
+ memset(file, 0, sizeof(*file));
+
+ file->path = strdup(filename);
+ error_unless(file->path, COUCHSTORE_ERROR_ALLOC_FAIL);
+
+ file->ops = couch_get_buffered_file_ops(ops, &file->handle);
+ error_unless(file->ops, COUCHSTORE_ERROR_ALLOC_FAIL);
+
+ error_pass(file->ops->open(&file->handle, filename, openflags));
+
+cleanup:
+ return errcode;
+}
+
+void tree_file_close(tree_file* file)
+{
+ if (file->ops) {
+ file->ops->close(file->handle);
+ file->ops->destructor(file->handle);
+ }
+ free((char*)file->path);
+}
+
/** Read bytes from the database file, skipping over the header-detection bytes at every block
boundary. */
-static couchstore_error_t read_skipping_prefixes(Db* db, off_t *pos, ssize_t len, void *dst) {
+static couchstore_error_t read_skipping_prefixes(tree_file *file, off_t *pos, ssize_t len, void *dst) {
if (*pos % COUCH_BLOCK_SIZE == 0) {
++*pos;
}
@@ -24,7 +65,7 @@ static couchstore_error_t read_skipping_prefixes(Db* db, off_t *pos, ssize_t len
if (read_size > len) {
read_size = len;
}
- ssize_t got_bytes = db->file_ops->pread(db->file_handle, dst, read_size, *pos);
+ ssize_t got_bytes = file->ops->pread(file->handle, dst, read_size, *pos);
if (got_bytes < 0) {
return (couchstore_error_t) got_bytes;
} else if (got_bytes == 0) {
@@ -43,14 +84,14 @@ static couchstore_error_t read_skipping_prefixes(Db* db, off_t *pos, ssize_t len
/** Common subroutine of pread_bin, pread_compressed and pread_header.
Parameters and return value are the same as for pread_bin,
except the 'header' parameter which is 1 if reading a header, 0 otherwise. */
-static int pread_bin_internal(Db *db, off_t pos, char **ret_ptr, int header)
+static int pread_bin_internal(tree_file *file, off_t pos, char **ret_ptr, int header)
{
struct {
uint32_t chunk_len;
uint32_t crc32;
} info;
- couchstore_error_t err = read_skipping_prefixes(db, &pos, sizeof(info), &info);
+ couchstore_error_t err = read_skipping_prefixes(file, &pos, sizeof(info), &info);
if (err < 0) {
return err;
}
@@ -67,7 +108,7 @@ static int pread_bin_internal(Db *db, off_t pos, char **ret_ptr, int header)
if (!buf) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- err = read_skipping_prefixes(db, &pos, info.chunk_len, buf);
+ err = read_skipping_prefixes(file, &pos, info.chunk_len, buf);
if (!err && info.crc32 && info.crc32 != hash_crc32(buf, info.chunk_len)) {
err = COUCHSTORE_ERROR_CHECKSUM_FAIL;
}
@@ -80,16 +121,16 @@ static int pread_bin_internal(Db *db, off_t pos, char **ret_ptr, int header)
return info.chunk_len;
}
-int pread_header(Db *db, off_t pos, char **ret_ptr)
+int pread_header(tree_file *file, off_t pos, char **ret_ptr)
{
- return pread_bin_internal(db, pos + 1, ret_ptr, 1);
+ return pread_bin_internal(file, pos + 1, ret_ptr, 1);
}
-int pread_compressed(Db *db, off_t pos, char **ret_ptr)
+int pread_compressed(tree_file *file, off_t pos, char **ret_ptr)
{
char *compressed_buf;
char *new_buf;
- int len = pread_bin_internal(db, pos, &compressed_buf, 0);
+ int len = pread_bin_internal(file, pos, &compressed_buf, 0);
if (len < 0) {
return len;
}
@@ -115,7 +156,7 @@ int pread_compressed(Db *db, off_t pos, char **ret_ptr)
return (int) uncompressed_len;
}
-int pread_bin(Db *db, off_t pos, char **ret_ptr)
+int pread_bin(tree_file *file, off_t pos, char **ret_ptr)
{
- return pread_bin_internal(db, pos, ret_ptr, 0);
+ return pread_bin_internal(file, pos, ret_ptr, 0);
}
View
30 src/couch_file_write.c
@@ -13,7 +13,7 @@
#include "crc32.h"
#include "util.h"
-static ssize_t raw_write(Db *db, const sized_buf *buf, off_t pos)
+static ssize_t raw_write(tree_file *file, const sized_buf *buf, off_t pos)
{
off_t write_pos = pos;
size_t buf_pos = 0;
@@ -27,7 +27,7 @@ static ssize_t raw_write(Db *db, const sized_buf *buf, off_t pos)
}
if (write_pos % COUCH_BLOCK_SIZE == 0) {
- written = db->file_ops->pwrite(db->file_handle, &blockprefix, 1, write_pos);
+ written = file->ops->pwrite(file->handle, &blockprefix, 1, write_pos);
if (written < 0) {
return written;
}
@@ -35,7 +35,7 @@ static ssize_t raw_write(Db *db, const sized_buf *buf, off_t pos)
continue;
}
- written = db->file_ops->pwrite(db->file_handle, buf->buf + buf_pos, block_remain, write_pos);
+ written = file->ops->pwrite(file->handle, buf->buf + buf_pos, block_remain, write_pos);
if (written < 0) {
return written;
}
@@ -46,9 +46,9 @@ static ssize_t raw_write(Db *db, const sized_buf *buf, off_t pos)
return (ssize_t)(write_pos - pos);
}
-couchstore_error_t db_write_header(Db *db, sized_buf *buf, off_t *pos)
+couchstore_error_t db_write_header(tree_file *file, sized_buf *buf, off_t *pos)
{
- off_t write_pos = db->file_pos;
+ off_t write_pos = file->pos;
ssize_t written;
uint32_t size = htonl(buf->size + 4); //Len before header includes hash len.
uint32_t crc32 = htonl(hash_crc32(buf->buf, buf->size));
@@ -64,26 +64,26 @@ couchstore_error_t db_write_header(Db *db, sized_buf *buf, off_t *pos)
memcpy(&headerbuf[1], &size, 4);
memcpy(&headerbuf[5], &crc32, 4);
- written = db->file_ops->pwrite(db->file_handle, &headerbuf, sizeof(headerbuf), write_pos);
+ written = file->ops->pwrite(file->handle, &headerbuf, sizeof(headerbuf), write_pos);
if (written < 0) {
return (couchstore_error_t)written;
}
write_pos += written;
//Write actual header
- written = raw_write(db, buf, write_pos);
+ written = raw_write(file, buf, write_pos);
if (written < 0) {
return (couchstore_error_t)written;
}
write_pos += written;
- db->file_pos = write_pos;
+ file->pos = write_pos;
return COUCHSTORE_SUCCESS;
}
-int db_write_buf(Db *db, const sized_buf *buf, off_t *pos, size_t *disk_size)
+int db_write_buf(tree_file *file, const sized_buf *buf, off_t *pos, size_t *disk_size)
{
- off_t write_pos = db->file_pos;
+ off_t write_pos = file->pos;
off_t end_pos = write_pos;
ssize_t written;
uint32_t size = htonl(buf->size | 0x80000000);
@@ -95,14 +95,14 @@ int db_write_buf(Db *db, const sized_buf *buf, off_t *pos, size_t *disk_size)
memcpy(&headerbuf[4], &crc32, 4);
sized_buf sized_headerbuf = { headerbuf, 8 };
- written = raw_write(db, &sized_headerbuf, end_pos);
+ written = raw_write(file, &sized_headerbuf, end_pos);
if (written < 0) {
return (int)written;
}
end_pos += written;
// Write actual buffer:
- written = raw_write(db, buf, end_pos);
+ written = raw_write(file, buf, end_pos);
if (written < 0) {
return (int)written;
}
@@ -112,7 +112,7 @@ int db_write_buf(Db *db, const sized_buf *buf, off_t *pos, size_t *disk_size)
*pos = write_pos;
}
- db->file_pos = end_pos;
+ file->pos = end_pos;
if (disk_size) {
*disk_size = (size_t) (end_pos - write_pos);
}
@@ -120,7 +120,7 @@ int db_write_buf(Db *db, const sized_buf *buf, off_t *pos, size_t *disk_size)
return 0;
}
-int db_write_buf_compressed(Db *db, const sized_buf *buf, off_t *pos, size_t *disk_size)
+int db_write_buf_compressed(tree_file *file, const sized_buf *buf, off_t *pos, size_t *disk_size)
{
int errcode = 0;
sized_buf to_write;
@@ -133,7 +133,7 @@ int db_write_buf_compressed(Db *db, const sized_buf *buf, off_t *pos, size_t *di
&to_write.size) == SNAPPY_OK,
COUCHSTORE_ERROR_WRITE);
- error_pass(db_write_buf(db, &to_write, pos, disk_size));
+ error_pass(db_write_buf(file, &to_write, pos, disk_size));
cleanup:
free(to_write.buf);
return errcode;
View
339 src/couch_index.c
@@ -0,0 +1,339 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+#include "config.h"
+#include <assert.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <libcouchstore/couch_index.h>
+#include "bitfield.h"
+#include "collate_json.h"
+#include "couch_btree.h"
+#include "internal.h"
+#include "json_reduce.h"
+#include "node_types.h"
+#include "tree_writer.h"
+#include "util.h"
+
+
+#define DST_SIZE 500
+
+
+// Header consists of a 32-bit root count, followed by the roots.
+// Each root has a 1-byte type field (0 for primary, 1 for back-index),
+// a 16-bit size, then its data (pointer, subtree size, reduce value.)
+
+typedef struct {
+ raw_08 type;
+ raw_16 size;
+ raw_btree_root root;
+} raw_index_file_root;
+
+typedef struct {
+ raw_32 rootCount;
+ raw_index_file_root firstRoot;
+} raw_index_file_header;
+
+
+/* Private data of the CouchStoreIndex structure */
+struct _CouchStoreIndex {
+ tree_file file;
+ uint32_t back_root_index;
+ uint32_t root_count;
+ node_pointer** roots;
+};
+
+
+static const JSONReducer* CurrentReducer;
+static couchstore_index_type CurrentIndexType;
+
+
+static void view_reduce(char *dst, size_t *size_r, nodelist *leaflist, int count);
+static void view_rereduce(char *dst, size_t *size_r, nodelist *leaflist, int count);
+
+
+LIBCOUCHSTORE_API
+couchstore_error_t couchstore_create_index(const char *filename,
+ CouchStoreIndex** index)
+{
+ couchstore_error_t errcode = COUCHSTORE_SUCCESS;
+
+ CouchStoreIndex* file = calloc(1, sizeof(*file));
+ error_unless(file != NULL, COUCHSTORE_ERROR_ALLOC_FAIL);
+ error_pass(tree_file_open(&file->file, filename, O_RDWR | O_CREAT | O_TRUNC,
+ couchstore_get_default_file_ops()));
+ file->back_root_index = UINT32_MAX;
+ *index = file;
+cleanup:
+ return errcode;
+}
+
+
+static couchstore_error_t write_index_header(CouchStoreIndex* index)
+{
+ couchstore_error_t errcode = COUCHSTORE_SUCCESS;
+
+ // Compute header size and allocate a buffer:
+ sized_buf writebuf = {NULL, 4};
+ for (uint32_t i = 0; i < index->root_count; ++i) {
+ writebuf.size += 3 + encode_root(NULL, index->roots[i]);
+ }
+ writebuf.buf = calloc(1, writebuf.size);
+ error_unless(writebuf.buf, COUCHSTORE_ERROR_ALLOC_FAIL);
+
+ // Write the root count:
+ raw_index_file_header* header = (raw_index_file_header*)writebuf.buf;
+ header->rootCount = encode_raw32(index->root_count);
+ raw_index_file_root* root = &header->firstRoot;
+
+ // Write the roots:
+ for (uint32_t i = 0; i < index->root_count; ++i) {
+ root->type = encode_raw08((i == index->back_root_index) ? COUCHSTORE_VIEW_BACK_INDEX
+ : COUCHSTORE_VIEW_PRIMARY_INDEX);
+ size_t rootSize = encode_root(&root->root, index->roots[i]);
+ root->size = encode_raw16((uint16_t)rootSize);
+ root = (raw_index_file_root*)((char*)root + 3 + rootSize);
+ }
+ assert((char*)root - writebuf.buf == (ssize_t)writebuf.size);