Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Remove EI dependency (File format changes)

Change-Id: Ib90f14b868f0f784ce291b14770cecff67c8d6ba
  • Loading branch information...
commit 871500b2e979fa6122a140e6be92cd37f94aa324 1 parent 829fffa
@apage43 authored
View
13 Makefile.am
@@ -59,8 +59,8 @@ libcouchstore_la_SOURCES = \
src/util.c \
src/util.h
-libcouchstore_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(LIBCOUCHSTORE_API_CURRENT):$(LIBCOUCHSTORE_API_REVISION):$(LIBCOUCHSTORE_API_AGE) -no-undefined $(ERL_INTERFACE_DIR_LIB) -lei -lsnappy
-libcouchstore_la_CFLAGS = $(AM_CFLAGS) -DLIBCOUCHSTORE_INTERNAL=1 $(ERL_INTERFACE_DIR_INCLUDE)
+libcouchstore_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(LIBCOUCHSTORE_API_CURRENT):$(LIBCOUCHSTORE_API_REVISION):$(LIBCOUCHSTORE_API_AGE) -no-undefined -lsnappy
+libcouchstore_la_CFLAGS = $(AM_CFLAGS) -DLIBCOUCHSTORE_INTERNAL=1
libcouchstore_la_LIBADD = librfc1321.la libbyteswap.la
couch_dbdump_SOURCES = src/dbdump.c
@@ -69,7 +69,7 @@ couch_dbdump_CFLAGS = $(AM_CFLAGS) -D__STDC_FORMAT_MACROS
couch_dbdump_LDADD = libcouchstore.la libbyteswap.la -lsnappy
couch_dbinfo_SOURCES = src/dbinfo.c src/util.c
-couch_dbinfo_CFLAGS = $(AM_CFLAGS) $(ERL_INTERFACE_DIR_INCLUDE) -D__STDC_FORMAT_MACROS
+couch_dbinfo_CFLAGS = $(AM_CFLAGS) -D__STDC_FORMAT_MACROS
couch_dbinfo_DEPENDENCIES = libcouchstore.la libbyteswap.la
couch_dbinfo_LDADD = libcouchstore.la libbyteswap.la
#writetest_SOURCES = src/writetest.c
@@ -110,10 +110,9 @@ check_PROGRAMS = testapp
TESTS = ${check_PROGRAMS}
testapp_SOURCES = tests/testapp.c src/util.c tests/macros.h
-testapp_CFLAGS = $(AM_CFLAGS) $(ERL_INTERFACE_DIR_INCLUDE)
-testapp_LDFLAGS = $(ERL_INTERFACE_DIR_LIB) -lei
-testapp_DEPENDENCIES = libcouchstore.la
-testapp_LDADD = libcouchstore.la
+testapp_CFLAGS = $(AM_CFLAGS)
+testapp_DEPENDENCIES = libcouchstore.la libbyteswap.la
+testapp_LDADD = libcouchstore.la libbyteswap.la
test: check-TESTS $(extra_tests)
View
11 configure.ac
@@ -55,17 +55,6 @@ AC_SEARCH_LIBS(floor, m)
AC_SEARCH_LIBS(pow, m)
AC_SEARCH_LIBS(fabs, m)
-AC_PATH_PROG([ERL], [erl])
-AS_IF([test x${ERL} = x], [
- AC_MSG_ERROR([Could not find the erl executable. Is Erlang installed?])
- ])
-
-ERL_INTERFACE_DIR_INCLUDE="-I`${ERL} -eval 'io:put_chars(code:lib_dir(erl_interface) ++ "/include"), erlang:halt().' -noshell -noinput`"
-AC_SUBST(ERL_INTERFACE_DIR_INCLUDE)
-ERL_INTERFACE_DIR_LIB="-L`${ERL} -eval 'io:put_chars(code:lib_dir(erl_interface) ++ "/lib"), erlang:halt().' -noshell -noinput`"
-AC_SUBST(ERL_INTERFACE_DIR_LIB)
-
-
AC_ARG_WITH([docs],
[AS_HELP_STRING([--with-docs],
[Generate documentation (yes|no) @<:@default=yes@:>@])],
View
61 src/bitfield.h
@@ -0,0 +1,61 @@
+#ifndef COUCH_BITFIELD_H
+#define COUCH_BITFIELD_H
+#include "config_static.h"
+static inline uint64_t get_48(char *buf)
+{
+ uint64_t num = 0;
+ char *numbuf = (char *) &num;
+ memcpy(numbuf + 2, buf, 6);
+ return ntohll(num);
+}
+
+
+static inline uint64_t get_40(char *buf)
+{
+ uint64_t num = 0;
+ char *numbuf = (char *) &num;
+ memcpy(numbuf + 3, buf, 5);
+ return ntohll(num);
+}
+
+static inline uint32_t get_32(char *buf)
+{
+ uint32_t num = *((uint32_t *) buf);
+ return ntohl(num);
+}
+
+
+static inline uint32_t get_16(char *buf)
+{
+ uint32_t num = 0;
+ char *numbuf = (char *) &num;
+ memcpy(numbuf + 2, buf, 2);
+ return ntohl(num);
+}
+
+static inline void get_kvlen(char *buf, int *klen, int *vlen)
+{
+ //12, 28 bit
+ uint32_t num = 0;
+ char *numbuf = (char *) &num;
+ memcpy(numbuf + 2, buf, 2);
+ *klen = ntohl(num) >> 4;
+ num = 0;
+ numbuf = (char *) &num;
+ memcpy(numbuf, buf + 1, 4);
+ *vlen = ntohl(num) & 0x0FFFFFFF;
+}
+
+//Flip on the the bits of num as a numbits-bit number, offset bitpos bits into
+//buf. MUST ZERO MEMORY _BEFORE_ WRITING TO IT!
+static inline void set_bits(char *buf, const int bitpos, const int numbits, uint64_t num)
+{
+ num = num << (64 - (numbits + bitpos));
+ num = htonll(num);
+ char *nbuf = (char *) &num;
+ int i = 0;
+ for (i = 0; i < ((numbits + bitpos + 7) / 8); i++) {
+ buf[i] |= nbuf[i];
+ }
+}
+#endif
View
547 src/btree_modify.c
@@ -5,16 +5,11 @@
#include <signal.h>
#include "couch_btree.h"
-#include <ei.h>
#include "util.h"
+#include "bitfield.h"
#define CHUNK_THRESHOLD 1279
-sized_buf empty_root = {
- // {kv_node, []}
- (char *) "\x68\x02\x64\x00\x07kv_node\x6A",
- 13
-};
static couchstore_error_t flush_mr(couchfile_modify_result *res);
@@ -24,40 +19,6 @@ static void append_buf(void *dst, int *dstpos, void *src, int len)
*dstpos += len;
}
-static couchstore_error_t find_first_gteq(char *buf, int pos, void *key,
- compare_info *lu, int at_least)
-{
- int list_arity, inner_arity;
- int list_pos = 0, cmp_val;
- off_t pair_pos = 0;
- couchstore_error_t errcode = COUCHSTORE_SUCCESS;
- error_nonzero(ei_decode_list_header(buf, &pos, &list_arity),
- COUCHSTORE_ERROR_PARSE_TERM);
- while (list_pos < list_arity) {
- //{<<"key", some other term}
- pair_pos = pos; //Save pos of kv/kp pair tuple
-
- error_nonzero(ei_decode_tuple_header(buf, &pos, &inner_arity),
- COUCHSTORE_ERROR_PARSE_TERM)
-
- lu->last_cmp_key = (*lu->from_ext)(lu, buf, pos);
- cmp_val = (*lu->compare) (lu->last_cmp_key, key);
- lu->last_cmp_val = cmp_val;
- lu->list_pos = list_pos;
- if (cmp_val >= 0 && list_pos >= at_least) {
- break;
- }
- error_nonzero(ei_skip_term(buf, &pos), COUCHSTORE_ERROR_PARSE_TERM); //skip over the key
- error_nonzero(ei_skip_term(buf, &pos), COUCHSTORE_ERROR_PARSE_TERM); //skip over the value
- list_pos++;
- }
-cleanup:
- if (errcode < 0) {
- return errcode;
- }
- return pair_pos;
-}
-
static couchstore_error_t maybe_flush(couchfile_modify_result *mr)
{
if (mr->modified && mr->node_len > CHUNK_THRESHOLD) {
@@ -72,20 +33,23 @@ static void free_nodelist(nodelist *nl)
{
while (nl) {
nodelist *next = nl->next;
- free(nl->value.mem);
+ free(nl->pointer);
free(nl);
nl = next;
}
}
-static nodelist *make_nodelist(void)
+static nodelist *make_nodelist(int bufsize)
{
- nodelist *r = malloc(sizeof(nodelist));
+ nodelist *r = malloc(sizeof(nodelist) + bufsize);
+ memset(r, 0, sizeof(nodelist) + bufsize);
if (!r) {
return NULL;
}
r->next = NULL;
- r->value.mem = NULL;
+ r->pointer = NULL;
+ r->data.size = bufsize;
+ r->data.buf = ((char *) r) + (sizeof(nodelist));
return r;
}
@@ -95,13 +59,13 @@ static couchfile_modify_result *make_modres(couchfile_modify_request *rq)
if (!res) {
return NULL;
}
- res->values = make_nodelist();
+ res->values = make_nodelist(0);
if (!res->values) {
free(res);
return NULL;
}
res->values_end = res->values;
- res->pointers = make_nodelist();
+ res->pointers = make_nodelist(0);
if (!res->pointers) {
free(res);
return NULL;
@@ -122,239 +86,151 @@ static void free_modres(couchfile_modify_result *mr)
free(mr);
}
-static couchstore_error_t mr_push_action(couchfile_modify_action *act,
- couchfile_modify_result *dst)
+static couchstore_error_t mr_push_item(sized_buf *k, sized_buf *v, couchfile_modify_result *dst)
{
- //For ACTION_INSERT
- sized_buf *lv = malloc(sizeof(sized_buf) +
- act->key->size + act->value.term->size + 2);
- if (!lv) {
+ nodelist *itm = make_nodelist(0);
+ if (!itm) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- //Allocate space for {K,V} term
- lv->buf = ((char *)lv) + sizeof(sized_buf);
- lv->size = (act->key->size + act->value.term->size + 2);
- //tuple of arity 2
- lv->buf[0] = 104;
- lv->buf[1] = 2;
- //copy terms from the action
- memcpy(lv->buf + 2, act->key->buf, act->key->size);
- memcpy(lv->buf + 2 + act->key->size,
- act->value.term->buf, act->value.term->size);
-
- nodelist *n = make_nodelist();
- if (!n) {
- free(lv);
- return COUCHSTORE_ERROR_ALLOC_FAIL;
- }
- dst->values_end->next = n;
- dst->values_end = n;
- n->value.leaf = lv;
-
- dst->node_len += lv->size;
+ itm->key = *k;
+ itm->data = *v;
+ itm->pointer = NULL;
+ dst->values_end->next = itm;
+ dst->values_end = itm;
+ //Encoded size
+ dst->node_len += k->size + v->size + 5;
dst->count++;
return maybe_flush(dst);
}
+static nodelist *encode_pointer(node_pointer *ptr)
+{
+ nodelist *pel = make_nodelist(14 + ptr->reduce_value.size);
+ if (!pel) {
+ return NULL;
+ }
+ set_bits(pel->data.buf, 0, 48, ptr->pointer);
+ set_bits(pel->data.buf + 6, 0, 48, ptr->subtreesize);
+ set_bits(pel->data.buf + 12, 0, 16, ptr->reduce_value.size);
+ memcpy(pel->data.buf + 14, ptr->reduce_value.buf, ptr->reduce_value.size);
+ pel->pointer = ptr;
+ pel->key = ptr->key;
+ return pel;
+}
+
static couchstore_error_t mr_push_pointerinfo(node_pointer *ptr,
couchfile_modify_result *dst)
{
- nodelist *pel = make_nodelist();
+ nodelist *pel = encode_pointer(ptr);
if (!pel) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- pel->value.pointer = ptr;
dst->values_end->next = pel;
dst->values_end = pel;
-
- //max len of {key,{pointer, reduce_value, subtreesize}}
- dst->node_len += ptr->key.size + ptr->reduce_value.size + 24;
+ dst->node_len += pel->key.size + pel->data.size + 5;
dst->count++;
return maybe_flush(dst);
}
-static couchstore_error_t mr_push_kv_range(char *buf, int pos,
- int bound, int end,
- couchfile_modify_result *dst)
-{
- int current = 0;
- int term_begin_pos;
- ei_decode_list_header(buf, &pos, NULL);
- couchstore_error_t errcode = COUCHSTORE_SUCCESS;
- sized_buf *lv = NULL;
-
- while (current < end && errcode == 0) {
- term_begin_pos = pos;
- ei_skip_term(buf, &pos);
- if (current >= bound) {
- //Parse KV pair into a leaf_value
- nodelist *n;
- lv = (sized_buf *) malloc(sizeof(sized_buf));
- error_unless(lv, COUCHSTORE_ERROR_ALLOC_FAIL);
-
- lv->buf = buf + term_begin_pos;
- lv->size = pos - term_begin_pos;
-
- n = make_nodelist();
- error_unless(n, COUCHSTORE_ERROR_ALLOC_FAIL);
- dst->values_end->next = n;
- dst->values_end = n;
- n->value.leaf = lv;
-
- dst->node_len += lv->size;
- dst->count++;
- lv = NULL;
- error_pass(maybe_flush(dst));
- }
- current++;
- }
-cleanup:
- free(lv);
- return errcode;
-}
-
-static node_pointer *read_pointer(char *buf, int pos)
+static node_pointer *read_pointer(sized_buf *key, char *buf)
{
//Parse KP pair into a node_pointer {K, {ptr, reduce_value, subtreesize}}
node_pointer *p = (node_pointer *) malloc(sizeof(node_pointer));
if (!p) {
return NULL;
}
- ei_decode_tuple_header(buf, &pos, NULL); //arity 2
- term_to_buf(&p->key, buf, &pos);
- ei_decode_tuple_header(buf, &pos, NULL); //arity 3
- ei_decode_uint64(buf, &pos, &p->pointer);
- term_to_buf(&p->reduce_value, buf, &pos);
- ei_decode_uint64(buf, &pos, &p->subtreesize);
-
+ p->pointer = get_48(buf);
+ p->subtreesize = get_48(buf + 6);
+ p->reduce_value.size = get_16(buf + 12);
+ p->reduce_value.buf = buf + 14;
+ p->key = *key;
return p;
}
-static void mr_push_kp_range(char *buf, int pos, int bound, int end,
- couchfile_modify_result *dst)
-{
- int current = 0;
- ei_decode_list_header(buf, &pos, NULL);
- while (current < end) {
- if (current >= bound) {
- mr_push_pointerinfo(read_pointer(buf, pos), dst);
- }
- ei_skip_term(buf, &pos);
- current++;
- }
-}
-
//Write the current contents of the values list to disk as a node
//and add the resulting pointer to the pointers list.
static couchstore_error_t flush_mr(couchfile_modify_result *res)
{
- int nbufpos = 0;
- uint64_t subtreesize = 0;
- sized_buf reduce_value;
+ int bufpos = 0;
+ int errcode = COUCHSTORE_SUCCESS;
+ char *nodebuf = NULL;
sized_buf writebuf;
- //default reduce value []
- reduce_value.buf = (char *) "\x6A"; //NIL_EXT
- reduce_value.size = 1;
- int reduced = 0;
- couchstore_error_t errcode = COUCHSTORE_SUCCESS;
- nodelist *pel = NULL;
+ char reducebuf[30];
+ size_t reducesize = 0;
+ uint64_t subtreesize = 0;
+ off_t diskpos;
+ sized_buf final_key;
- if (res->values_end == res->values || !res->modified) {
+ if (res->values_end == res->values || ! res->modified) {
//Empty
- return 0;
+ return COUCHSTORE_SUCCESS;
}
- res->node_len += 19; //tuple header and node type tuple, list header and tail
- char *nodebuf = (char *) malloc(res->node_len);
-
- //External term header; tuple header arity 2;
- ei_encode_version(nodebuf, &nbufpos);
- ei_encode_tuple_header(nodebuf, &nbufpos, 2);
- switch (res->node_type) {
- case KV_NODE:
- ei_encode_atom_len(nodebuf, &nbufpos, "kv_node", 7);
- if (res->rq->reduce) {
- (*res->rq->reduce)(&reduce_value, res->values->next, res->count);
- reduced = 1;
- }
- break;
- case KP_NODE:
- ei_encode_atom_len(nodebuf, &nbufpos, "kp_node", 7);
- if (res->rq->rereduce) {
- (*res->rq->rereduce)(&reduce_value, res->values->next, res->count);
- reduced = 1;
- }
- break;
+ nodebuf = malloc(res->node_len + 1);
+ if (!nodebuf) {
+ return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- ei_encode_list_header(nodebuf, &nbufpos, res->count);
+ writebuf.buf = nodebuf;
+ writebuf.size = res->node_len + 1;
+ memset(nodebuf, 0, res->node_len + 1);
+
+ nodebuf[0] = res->node_type;
+ bufpos = 1;
nodelist *i = res->values->next;
-
- sized_buf last_key = {NULL, 0};
while (i != NULL) {
- if (res->node_type == KV_NODE) { //writing value in a kv_node
- append_buf(nodebuf, &nbufpos, i->value.leaf->buf, i->value.leaf->size);
- if (i->next == NULL) {
- int pos = 0;
- term_to_buf(&last_key, i->value.leaf->buf + 2, &pos);
- }
- } else if (res->node_type == KP_NODE) { //writing value in a kp_node
- //waitpointer used to live here
- subtreesize += i->value.pointer->subtreesize;
- ei_encode_tuple_header(nodebuf, &nbufpos, 2); //tuple arity 2
- append_buf(nodebuf, &nbufpos, i->value.pointer->key.buf, i->value.pointer->key.size);
- ei_encode_tuple_header(nodebuf, &nbufpos, 3); //tuple arity 3
- //pointer
- // v- between 2 and 10 bytes (ERL_SMALL_INTEGER_EXT to ERL_SMALL_BIG_EXT/8)
- ei_encode_ulonglong(nodebuf, &nbufpos, i->value.pointer->pointer);
- //reduce_value
- append_buf(nodebuf, &nbufpos, i->value.pointer->reduce_value.buf,
- i->value.pointer->reduce_value.size);
- //subtreesize
- // v- between 2 and 10 bytes (ERL_SMALL_INTEGER_EXT to ERL_SMALL_BIG_EXT/8)
- ei_encode_ulonglong(nodebuf, &nbufpos, i->value.pointer->subtreesize);
- if (i->next == NULL) {
- last_key = i->value.pointer->key;
- }
+ set_bits(nodebuf + bufpos, 0, 12, i->key.size);
+ set_bits(nodebuf + bufpos + 1, 4, 28, i->data.size);
+ memcpy(nodebuf + bufpos + 5, i->key.buf, i->key.size);
+ memcpy(nodebuf + bufpos + 5 + i->key.size, i->data.buf, i->data.size);
+ bufpos = bufpos + 5 + i->data.size + i->key.size;
+ if (i->pointer) {
+ subtreesize += i->pointer->subtreesize;
+ }
+ if (i->next == NULL) {
+ final_key = i->key;
}
i = i->next;
}
- //NIL_EXT (list tail)
- ei_encode_empty_list(nodebuf, &nbufpos);
+ errcode = db_write_buf_compressed(res->rq->db, &writebuf, &diskpos);
+ free(nodebuf);
+ if (errcode != COUCHSTORE_SUCCESS) {
+ return errcode;
+ }
- node_pointer *ptr = (node_pointer *) malloc(sizeof(node_pointer) +
- last_key.size + reduce_value.size);
- if (!ptr) {
- errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
- goto cleanup;
+ if (res->node_type == KV_NODE && res->rq->reduce) {
+ res->rq->reduce(reducebuf, &reducesize, res->values->next, res->count);
}
- ptr->pointer = 0;
+ if (res->node_type == KP_NODE && res->rq->rereduce) {
+ res->rq->rereduce(reducebuf, &reducesize, res->values->next, res->count);
+ }
- writebuf.buf = nodebuf;
- writebuf.size = nbufpos;
- db_write_buf_compressed(res->rq->db, &writebuf, (off_t *) &ptr->pointer);
+ node_pointer *ptr = (node_pointer *) malloc(sizeof(node_pointer) + final_key.size + reducesize);
+ if (!ptr) {
+ return COUCHSTORE_ERROR_ALLOC_FAIL;
+ }
ptr->key.buf = ((char *)ptr) + sizeof(node_pointer);
- ptr->reduce_value.buf = ((char *)ptr) + sizeof(node_pointer) + last_key.size;
+ ptr->reduce_value.buf = ((char *)ptr) + sizeof(node_pointer) + final_key.size;
- ptr->key.size = last_key.size;
- ptr->reduce_value.size = reduce_value.size;
+ ptr->key.size = final_key.size;
+ ptr->reduce_value.size = reducesize;
- memcpy(ptr->key.buf, last_key.buf, last_key.size);
- memcpy(ptr->reduce_value.buf, reduce_value.buf, reduce_value.size);
+ memcpy(ptr->key.buf, final_key.buf, final_key.size);
+ memcpy(ptr->reduce_value.buf, reducebuf, reducesize);
- ptr->subtreesize = subtreesize + nbufpos;
+ ptr->subtreesize = subtreesize + writebuf.size;
+ ptr->pointer = diskpos;
- pel = make_nodelist();
+ nodelist *pel = encode_pointer(ptr);
if (!pel) {
- errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
free(ptr);
- goto cleanup;
+ return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- pel->value.pointer = ptr;
+
res->pointers_end->next = pel;
res->pointers_end = pel;
@@ -364,16 +240,8 @@ static couchstore_error_t flush_mr(couchfile_modify_result *res)
res->values_end = res->values;
free_nodelist(res->values->next);
res->values->next = NULL;
-cleanup:
- free(nodebuf);
- if (errcode < 0) {
- free(ptr);
- }
- if (reduced) {
- free(reduce_value.buf);
- }
- return errcode;
+ return COUCHSTORE_SUCCESS;
}
//Move this node's pointers list to dst node's values list.
@@ -388,9 +256,7 @@ static couchstore_error_t mr_move_pointers(couchfile_modify_result *src,
nodelist *ptr = src->pointers->next;
nodelist *next = ptr;
while (ptr != NULL && errcode == 0) {
- //max on disk len of a pointer node
- dst->node_len += ptr->value.pointer->key.size +
- ptr->value.pointer->reduce_value.size + 24;
+ dst->node_len += ptr->data.size + ptr->key.size + 5;
dst->count++;
next = ptr->next;
@@ -413,49 +279,34 @@ static couchstore_error_t modify_node(couchfile_modify_request *rq,
int start, int end,
couchfile_modify_result *dst)
{
- sized_buf current_node;
- int curnode_pos = 0;
- int read_size = 0;
- int list_start_pos = 0;
- int node_len = 0;
- int node_bound = 0;
- couchstore_error_t errcode = COUCHSTORE_SUCCESS;
- int kpos = 0;
- int node_type_pos = 0;
+ char *nodebuf = NULL;
+ int bufpos = 1;
+ int nodebuflen = 0;
+ int errcode = 0;
couchfile_modify_result *local_result = NULL;
if (start == end) {
return 0;
}
- if (nptr == NULL) {
- current_node = empty_root;
- } else {
- if ((read_size = pread_compressed(rq->db, nptr->pointer, (char **) &current_node.buf)) < 0) {
+ if (nptr) {
+ if ((nodebuflen = pread_compressed(rq->db, nptr->pointer, (char **) &nodebuf)) < 0) {
error_pass(COUCHSTORE_ERROR_READ);
}
- current_node.size = read_size;
- curnode_pos++; //Skip over 131.
}
local_result = make_modres(rq);
error_unless(local_result, COUCHSTORE_ERROR_ALLOC_FAIL);
- ei_decode_tuple_header(current_node.buf, &curnode_pos, NULL);
- node_type_pos = curnode_pos;
- ei_skip_term(current_node.buf, &curnode_pos);
- list_start_pos = curnode_pos;
- error_nonzero(ei_decode_list_header(current_node.buf, &curnode_pos, &node_len), COUCHSTORE_ERROR_PARSE_TERM);
-
- if (atom_check(current_node.buf + node_type_pos, "kv_node")) {
+ if (nptr == NULL || nodebuf[0] == 1) { //KV Node
local_result->node_type = KV_NODE;
while (start < end) {
- if (node_bound >= node_len) {
+ if (bufpos >= nodebuflen) {
//We're at the end of a leaf node.
switch (rq->actions[start].type) {
case ACTION_INSERT:
local_result->modified = 1;
- mr_push_action(&rq->actions[start], local_result);
+ mr_push_item(rq->actions[start].key, rq->actions[start].value.data, local_result);
break;
case ACTION_REMOVE:
@@ -471,22 +322,23 @@ static couchstore_error_t modify_node(couchfile_modify_request *rq,
}
start++;
} else {
- kpos = find_first_gteq(current_node.buf, list_start_pos,
- rq->actions[start].cmp_key,
- &rq->cmp, node_bound);
-
- error_unless(kpos >= 0, COUCHSTORE_ERROR_PARSE_TERM);
-
- //Add items from node_bound up to but not including the current
- mr_push_kv_range(current_node.buf, list_start_pos, node_bound,
- rq->cmp.list_pos, local_result);
+ int klen, vlen;
+ get_kvlen(nodebuf + bufpos, &klen, &vlen);
+ sized_buf cmp_key = {nodebuf + bufpos + 5, klen};
+ sized_buf val_buf = {nodebuf + bufpos + 5 + klen, vlen};
+ bufpos += 5 + klen + vlen;
+ int cmp_val = rq->cmp.compare(&cmp_key, rq->actions[start].key);
+
+ if (cmp_val < 0) { //Key less than action key
+ mr_push_item(&cmp_key, &val_buf, local_result);
+ continue;
+ }
- if (rq->cmp.last_cmp_val > 0) { // Node key > action key
+ if (cmp_val > 0) { //Key greater than action key
switch (rq->actions[start].type) {
case ACTION_INSERT:
local_result->modified = 1;
- mr_push_action(&rq->actions[start], local_result);
- break;
+ mr_push_item(rq->actions[start].key, rq->actions[start].value.data, local_result);
case ACTION_REMOVE:
local_result->modified = 1;
@@ -499,99 +351,127 @@ static couchstore_error_t modify_node(couchfile_modify_request *rq,
}
break;
}
-
start++;
- node_bound = rq->cmp.list_pos;
- } else if (rq->cmp.last_cmp_val < 0) { // Node key < action key
- node_bound = rq->cmp.list_pos + 1;
- mr_push_kv_range(current_node.buf, list_start_pos, node_bound - 1,
- node_bound, local_result);
- } else { //Node key == action key
+ continue;
+ }
+
+ if (cmp_val == 0) { //Node key is equal to action key
+repeatkey:
switch (rq->actions[start].type) {
case ACTION_INSERT:
local_result->modified = 1;
- mr_push_action(&rq->actions[start], local_result);
- node_bound = rq->cmp.list_pos + 1;
+ mr_push_item(rq->actions[start].key, rq->actions[start].value.data, local_result);
break;
case ACTION_REMOVE:
local_result->modified = 1;
- node_bound = rq->cmp.list_pos + 1;
break;
case ACTION_FETCH:
if (rq->fetch_callback) {
- sized_buf cb_tmp;
- int cb_vpos = kpos;
- ei_decode_tuple_header(current_node.buf, &cb_vpos, NULL);
- ei_skip_term(current_node.buf, &cb_vpos);
- cb_tmp.buf = current_node.buf + cb_vpos;
- cb_tmp.size = cb_vpos;
- ei_skip_term(current_node.buf, &cb_vpos);
- cb_tmp.size = cb_vpos - cb_tmp.size;
- (*rq->fetch_callback)(rq, rq->actions[start].key, &cb_tmp, rq->actions[start].value.arg);
+ (*rq->fetch_callback)(rq, rq->actions[start].key, &val_buf, rq->actions[start].value.arg);
+ //Try the next action on the same item.
+ start++;
+ goto repeatkey;
}
- node_bound = rq->cmp.list_pos;
break;
}
start++;
+ continue;
}
}
}
//Push any items past the end of what we dealt with onto result.
- if (node_bound < node_len) {
- mr_push_kv_range(current_node.buf, list_start_pos, node_bound,
- node_len, local_result);
+ while (bufpos < nodebuflen) {
+ int klen, vlen;
+ get_kvlen(nodebuf + bufpos, &klen, &vlen);
+ sized_buf cmp_key = {nodebuf + bufpos + 5, klen};
+ sized_buf val_buf = {nodebuf + bufpos + 5 + klen, vlen};
+ bufpos += 5 + klen + vlen;
+ mr_push_item(&cmp_key, &val_buf, local_result);
}
- } else if (atom_check(current_node.buf + node_type_pos, "kp_node")) {
- local_result->node_type = KP_NODE;
- while (start < end) {
- kpos = find_first_gteq(current_node.buf, list_start_pos,
- rq->actions[start].cmp_key,
- &rq->cmp, node_bound);
-
- error_unless(kpos >= 0, COUCHSTORE_ERROR_PARSE_TERM);
+ } else if (nodebuf[0] == 0) {
+ while (bufpos < nodebuflen && start < end) {
+ int klen, vlen;
+ get_kvlen(nodebuf + bufpos, &klen, &vlen);
+ sized_buf cmp_key = {nodebuf + bufpos + 5, klen};
+ sized_buf val_buf = {nodebuf + bufpos + 5 + klen, vlen};
+ bufpos += 5 + klen + vlen;
+ int cmp_val = rq->cmp.compare(&cmp_key, rq->actions[start].key);
+ if (bufpos == nodebuflen) {
+ //We're at the last item in the kpnode, must apply all our
+ //actions here.
+ node_pointer *desc = read_pointer(&cmp_key, val_buf.buf);
+ if (!desc) {
+ errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
+ goto cleanup;
+ }
- if (rq->cmp.list_pos == (node_len - 1)) { //got last item in kp_node
- //Push all items except last onto mr
- mr_push_kp_range(current_node.buf, list_start_pos, node_bound,
- rq->cmp.list_pos, local_result);
- node_pointer *desc = read_pointer(current_node.buf, kpos);
errcode = modify_node(rq, desc, start, end, local_result);
- if (local_result->values_end->value.pointer != desc) {
+ if (local_result->values_end->pointer != desc) {
free(desc);
}
- error_pass(errcode);
- node_bound = node_len;
+ if (errcode != COUCHSTORE_SUCCESS) {
+ goto cleanup;
+ }
break;
- } else {
- //Get all actions with key <= the key of the current item in the
- //kp_node
+ }
+
+ if (cmp_val < 0) {
+ //Key in node item less than action item and not at end
+ //position, so just add it and continue.
+ node_pointer *add = read_pointer(&cmp_key, val_buf.buf);
+ if (!add) {
+ errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
+ goto cleanup;
+ }
- //Push items in node up to but not including current onto mr
- mr_push_kp_range(current_node.buf, list_start_pos, node_bound,
- rq->cmp.list_pos, local_result);
+ errcode = mr_push_pointerinfo(add, local_result);
+ if (errcode != COUCHSTORE_SUCCESS) {
+ goto cleanup;
+ }
+ } else if (cmp_val >= 0) {
+ //Found a key in the node greater than the one in the current
+ //action. Descend into the pointed node with as many actions as
+ //are less than the key here.
int range_end = start;
while (range_end < end &&
- ((*rq->cmp.compare)(rq->actions[range_end].cmp_key, rq->cmp.last_cmp_key) <= 0)) {
+ rq->cmp.compare(rq->actions[range_end].key, &cmp_key) <= 0) {
range_end++;
}
- node_bound = rq->cmp.list_pos + 1;
- node_pointer *desc = read_pointer(current_node.buf, kpos);
- errcode = modify_node(rq, desc, start, range_end, local_result);
- if (local_result->values_end->value.pointer != desc) {
+ node_pointer *desc = read_pointer(&cmp_key, val_buf.buf);
+ if (!desc) {
+ errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
+ goto cleanup;
+ }
+
+ errcode = modify_node(rq, desc, start, end, local_result);
+ if (local_result->values_end->pointer != desc) {
free(desc);
}
- error_pass(errcode);
- start = range_end;
+ if (errcode != COUCHSTORE_SUCCESS) {
+ goto cleanup;
+ }
}
}
- if (node_bound < node_len) {
- //Processed all the actions but haven't exhausted this kpnode.
- //push the rest of it onto the mr.
- mr_push_kp_range(current_node.buf, list_start_pos, node_bound, node_len,
- local_result);
+ while (bufpos < nodebuflen) {
+ int klen, vlen;
+ get_kvlen(nodebuf + bufpos, &klen, &vlen);
+ sized_buf cmp_key = {nodebuf + bufpos + 5, klen};
+ sized_buf val_buf = {nodebuf + bufpos + 5 + klen, vlen};
+ bufpos += 5 + klen + vlen;
+
+ node_pointer *add = read_pointer(&cmp_key, val_buf.buf);
+ if (!add) {
+ errcode = COUCHSTORE_ERROR_ALLOC_FAIL;
+ goto cleanup;
+ }
+
+ errcode = mr_push_pointerinfo(add, local_result);
+ if (errcode != COUCHSTORE_SUCCESS) {
+ goto cleanup;
+ }
}
} else {
errcode = COUCHSTORE_ERROR_PARSE_TERM;
@@ -610,8 +490,8 @@ static couchstore_error_t modify_node(couchfile_modify_request *rq,
cleanup:
free_modres(local_result);
- if (current_node.buf != empty_root.buf) {
- free(current_node.buf);
+ if (nodebuf) {
+ free(nodebuf);
}
return errcode;
@@ -634,13 +514,14 @@ static node_pointer *finish_root(couchfile_modify_request *rq,
if (root_result->pointers_end == root_result->pointers->next) {
//The root result split into exactly one kp_node.
//Return the pointer to it.
- ret_ptr = root_result->pointers_end->value.pointer;
- root_result->pointers_end->value.mem = NULL;
+ ret_ptr = root_result->pointers_end->pointer;
+ root_result->pointers_end->pointer = NULL;
break;
} else {
//The root result split into more than one kp_node.
//Move the pointer list to the value list and write out the new node.
*errcode = mr_move_pointers(root_result, collector);
+
if (*errcode < 0) {
goto cleanup;
}
@@ -678,10 +559,10 @@ node_pointer *modify_btree(couchfile_modify_request *rq,
return NULL;
}
- if (root_result->values_end->value.pointer == root) {
+ if (root_result->values_end->pointer == root) {
//If we got the root pointer back, remove it from the list
//so we don't try to free it.
- root_result->values_end->value.mem = NULL;
+ root_result->values_end->pointer = NULL;
}
if (!root_result->modified) {
@@ -695,8 +576,8 @@ node_pointer *modify_btree(couchfile_modify_request *rq,
ret_ptr = NULL;
}
} else {
- ret_ptr = root_result->values_end->value.pointer;
- root_result->values_end->value.mem = NULL;
+ ret_ptr = root_result->values_end->pointer;
+ root_result->values_end->pointer = NULL;
free_modres(root_result);
}
}
View
74 src/btree_read.c
@@ -3,15 +3,14 @@
#include <stdlib.h>
#include "couch_btree.h"
#include "util.h"
+#include "bitfield.h"
static couchstore_error_t btree_lookup_inner(couchfile_lookup_request *rq,
uint64_t diskpos,
int current,
int end)
{
- int bufpos = 0;
- int type_pos;
- int list_size = 0;
+ int bufpos = 1, nodebuflen = 0;
sized_buf v;
if (current == end) {
@@ -21,66 +20,48 @@ static couchstore_error_t btree_lookup_inner(couchfile_lookup_request *rq,
char *nodebuf = NULL;
- int nodebuflen = pread_compressed(rq->db, diskpos, &nodebuf);
+ nodebuflen = pread_compressed(rq->db, diskpos, &nodebuf);
error_unless(nodebuflen > 0, COUCHSTORE_ERROR_READ);
- bufpos++; //Skip over term version
- error_unless(tuple_check(nodebuf, &bufpos, 2), COUCHSTORE_ERROR_PARSE_TERM);
- type_pos = bufpos;
- ei_skip_term(nodebuf, &bufpos); //node type
- error_nonzero(ei_decode_list_header(nodebuf, &bufpos, &list_size),
- COUCHSTORE_ERROR_PARSE_TERM);
+ if (nodebuf[0] == 0) { //KP Node
+ while (bufpos < nodebuflen && current < end) {
+ int klen, vlen;
+ get_kvlen(nodebuf + bufpos, &klen, &vlen);
+ sized_buf cmp_key = {nodebuf + bufpos + 5, klen};
+ sized_buf val_buf = {nodebuf + bufpos + 5 + klen, vlen};
+ bufpos += 5 + klen + vlen;
- if (atom_check(nodebuf + type_pos, "kp_node")) {
- int list_item = 0;
- while (list_item < list_size && current < end) {
- //{K,P}
- error_unless(tuple_check(nodebuf, &bufpos, 2),
- COUCHSTORE_ERROR_PARSE_TERM);
- void *cmp_key = rq->cmp.from_ext(&rq->cmp, nodebuf, bufpos);
- ei_skip_term(nodebuf, &bufpos); //Skip key
-
- if (rq->cmp.compare(cmp_key, rq->keys[current]) >= 0) {
+ if (rq->cmp.compare(&cmp_key, rq->keys[current]) >= 0) {
if (rq->fold) {
rq->in_fold = 1;
}
- uint64_t pointer = 0;
- int last_item = current;
+
+ uint64_t pointer = 0, last_item = current;
//Descend into the pointed to node.
//with all keys < item key.
do {
last_item++;
- } while (last_item < end &&
- rq->cmp.compare(cmp_key, rq->keys[last_item]) >= 0);
+ } while (last_item < end && rq->cmp.compare(&cmp_key, rq->keys[last_item]) >= 0);
- error_unless(tuple_check(nodebuf, &bufpos, 3), COUCHSTORE_ERROR_PARSE_TERM);
- ei_decode_uint64(nodebuf, &bufpos, &pointer);
- ei_skip_term(nodebuf, &bufpos); //Skip reduce
- ei_skip_term(nodebuf, &bufpos); //Skip subtreesize
+ pointer = get_48(val_buf.buf);
error_pass(btree_lookup_inner(rq, pointer, current, last_item));
if (!rq->in_fold) {
current = last_item + 1;
}
- } else {
- ei_skip_term(nodebuf, &bufpos); //Skip pointer
}
- list_item++;
}
- } else if (atom_check(nodebuf + type_pos, "kv_node")) {
- int list_item = 0;
- while (list_item < list_size && current < end) {
- int cmp_val, keypos;
- sized_buf key_term;
- //{K,V}
- error_unless(tuple_check(nodebuf, &bufpos, 2), COUCHSTORE_ERROR_PARSE_TERM);
- void *cmp_key = rq->cmp.from_ext(&rq->cmp, nodebuf, bufpos);
- keypos = bufpos;
- ei_skip_term(nodebuf, &bufpos); //Skip key
- cmp_val = rq->cmp.compare(cmp_key, rq->keys[current]);
+ } else if (nodebuf[0] == 1) { //KV Node
+ while (bufpos < nodebuflen && current < end) {
+ int klen, vlen;
+ get_kvlen(nodebuf + bufpos, &klen, &vlen);
+ sized_buf cmp_key = {nodebuf + bufpos + 5, klen};
+ sized_buf val_buf = {nodebuf + bufpos + 5 + klen, vlen};
+ bufpos += 5 + klen + vlen;
+ int cmp_val = rq->cmp.compare(&cmp_key, rq->keys[current]);
if (cmp_val >= 0 && rq->fold && !rq->in_fold) {
rq->in_fold = 1;
} else if (rq->in_fold && (current + 1) < end &&
- (rq->cmp.compare(cmp_key, rq->keys[current + 1])) > 0) {
+ (rq->cmp.compare(&cmp_key, rq->keys[current + 1])) > 0) {
//We've hit a key past the end of our range.
rq->in_fold = 0;
rq->fold = 0;
@@ -89,16 +70,11 @@ static couchstore_error_t btree_lookup_inner(couchfile_lookup_request *rq,
if (cmp_val == 0 || (cmp_val > 0 && rq->in_fold)) {
//Found
- term_to_buf(&key_term, nodebuf, &keypos);
- term_to_buf(&v, nodebuf, &bufpos); //Read value
- error_pass(rq->fetch_callback(rq, &key_term, &v));
+ error_pass(rq->fetch_callback(rq, &cmp_key, &val_buf));
if (!rq->in_fold) {
current++;
}
- } else {
- ei_skip_term(nodebuf, &bufpos); //Skip value
}
- list_item++;
}
}
View
29 src/couch_btree.h
@@ -10,13 +10,10 @@ extern "C" {
typedef struct compare_info {
/* used by find_first_gteq */
int last_cmp_val;
- void *last_cmp_key;
+ sized_buf *last_cmp_key;
int list_pos;
/* Compare function */
- int (*compare)(void *k1, void *k2);
- /* Given an erlang term, return a pointer accepted by the compare function */
- void *(*from_ext)(struct compare_info *ci, char *buf, int pos);
- /* Use to set up any context needed by from_ext */
+ int (*compare)(sized_buf *k1, sized_buf *k2);
void *arg;
} compare_info;
@@ -33,7 +30,7 @@ extern "C" {
int fold;
/* v-- Flag used during lookup, do not set. */
int in_fold;
- void **keys;
+ sized_buf **keys;
void *callback_ctx;
couchstore_error_t (*fetch_callback) (struct couchfile_lookup_request *rq, void *k, sized_buf *v);
node_pointer *root;
@@ -45,11 +42,9 @@ extern "C" {
/* Modify */
typedef struct nodelist {
- union _nl_value {
- sized_buf *leaf;
- node_pointer *pointer;
- void *mem;
- } value;
+ sized_buf data;
+ sized_buf key;
+ node_pointer *pointer;
struct nodelist *next;
} nodelist;
@@ -60,9 +55,9 @@ extern "C" {
typedef struct couchfile_modify_action {
int type;
sized_buf *key;
- void *cmp_key;
+ sized_buf *cmp_key;
union _act_value {
- sized_buf *term;
+ sized_buf *data;
void *arg;
} value;
} couchfile_modify_action;
@@ -74,13 +69,13 @@ extern "C" {
couchfile_modify_action *actions;
void (*fetch_callback) (struct couchfile_modify_request *rq, sized_buf *k, sized_buf *v, void *arg);
/* Put result term into the sized_buf. */
- void (*reduce) (sized_buf *dst, nodelist *leaflist, int count);
- void (*rereduce) (sized_buf *dst, nodelist *ptrlist, int count);
+ void (*reduce) (char *dst, size_t *size_r, nodelist *leaflist, int count);
+ void (*rereduce) (char *dst, size_t *size_r, nodelist *ptrlist, int count);
node_pointer root;
} couchfile_modify_request;
-#define KV_NODE 0
-#define KP_NODE 1
+#define KP_NODE 0
+#define KV_NODE 1
/* Used to build and chunk modified nodes */
typedef struct couchfile_modify_result {
View
452 src/couch_db.c
@@ -2,11 +2,11 @@
#include <fcntl.h>
#include <string.h>
#include <stdlib.h>
-#include <ei.h>
#include <snappy-c.h>
#include "internal.h"
#include "couch_btree.h"
+#include "bitfield.h"
#include "util.h"
#include "reduces.h"
@@ -20,7 +20,7 @@ sized_buf nil_atom = {
static couchstore_error_t find_header(Db *db)
{
uint64_t block = db->file_pos / COUCH_BLOCK_SIZE;
- int errcode = 0;
+ int errcode = COUCHSTORE_SUCCESS;
int readsize;
char *header_buf = NULL;
uint8_t buf[2];
@@ -30,40 +30,33 @@ static couchstore_error_t find_header(Db *db)
if (readsize == 2 && buf[0] == 1) {
//Found a header block.
int header_len = pread_header(db, block * COUCH_BLOCK_SIZE, &header_buf);
- int arity = 0;
- int purged_docs_index = 0;
if (header_len > 0) {
- int idx = 0;
- if (ei_decode_version(header_buf, &idx, &arity) < 0) {
- errcode = COUCHSTORE_ERROR_PARSE_TERM;
- break;
+ db->header.disk_version = header_buf[0];
+ error_unless(db->header.disk_version == COUCH_DISK_VERSION, COUCHSTORE_ERROR_HEADER_VERSION)
+ db->header.update_seq = get_48(header_buf + 1);
+ db->header.purge_seq = get_48(header_buf + 7);
+ db->header.purge_ptr = get_48(header_buf + 13);
+ int seqrootsize = get_16(header_buf + 19);
+ int idrootsize = get_16(header_buf + 21);
+ int localrootsize = get_16(header_buf + 23);
+ int rootpos = 25;
+ if (seqrootsize > 0) {
+ db->header.by_seq_root = read_root(header_buf + rootpos, seqrootsize);
+ } else {
+ db->header.by_seq_root = NULL;
}
- if (ei_decode_tuple_header(header_buf, &idx, &arity) < 0) {
- errcode = COUCHSTORE_ERROR_PARSE_TERM;
- break;
+ rootpos += seqrootsize;
+ if (idrootsize > 0) {
+ db->header.by_id_root = read_root(header_buf + rootpos, idrootsize);
+ } else {
+ db->header.by_id_root = NULL;
}
- ei_skip_term(header_buf, &idx); //db_header
- ei_decode_uint64(header_buf, &idx, &db->header.disk_version);
-
- if (db->header.disk_version != COUCH_DISK_VERSION) {
- errcode = COUCHSTORE_ERROR_HEADER_VERSION;
- break;
+ rootpos += idrootsize;
+ if (localrootsize > 0) {
+ db->header.local_docs_root = read_root(header_buf + rootpos, localrootsize);
+ } else {
+ db->header.local_docs_root = NULL;
}
-
- ei_decode_uint64(header_buf, &idx, &db->header.update_seq);
- db->header.by_id_root = read_root(header_buf, &idx);
- db->header.by_seq_root = read_root(header_buf, &idx);
- db->header.local_docs_root = read_root(header_buf, &idx);
- ei_decode_uint64(header_buf, &idx, &db->header.purge_seq);
-
- purged_docs_index = idx;
- ei_skip_term(header_buf, &idx); //purged_docs
- db->header.purged_docs = malloc(sizeof(sized_buf) + (idx - purged_docs_index));
- db->header.purged_docs->buf = ((char *)db->header.purged_docs) + sizeof(sized_buf);
- memcpy(db->header.purged_docs->buf, header_buf + purged_docs_index, idx - purged_docs_index);
- db->header.purged_docs->size = idx - purged_docs_index;
-
- ei_skip_term(header_buf, &idx); //security ptr
db->header.position = block * COUCH_BLOCK_SIZE;
break;
}
@@ -79,34 +72,45 @@ static couchstore_error_t find_header(Db *db)
block--;
}
+cleanup:
free(header_buf);
return errcode;
}
static couchstore_error_t write_header(Db *db)
{
- ei_x_buff x_header;
sized_buf writebuf;
-
- ei_x_new_with_version(&x_header);
- ei_x_encode_tuple_header(&x_header, 8);
- ei_x_encode_atom(&x_header, "db_header");
- ei_x_encode_ulonglong(&x_header, db->header.disk_version);
- ei_x_encode_ulonglong(&x_header, db->header.update_seq);
- ei_x_encode_nodepointer(&x_header, db->header.by_id_root);
- ei_x_encode_nodepointer(&x_header, db->header.by_seq_root);
- ei_x_encode_nodepointer(&x_header, db->header.local_docs_root);
- ei_x_encode_ulonglong(&x_header, db->header.purge_seq);
- ei_x_append_buf(&x_header, db->header.purged_docs->buf, db->header.purged_docs->size);
- ei_x_encode_atom(&x_header, "nil"); //security_ptr;
- writebuf.buf = x_header.buff;
- writebuf.size = x_header.index;
+ int seqrootsize = 0, idrootsize = 0, localrootsize = 0;
+ if (db->header.by_seq_root) {
+ seqrootsize = 12 + db->header.by_seq_root->reduce_value.size;
+ }
+ if (db->header.by_id_root) {
+ idrootsize = 12 + db->header.by_id_root->reduce_value.size;
+ }
+ if (db->header.local_docs_root) {
+ localrootsize = 12 + db->header.local_docs_root->reduce_value.size;
+ }
+ writebuf.size = 25 + seqrootsize + idrootsize + localrootsize;
+ writebuf.buf = (char *) malloc(writebuf.size);
+ int bufpos = 0;
+ memset(writebuf.buf, 0, writebuf.size);
+ writebuf.buf[0] = COUCH_DISK_VERSION;
+ set_bits(writebuf.buf + 1, 0, 48, db->header.update_seq);
+ set_bits(writebuf.buf + 7, 0, 48, db->header.purge_seq);
+ set_bits(writebuf.buf + 13, 0, 48, db->header.purge_ptr);
+ set_bits(writebuf.buf + 19, 0, 16, seqrootsize);
+ set_bits(writebuf.buf + 21, 0, 16, idrootsize);
+ set_bits(writebuf.buf + 23, 0, 16, localrootsize);
+ encode_root(writebuf.buf + 25, db->header.by_seq_root);
+ encode_root(writebuf.buf + 25 + seqrootsize, db->header.by_id_root);
+ encode_root(writebuf.buf + 25 + seqrootsize + idrootsize,
+ db->header.local_docs_root);
off_t pos;
couchstore_error_t errcode = db_write_header(db, &writebuf, &pos);
if (errcode == COUCHSTORE_SUCCESS) {
db->header.position = pos;
}
- ei_x_free(&x_header);
+ free(writebuf.buf);
return errcode;
}
@@ -118,7 +122,6 @@ static couchstore_error_t create_header(Db *db)
db->header.by_seq_root = NULL;
db->header.local_docs_root = NULL;
db->header.purge_seq = 0;
- db->header.purged_docs = &nil_atom;
db->header.position = 0;
return write_header(db);
}
@@ -161,12 +164,12 @@ couchstore_error_t couchstore_open_db_ex(const char *filename,
/* Sanity check input parameters */
if (filename == NULL || pDb == NULL || ops == NULL ||
- ops->version != 1 || ops->open == NULL ||
- ops->close == NULL || ops->pread == NULL ||
- ops->pwrite == NULL || ops->goto_eof == NULL ||
- ops->sync == NULL || ops->destructor == NULL ||
- ((flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
- (flags & COUCHSTORE_OPEN_FLAG_CREATE))) {
+ ops->version != 1 || ops->open == NULL ||
+ ops->close == NULL || ops->pread == NULL ||
+ ops->pwrite == NULL || ops->goto_eof == NULL ||
+ ops->sync == NULL || ops->destructor == NULL ||
+ ((flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
+ (flags & COUCHSTORE_OPEN_FLAG_CREATE))) {
return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
}
@@ -226,19 +229,14 @@ couchstore_error_t couchstore_close_db(Db *db)
free(db->header.by_seq_root);
free(db->header.local_docs_root);
- if (db->header.purged_docs != &nil_atom) {
- free(db->header.purged_docs);
- }
memset(db, 0xa5, sizeof(*db));
free(db);
return COUCHSTORE_SUCCESS;
}
-static int ebin_cmp(void *k1, void *k2)
+static int ebin_cmp(sized_buf *e1, sized_buf *e2)
{
- sized_buf *e1 = (sized_buf *)k1;
- sized_buf *e2 = (sized_buf *)k2;
size_t size;
if (e2->size < e1->size) {
size = e2->size;
@@ -257,121 +255,75 @@ static int ebin_cmp(void *k1, void *k2)
return cmp;
}
-static void *ebin_from_ext(compare_info *c, char *buf, int pos)
-{
- int binsize;
- int type;
- sized_buf *ebcmp = (sized_buf *) c->arg;
- ei_get_type(buf, &pos, &type, &binsize);
- ebcmp->buf = buf + pos + 5;
- ebcmp->size = binsize;
- return ebcmp;
-}
-
-static void *term_from_ext(compare_info *c, char *buf, int pos)
-{
- int endpos = pos;
- sized_buf *ebcmp = (sized_buf *) c->arg;
- ei_skip_term(buf, &endpos);
- ebcmp->buf = buf + pos;
- ebcmp->size = endpos - pos;
- return ebcmp;
-}
-
-static int long_term_cmp(void *k1, void *k2)
+static int seq_cmp(sized_buf *k1, sized_buf *k2)
{
- sized_buf *e1 = (sized_buf *)k1;
- sized_buf *e2 = (sized_buf *)k2;
- int pos = 0;
- uint64_t e1val, e2val;
- ei_decode_uint64(e1->buf, &pos, &e1val);
- pos = 0;
- ei_decode_uint64(e2->buf, &pos, &e2val);
+ uint64_t e1val = get_48(k1->buf);
+ uint64_t e2val = get_48(k2->buf);
if (e1val == e2val) {
return 0;
}
return (e1val < e2val ? -1 : 1);
}
-static couchstore_error_t docinfo_from_buf(DocInfo **pInfo,
- sized_buf *v,
- int idBytes)
+static int by_seq_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
{
- couchstore_error_t errcode = COUCHSTORE_SUCCESS;
- int term_index = 0, fterm_pos = 0, fterm_size = 0;
- int metabin_pos = 0, metabin_size = 0;
- uint64_t deleted;
- uint64_t content_meta;
- uint64_t seq = 0, rev = 0, bp = 0;
- uint64_t size;
- char *infobuf = NULL;
- *pInfo = NULL;
-
- if (v == NULL) {
- return COUCHSTORE_ERROR_DOC_NOT_FOUND;
- }
-
- //Id/Seq
- error_unless(tuple_check(v->buf, &term_index, 6), COUCHSTORE_ERROR_PARSE_TERM);
- fterm_pos = term_index; //Save position of first term
- ei_skip_term(v->buf, &term_index);
- fterm_size = term_index - fterm_pos; //and size.
-
- //Rev = {RevNum, MetaBin}
- error_unless(tuple_check(v->buf, &term_index, 2), COUCHSTORE_ERROR_PARSE_TERM);
- error_nonzero(ei_decode_uint64(v->buf, &term_index, &rev), COUCHSTORE_ERROR_PARSE_TERM);
- metabin_pos = term_index + 5; //Save position of meta term
- //We know it's an ERL_BINARY_EXT, so the contents are from
- //5 bytes in to the end of the term.
- ei_skip_term(v->buf, &term_index);
- metabin_size = term_index - metabin_pos; //and size.
-
- error_nonzero(ei_decode_uint64(v->buf, &term_index, &bp), COUCHSTORE_ERROR_PARSE_TERM);
- error_nonzero(ei_decode_uint64(v->buf, &term_index, &deleted), COUCHSTORE_ERROR_PARSE_TERM);
- error_nonzero(ei_decode_uint64(v->buf, &term_index, &content_meta), COUCHSTORE_ERROR_PARSE_TERM);
- error_nonzero(ei_decode_uint64(v->buf, &term_index, &size), COUCHSTORE_ERROR_PARSE_TERM);
-
- //If first term is seq, we don't need to include it in the buffer
- if (idBytes != 0) {
- fterm_size = 0;
- }
- infobuf = (char *) malloc(sizeof(DocInfo) + metabin_size + fterm_size + idBytes);
- error_unless(infobuf, COUCHSTORE_ERROR_ALLOC_FAIL);
- *pInfo = (DocInfo *) infobuf;
-
- (*pInfo)->rev_meta.buf = infobuf + sizeof(DocInfo);
- (*pInfo)->rev_meta.size = metabin_size;
-
- if (metabin_size > 0) {
- memcpy((*pInfo)->rev_meta.buf, v->buf + metabin_pos, metabin_size);
- }
-
- (*pInfo)->id.buf = infobuf + sizeof(DocInfo) + metabin_size;
-
- if (idBytes != 0) { //First term is Seq
-
- (*pInfo)->id.size = idBytes;
- ei_decode_uint64(v->buf, &fterm_pos, &seq);
- //Let the caller fill in the Id.
- } else { //First term is Id
- (*pInfo)->id.size = fterm_size - 5; //Id will be a binary.
- memcpy((*pInfo)->id.buf, v->buf + fterm_pos + 5, fterm_size);
- //Let the caller fill in the Seq
+ uint32_t idsize, datasize, deleted, revnum;
+ uint8_t content_meta;
+ uint64_t bp, seq;
+ get_kvlen(v->buf, &idsize, &datasize);
+ deleted = v->buf[5] >> 7;
+ bp = get_48(v->buf + 5) &~ 0x800000000000;
+ content_meta = v->buf[11];
+ revnum = get_32(v->buf + 12);
+ seq = get_48(k->buf);
+ char *rbuf = (char *) malloc(sizeof(DocInfo) + (v->size - 16));
+ if (!rbuf) {
+ return COUCHSTORE_ERROR_ALLOC_FAIL;
}
-
+ memcpy(rbuf + sizeof(DocInfo), v->buf + 16, v->size - 16);
+ *pInfo = (DocInfo *) rbuf;
(*pInfo)->db_seq = seq;
- (*pInfo)->rev_seq = rev;
- (*pInfo)->bp = bp;
- (*pInfo)->size = size;
+ (*pInfo)->rev_seq = revnum;
(*pInfo)->deleted = deleted;
+ (*pInfo)->bp = bp;
+ (*pInfo)->size = datasize;
(*pInfo)->content_meta = content_meta;
+ (*pInfo)->id.buf = rbuf + sizeof(DocInfo);
+ (*pInfo)->id.size = idsize;
+ (*pInfo)->rev_meta.buf = rbuf + sizeof(DocInfo) + idsize;
+ (*pInfo)->rev_meta.size = v->size - 16 - idsize;
+ return 0;
+}
-cleanup:
- if (errcode < 0) {
- free(*pInfo);
- *pInfo = NULL;
+int by_id_read_docinfo(DocInfo **pInfo, sized_buf *k, sized_buf *v)
+{
+ uint32_t idsize, datasize, deleted, revnum;
+ uint8_t content_meta;
+ uint64_t bp, seq;
+ seq = get_48(v->buf);
+ datasize = get_32(v->buf + 6);
+ deleted = v->buf[10] >> 7;
+ bp = get_48(v->buf + 10) &~ 0x800000000000;
+ content_meta = v->buf[16];
+ revnum = get_32(v->buf + 17);
+ char *rbuf = (char *) malloc(sizeof(DocInfo) + (v->size - 21) + k->size);
+ if (!rbuf) {
+ return COUCHSTORE_ERROR_ALLOC_FAIL;
}
- return errcode;
+ memcpy(rbuf + sizeof(DocInfo), v->buf + 21, v->size - 21);
+ *pInfo = (DocInfo *) rbuf;
+ (*pInfo)->db_seq = seq;
+ (*pInfo)->rev_seq = revnum;
+ (*pInfo)->deleted = deleted;
+ (*pInfo)->bp = bp;
+ (*pInfo)->size = datasize;
+ (*pInfo)->content_meta = content_meta;
+ (*pInfo)->rev_meta.buf = rbuf + sizeof(DocInfo);
+ (*pInfo)->rev_meta.size = v->size - 21;
+ (*pInfo)->id.buf = (*pInfo)->rev_meta.buf + (*pInfo)->rev_meta.size;
+ (*pInfo)->id.size = k->size;
+ memcpy((*pInfo)->id.buf, k->buf, k->size);
+ return 0;
}
#define COMPRESSED_BODY 1
@@ -418,12 +370,10 @@ static couchstore_error_t docinfo_fetch(couchfile_lookup_request *rq,
{
sized_buf *id = (sized_buf *) k;
DocInfo **pInfo = (DocInfo **) rq->callback_ctx;
- int errcode = docinfo_from_buf(pInfo, v, id->size - 5);
- if (errcode == COUCHSTORE_SUCCESS) {
- memcpy((*pInfo)->id.buf, id->buf + 5, id->size - 5);
+ if (v == NULL) {
+ return COUCHSTORE_ERROR_DOC_NOT_FOUND;
}
-
- return errcode;
+ return by_id_read_docinfo(pInfo, id, v);
}
LIBCOUCHSTORE_API
@@ -433,7 +383,7 @@ couchstore_error_t couchstore_docinfo_by_id(Db *db,
DocInfo **pInfo)
{
sized_buf key;
- void *keylist = &key;
+ sized_buf *keylist = &key;
couchfile_lookup_request rq;
sized_buf cmptmp;
couchstore_error_t errcode;
@@ -446,7 +396,6 @@ couchstore_error_t couchstore_docinfo_by_id(Db *db,
key.size = idlen;
rq.cmp.compare = ebin_cmp;
- rq.cmp.from_ext = ebin_from_ext;
rq.cmp.arg = &cmptmp;
rq.db = db;
rq.num_keys = 1;
@@ -535,15 +484,14 @@ static couchstore_error_t byseq_do_callback(couchfile_lookup_request *rq,
sized_buf *seqterm = (sized_buf *) k;
int seqindex = 0;
DocInfo *docinfo;
- docinfo_from_buf(&docinfo, v, 0);
- ei_decode_uint64(seqterm->buf, &seqindex, &docinfo->db_seq);
+ by_seq_read_docinfo(&docinfo, seqterm, v);
union c99hack hack;
hack.voidptr = ((void **)rq->callback_ctx)[0];
- Db *db = ((void **)rq->callback_ctx)[1];
+ Db *db = ((void **)rq->callback_ctx)[1];
void *ctx = ((void **)rq->callback_ctx)[2];
- if (hack.callback(db, docinfo, ctx) == 0) {
+ if (hack.callback(db, docinfo, ctx) == 0) {
couchstore_free_docinfo(docinfo);
}
@@ -557,10 +505,9 @@ couchstore_error_t couchstore_changes_since(Db *db,
couchstore_changes_callback_fn callback,
void *ctx)
{
- (void)options;
- char since_termbuf[10];
+ char since_termbuf[6];
sized_buf since_term;
- void *keylist = &since_term;
+ sized_buf *keylist = &since_term;
void *cbctx[3];
couchfile_lookup_request rq;
sized_buf cmptmp;
@@ -573,15 +520,15 @@ couchstore_error_t couchstore_changes_since(Db *db,
}
since_term.buf = since_termbuf;
- since_term.size = 0;
- ei_encode_ulonglong(since_termbuf, (int *) &since_term.size, since);
+ since_term.size = 6;
+ memset(since_term.buf, 0, 6);
+ set_bits(since_term.buf, 0, 48, since);
cbctx[0] = hack.voidptr;
cbctx[1] = db;
cbctx[2] = ctx;
- rq.cmp.compare = long_term_cmp;
- rq.cmp.from_ext = term_from_ext;
+ rq.cmp.compare = seq_cmp;
rq.cmp.arg = &cmptmp;
rq.db = db;
rq.num_keys = 1;
@@ -615,29 +562,32 @@ static void copy_term(char *dst, int *idx, sized_buf *term)
*idx += term->size;
}
-static int assemble_index_value(DocInfo *docinfo, char *dst,
- sized_buf *first_term)
+static int assemble_seq_index_value(DocInfo *docinfo, char *dst)
+{
+ memset(dst, 0, 16);
+ set_bits(dst, 0, 12, docinfo->id.size);
+ set_bits(dst + 1, 4, 28, docinfo->size);
+ set_bits(dst + 5, 0, 1, docinfo->deleted);
+ set_bits(dst + 5, 1, 47, docinfo->bp);
+ dst[11] = docinfo->content_meta;
+ set_bits(dst + 12, 0, 32, docinfo->rev_seq); //16 bytes in.
+ memcpy(dst + 16, docinfo->id.buf, docinfo->id.size);
+ memcpy(dst + 16 + docinfo->id.size, docinfo->rev_meta.buf,
+ docinfo->rev_meta.size);
+ return 16 + docinfo->id.size + docinfo->rev_meta.size;
+}
+
+int assemble_id_index_value(DocInfo *docinfo, char *dst)
{
- int pos = 0;
- ei_encode_tuple_header(dst, &pos, 6); //2 bytes.
-
- //Id or Seq (possibly encoded as a binary)
- copy_term(dst, &pos, first_term); //first_term.size
- //Rev
- ei_encode_tuple_header(dst, &pos, 2); //3 bytes.
- ei_encode_ulonglong(dst, &pos, docinfo->rev_seq); //Max 10 bytes
- ei_encode_binary(dst, &pos, docinfo->rev_meta.buf, docinfo->rev_meta.size); //meta.size + 5
- //Bp
- ei_encode_ulonglong(dst, &pos, docinfo->bp); //Max 10 bytes
- //Deleted
- ei_encode_ulonglong(dst, &pos, docinfo->deleted); //2 bytes
- //
- ei_encode_ulonglong(dst, &pos, docinfo->content_meta); //2 bytes
- //Size
- ei_encode_ulonglong(dst, &pos, docinfo->size); //Max 10 bytes
-
- //Max 44 + first_term.size + meta.size bytes.
- return pos;
+ memset(dst, 0, 21);
+ set_bits(dst, 0, 48, docinfo->db_seq);
+ set_bits(dst + 6, 0, 32, docinfo->size);
+ set_bits(dst + 10, 0, 1, docinfo->deleted);
+ set_bits(dst + 10, 1, 47, docinfo->bp);
+ dst[16] = docinfo->content_meta;
+ set_bits(dst + 17, 0, 32, docinfo->rev_seq); //21 bytes in
+ memcpy(dst + 21, docinfo->rev_meta.buf, docinfo->rev_meta.size);
+ return 21 + docinfo->rev_meta.size;
}
static couchstore_error_t write_doc(Db *db, Doc *doc, uint64_t *bp,
@@ -659,7 +609,7 @@ static int id_action_compare(const void *actv1, const void *actv2)
act1 = (const couchfile_modify_action *) actv1;
act2 = (const couchfile_modify_action *) actv2;
- int cmp = ebin_cmp(act1->cmp_key, act2->cmp_key);
+ int cmp = ebin_cmp(act1->key, act2->key);
if (cmp == 0) {
if (act1->type < act2->type) {
return -1;
@@ -678,11 +628,9 @@ static int seq_action_compare(const void *actv1, const void *actv2)
act2 = (const couchfile_modify_action *) actv2;
uint64_t seq1, seq2;
- int pos = 0;
- ei_decode_uint64(act1->key->buf, &pos, &seq1);
- pos = 0;
- ei_decode_uint64(act2->key->buf, &pos, &seq2);
+ seq1 = get_48(act1->key->buf);
+ seq2 = get_48(act2->key->buf);
if (seq1 < seq2) {
return -1;
@@ -716,7 +664,8 @@ typedef struct _idxupdatectx {
static void idfetch_update_cb(couchfile_modify_request *rq,
sized_buf *k, sized_buf *v, void *arg)
{
- (void)k; (void)rq;
+ (void)k;
+ (void)rq;
//v contains a seq we need to remove ( {Seq,_,_,_,_} )
int termpos = 0;
uint64_t oldseq;
@@ -727,16 +676,14 @@ static void idfetch_update_cb(couchfile_modify_request *rq,
return;
}
- ei_decode_tuple_header(v->buf, &termpos, NULL);
- ei_decode_uint64(v->buf, &termpos, &oldseq);
+ oldseq = get_48(v->buf);
delbuf = (sized_buf *) fatbuf_get(ctx->deltermbuf, sizeof(sized_buf));
delbuf->buf = (char *) fatbuf_get(ctx->deltermbuf, 10);
delbuf->size = 0;
- ei_encode_ulonglong(delbuf->buf, (int *) &delbuf->size, oldseq);
ctx->seqacts[ctx->actpos].type = ACTION_REMOVE;
- ctx->seqacts[ctx->actpos].value.term = NULL;
+ ctx->seqacts[ctx->actpos].value.data = NULL;
ctx->seqacts[ctx->actpos].key = delbuf;
ctx->seqacts[ctx->actpos].cmp_key = delbuf;
@@ -793,7 +740,7 @@ static couchstore_error_t update_indexes(Db *db,
idacts[ii * 2].type = ACTION_FETCH;
idacts[ii * 2].value.arg = &fetcharg;
idacts[ii * 2 + 1].type = ACTION_INSERT;
- idacts[ii * 2 + 1].value.term = &idvals[ii];
+ idacts[ii * 2 + 1].value.data = &idvals[ii];
idacts[ii * 2].key = &ids[ii];
idacts[ii * 2].cmp_key = &idcmps[ii];
@@ -806,7 +753,6 @@ static couchstore_error_t update_indexes(Db *db,
id_action_compare);
idrq.cmp.compare = ebin_cmp;
- idrq.cmp.from_ext = ebin_from_ext;
idrq.cmp.arg = &tmpsb;
idrq.db = db;
idrq.actions = idacts;
@@ -824,7 +770,7 @@ static couchstore_error_t update_indexes(Db *db,
while (fetcharg.valpos < numdocs) {
seqacts[fetcharg.actpos].type = ACTION_INSERT;
- seqacts[fetcharg.actpos].value.term = &seqvals[fetcharg.valpos];
+ seqacts[fetcharg.actpos].value.data = &seqvals[fetcharg.valpos];
seqacts[fetcharg.actpos].key = &seqs[fetcharg.valpos];
seqacts[fetcharg.actpos].cmp_key = &seqs[fetcharg.valpos];
fetcharg.valpos++;
@@ -835,8 +781,7 @@ static couchstore_error_t update_indexes(Db *db,
qsort(seqacts, fetcharg.actpos, sizeof(couchfile_modify_action),
seq_action_compare);
- seqrq.cmp.compare = long_term_cmp;
- seqrq.cmp.from_ext = term_from_ext;
+ seqrq.cmp.compare = seq_cmp;
seqrq.cmp.arg = &tmpsb;
seqrq.actions = seqacts;
seqrq.num_actions = fetcharg.actpos;
@@ -879,17 +824,11 @@ static couchstore_error_t add_doc_to_update_list(Db *db,
DocInfo updated = *info;
updated.db_seq = seq;
- seqterm->buf = fatbuf_get(fb, 10);
- idterm->buf = fatbuf_get(fb, updated.id.size + 5);
- seqval->buf = fatbuf_get(fb, (44 + updated.id.size + updated.rev_meta.size));
- idval->buf = fatbuf_get(fb, (44 + 10 + updated.rev_meta.size));
-
- if (!(seqterm->buf && idterm->buf && seqval->buf && idval->buf)) {
- return COUCHSTORE_ERROR_ALLOC_FAIL;
- }
- seqterm->size = 0;
- idterm->size = 0;
- ei_encode_ulonglong(seqterm->buf, (int *) &seqterm->size, seq);
+ seqterm->buf = (char *) fatbuf_get(fb, 6);
+ seqterm->size = 6;
+ error_unless(seqterm->buf, COUCHSTORE_ERROR_ALLOC_FAIL);
+ memset(seqterm->buf, 0, 6);
+ set_bits(seqterm->buf, 0, 48, seq);
if (doc) {
if ((options & COMPRESS_DOC_BODIES) && (info->content_meta & SNAPPY_META_FLAG)) {
@@ -908,13 +847,18 @@ static couchstore_error_t add_doc_to_update_list(Db *db,
updated.size = 0;
}
- ei_encode_binary(idterm->buf, (int *) &idterm->size, updated.id.buf, updated.id.size);
- seqval->size = assemble_index_value(&updated, seqval->buf, idterm);
- idval->size = assemble_index_value(&updated, idval->buf, seqterm);
+ *idterm = updated.id;
- //Use max of 10 +, id.size + 5 +, 42 + rev_meta.size + id.size, + 52 + rev_meta.size
- // == id.size *2 + rev_meta.size *2 + 109 bytes
+ seqval->buf = (char *) fatbuf_get(fb, (44 + updated.id.size + updated.rev_meta.size));
+ error_unless(seqval->buf, COUCHSTORE_ERROR_ALLOC_FAIL);
+ seqval->size = assemble_seq_index_value(&updated, seqval->buf);
+ idval->buf = (char *) fatbuf_get(fb, (44 + 10 + updated.rev_meta.size));
+ error_unless(idval->buf, COUCHSTORE_ERROR_ALLOC_FAIL);
+ idval->size = assemble_id_index_value(&updated, idval->buf);
+
+ //We use 37 + id.size + 2 * rev_meta.size bytes
+cleanup:
return errcode;
}
@@ -936,8 +880,8 @@ couchstore_error_t couchstore_save_documents(Db *db,
for (ii = 0; ii < numdocs; ii++) {
// Get additional size for terms to be inserted into indexes
- size_t size = infos[ii]->id.size + infos[ii]->rev_meta.size;
- term_meta_size += 113 + (2 * size);
+ size_t size = 37 + infos[ii]->id.size + 2 * infos[ii]->rev_meta.size;
+ term_meta_size += (2 * size);
}
fb = fatbuf_alloc(term_meta_size +
@@ -1008,16 +952,16 @@ static couchstore_error_t local_doc_fetch(couchfile_lookup_request *rq,
}
dp = *lDoc = (LocalDoc *) fatbuf_get(ldbuf, sizeof(LocalDoc));
- dp->id.buf = (char *) fatbuf_get(ldbuf, id->size - 5);
- dp->id.size = id->size - 5;
+ dp->id.buf = (char *) fatbuf_get(ldbuf, id->size);
+ dp->id.size = id->size;
- dp->json.buf = (char *) fatbuf_get(ldbuf, v->size - 5);
- dp->json.size = v->size - 5;
+ dp->json.buf = (char *) fatbuf_get(ldbuf, v->size);
+ dp->json.size = v->size;
dp->deleted = 0;
- memcpy(dp->id.buf, id->buf + 5, id->size - 5);
- memcpy(dp->json.buf, v->buf + 5, v->size - 5);
+ memcpy(dp->id.buf, id->buf, id->size);
+ memcpy(dp->json.buf, v->buf, v->size);
return COUCHSTORE_SUCCESS;
}
@@ -1029,7 +973,7 @@ couchstore_error_t couchstore_open_local_document(Db *db,
LocalDoc **pDoc)
{
sized_buf key;
- void *keylist = &key;
+ sized_buf *keylist = &key;
couchfile_lookup_request rq;
sized_buf cmptmp;
couchstore_error_t errcode;
@@ -1042,7 +986,6 @@ couchstore_error_t couchstore_open_local_document(Db *db,
key.size = idlen;
rq.cmp.compare = ebin_cmp;
- rq.cmp.from_ext = ebin_from_ext;
rq.cmp.arg = &cmptmp;
rq.db = db;
rq.num_keys = 1;
@@ -1065,39 +1008,23 @@ couchstore_error_t couchstore_save_local_document(Db *db, LocalDoc *lDoc)
{
couchstore_error_t errcode;
couchfile_modify_action ldupdate;
- fatbuf *binbufs = fatbuf_alloc(10 + lDoc->id.size + lDoc->json.size);
sized_buf idterm;
sized_buf jsonterm;
sized_buf cmptmp;
node_pointer *nroot = NULL;
- if (binbufs == NULL) {
- return COUCHSTORE_ERROR_ALLOC_FAIL;
- }
-
if (lDoc->deleted) {
ldupdate.type = ACTION_REMOVE;
} else {
ldupdate.type = ACTION_INSERT;
}
- idterm.buf = (char *) fatbuf_get(binbufs, lDoc->id.size + 5);
- idterm.size = 0;
- ei_encode_binary(idterm.buf, (int *) &idterm.size, lDoc->id.buf,
- lDoc->id.size);
-
- jsonterm.buf = (char *) fatbuf_get(binbufs, lDoc->json.size + 5);
- jsonterm.size = 0;
- ei_encode_binary(jsonterm.buf, (int *) &jsonterm.size,
- lDoc->json.buf, lDoc->json.size);
-
- ldupdate.cmp_key = (void *) &lDoc->id;
- ldupdate.key = &idterm;
- ldupdate.value.term = &jsonterm;
+ ldupdate.cmp_key = &lDoc->id;
+ ldupdate.key = &lDoc->id;
+ ldupdate.value.data = &lDoc->json;
couchfile_modify_request rq;
rq.cmp.compare = ebin_cmp;
- rq.cmp.from_ext = ebin_from_ext;
rq.cmp.arg = &cmptmp;
rq.num_actions = 1;
rq.actions = &ldupdate;
@@ -1112,7 +1039,6 @@ couchstore_error_t couchstore_save_local_document(Db *db, LocalDoc *lDoc)
db->header.local_docs_root = nroot;
}
- fatbuf_free(binbufs);
return errcode;
}
View
17 src/couch_file_read.c
@@ -6,7 +6,6 @@
#include <string.h>
#include <netinet/in.h>
#include <snappy-c.h>
-#include <ei.h>
#include "internal.h"
#include "crc32.h"
@@ -106,13 +105,11 @@ static int pread_bin_int(Db *db, off_t pos, char **ret_ptr, int header)
chunk_len = ntohl(chunk_len) & ~0x80000000;
skip += 4;
if (header) {
- chunk_len -= 16; //Header len includes hash len.
- skip += 16; //Header is still md5, and the md5 is always present.
- } else if (prefix) {
- memcpy(&crc32, bufptr + 4, 4);
- crc32 = ntohl(crc32);
- skip += 4;
+ chunk_len -= 4; //Header len includes hash len.
}
+ memcpy(&crc32, bufptr + 4, 4);
+ crc32 = ntohl(crc32);
+ skip += 4;
if (chunk_len == 0) {
free(bufptr);
@@ -134,7 +131,8 @@ static int pread_bin_int(Db *db, off_t pos, char **ret_ptr, int header)
return chunk_len;
} else {
int rest_len = raw_read(db, &pos, chunk_len - buf_len, &bufptr_rest);
- error_unless(rest_len > 0, COUCHSTORE_ERROR_READ);
+ error_unless(rest_len != -1, COUCHSTORE_ERROR_READ);
+ error_unless(rest_len + buf_len == chunk_len, COUCHSTORE_ERROR_READ);
newbufptr = (char *) realloc(bufptr, buf_len + rest_len);
error_unless(newbufptr, COUCHSTORE_ERROR_READ);
@@ -173,12 +171,14 @@ int pread_compressed(Db *db, off_t pos, char **ret_ptr)
!= SNAPPY_OK) {
//should be compressed but snappy doesn't see it as valid.
free(*ret_ptr);
+ *ret_ptr = NULL;
return -1;
}
new_buf = (char *) malloc(new_len);
if (!new_buf) {
free(*ret_ptr);
+ *ret_ptr = NULL;
return -1;
}
snappy_status ss = (snappy_uncompress((*ret_ptr), len, new_buf, &new_len));
@@ -188,6 +188,7 @@ int pread_compressed(Db *db, off_t pos, char **ret_ptr)
return new_len;
} else {
free(*ret_ptr);
+ *ret_ptr = NULL;
return -1;
}
}
View
17 src/couch_file_write.c
@@ -52,11 +52,10 @@ couchstore_error_t db_write_header(Db *db, sized_buf *buf, off_t *pos)
off_t write_pos = db->file_pos;
ssize_t written;
char blockheader = 1;
- uint32_t size = htonl(buf->size + 16); //Len before header includes hash len.
+ uint32_t size = htonl(buf->size + 4); //Len before header includes hash len.
+ uint32_t crc32 = htonl(hash_crc32(buf->buf, buf->size));
sized_buf lenbuf = { (char *) &size, 4 };
- MD5_CTX hashctx;
- char hash[16];
- sized_buf hashbuf = { hash, 16 };
+ sized_buf crcbuf = { (char *) &crc32, 4 };
if (write_pos % COUCH_BLOCK_SIZE != 0) {
write_pos += COUCH_BLOCK_SIZE - (write_pos % COUCH_BLOCK_SIZE); //Move to next block boundary.
}
@@ -75,12 +74,8 @@ couchstore_error_t db_write_header(Db *db, sized_buf *buf, off_t *pos)
}
write_pos += written;
- //Write MD5
- MD5Init(&hashctx);
- MD5Update(&hashctx, (uint8_t *) buf->buf, buf->size);
- MD5Final((uint8_t *) hash, &hashctx);
-
- written = raw_write(db, &hashbuf, write_pos);
+ //Write CRC32
+ written = raw_write(db, &crcbuf, write_pos);
if (written < 0) {
return COUCHSTORE_ERROR_WRITE;
}
@@ -144,7 +139,7 @@ int db_write_buf_compressed(Db *db, sized_buf *buf, off_t *pos)
error_unless(to_write.buf, COUCHSTORE_ERROR_ALLOC_FAIL);
error_unless(snappy_compress(buf->buf, buf->size, to_write.buf,
&to_write.size) == SNAPPY_OK,
- COUCHSTORE_ERROR_WRITE);
+ COUCHSTORE_ERROR_WRITE);
error_pass(db_write_buf(db, &to_write, pos));
cleanup:
View
1  src/couchscript.cc
@@ -2,6 +2,7 @@
#include "config.h"
#include <iostream>
#include <cassert>
+#include <cstdlib>
#include <sysexits.h>
#include <stdlib.h>
#include <unistd.h>
View
12 src/dbdump.c
@@ -6,6 +6,7 @@
#include <inttypes.h>
#include <libcouchstore/couch_db.h>
#include <snappy-c.h>
+#include "bitfield.h"
#define SNAPPY_FLAG 128
@@ -24,15 +25,18 @@ static int foldprint(Db *db, DocInfo *docinfo, void *ctx)
Doc *doc;
uint64_t cas;
uint32_t expiry, flags;
- cas = htonll(*((uint64_t *) docinfo->rev_meta.buf));
- expiry = htonl(*((uint32_t *) (docinfo->rev_meta.buf + 8)));
- flags = htonl(*((uint32_t *) (docinfo->rev_meta.buf + 12)));
couchstore_open_doc_with_docinfo(db, docinfo, &doc, 0);
printf("Doc seq: %"PRIu64"\n", docinfo->db_seq);
printf(" id: ");
printsb(&docinfo->id);
+ printf(" rev: %"PRIu64"\n", docinfo->rev_seq);
printf(" content_meta: %d\n", docinfo->content_meta);
- printf(" cas: %"PRIu64", expiry: %"PRIu32", flags: %"PRIu32"\n", cas, expiry, flags);
+ if (docinfo->rev_meta.size == 16) {
+ cas = ntohll(*((uint64_t *)docinfo->rev_meta.buf));
+ expiry = get_32(docinfo->rev_meta.buf + 8);
+ flags = get_32(docinfo->rev_meta.buf + 12);
+ printf(" cas: %"PRIu64", expiry: %"PRIu32", flags: %"PRIu32"\n", cas, expiry, flags);
+ }
if (docinfo->deleted) {
printf(" doc deleted\n");
}
View
10 src/dbinfo.c
@@ -6,10 +6,10 @@
#include <inttypes.h>
#include <libcouchstore/couch_db.h>
#include <snappy-c.h>
-#include <ei.h>
#include "internal.h"
#include "util.h"
+#include "bitfield.h"
static char *size_str(double size)
{
@@ -26,16 +26,14 @@ static char *size_str(double size)
static uint64_t id_reduce_info(node_pointer *root)
{
- int pos = 0;
uint64_t total, deleted, size;
if (root == NULL) {
printf(" no documents\n");
return 0;
}
- ei_decode_tuple_header(root->reduce_value.buf, &pos, NULL);
- ei_decode_uint64(root->reduce_value.buf, &pos, &total);
- ei_decode_uint64(root->reduce_value.buf, &pos, &deleted);
- ei_decode_uint64(root->reduce_value.buf, &pos, &size);
+ total = get_40(root->reduce_value.buf);
+ deleted = get_40(root->reduce_value.buf + 5);
+ size = get_48(root->reduce_value.buf + 10);
printf(" doc count: %"PRIu64"\n", total);
printf(" deleted doc count: %"PRIu64"\n", deleted);
printf(" data size: %s\n", size_str(size));
View
7 src/internal.h
@@ -12,7 +12,7 @@
#include <libcouchstore/couch_db.h>
#define COUCH_BLOCK_SIZE 4096
-#define COUCH_DISK_VERSION 9
+#define COUCH_DISK_VERSION 10
#define COUCH_SNAPPY_THRESHOLD 64
#ifdef __cplusplus
@@ -33,7 +33,7 @@ extern "C" {
node_pointer *by_seq_root;
node_pointer *local_docs_root;
uint64_t purge_seq;
- sized_buf *purged_docs;
+ uint64_t purge_ptr;
uint64_t position;
} db_header;
@@ -63,6 +63,9 @@ extern "C" {
int db_write_buf(Db *db, sized_buf *buf, off_t *pos);
int db_write_buf_compressed(Db *db, sized_buf *buf, off_t *pos);
+ node_pointer *read_root(char *buf, int size);
+ void encode_root(char *buf, node_pointer *node);
+
#ifdef __cplusplus
}
#endif
View
126 src/reduces.c
@@ -1,119 +1,71 @@
/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include <stdlib.h>
-#include <ei.h>
+#include <string.h>
#include "reduces.h"
+#include "bitfield.h"
-void by_seq_reduce (sized_buf *dst, nodelist *leaflist, int count)
+void by_seq_reduce (char *dst, size_t *size_r, nodelist *leaflist, int count)
{
(void)leaflist;
- //will be freed by flush_mr
- dst->buf = (char *) malloc(12);
- if (!dst->buf) {
- return;
- }
- int pos = 0;
- ei_encode_long(dst->buf, &pos, count);
- dst->size = pos;
+ memset(dst, 0, 5);
+ set_bits(dst, 0, 40, count);
+ *size_r = 5;
}
-void by_seq_rereduce (sized_buf *dst, nodelist *leaflist, int count)
+void by_seq_rereduce (char *dst, size_t *size_r, nodelist *ptrlist, int count)
{
(void)count;
- long total = 0;
- long current = 0;
- int r_pos = 0;
- int pos = 0;
-
- //will be freed by flush_mr
- dst->buf = (char *) malloc(12);
- if (!dst->buf) {
- return;
- }
- nodelist *i = leaflist;
+ uint64_t total = 0;
+ nodelist *i = ptrlist;
while (i != NULL) {
- r_pos = 0;
- ei_decode_long(i->value.pointer->reduce_value.buf, &r_pos, &current);
- total += current;
+ total += get_40(i->pointer->reduce_value.buf);
i = i->next;
}
- ei_encode_long(dst->buf, &pos, total);
- dst->size = pos;
+ memset(dst, 0, 5);
+ set_bits(dst, 0, 40, total);
+ *size_r = 5;
}
-void by_id_rereduce(sized_buf *dst, nodelist *leaflist, int count)
+void by_id_reduce(char *dst, size_t *size_r, nodelist *leaflist, int count)
{
(void)count;
- //Source term {NotDeleted, Deleted, Size}
- //Result term {NotDeleted, Deleted, Size}
- dst->buf = (char *) malloc(30);
- if (!dst->buf) {
- return;
- }
- int dstpos = 0;
- int srcpos = 0;
- long notdeleted = 0, deleted = 0, size = 0;
+ uint64_t notdeleted = 0, deleted = 0, size = 0;
nodelist *i = leaflist;
while (i != NULL) {
- srcpos = 0;
- long long src_deleted = 0;
- long long src_notdeleted = 0;
- long long src_size = 0;
- ei_decode_tuple_header(i->value.pointer->reduce_value.buf, &srcpos, NULL);
- ei_decode_longlong(i->value.pointer->reduce_value.buf, &srcpos, &src_notdeleted);
- ei_decode_longlong(i->value.pointer->reduce_value.buf, &srcpos, &src_deleted);
- ei_decode_longlong(i->value.pointer->reduce_value.buf, &srcpos, &src_size);
- size += src_size;
- deleted += src_deleted;
- notdeleted += src_notdeleted;
+ int isdeleted = i->data.buf[10] >> 7;
+ deleted += isdeleted;
+ if (!isdeleted) {
+ notdeleted++;
+ }
+ size += get_32(i->data.buf + 6);
+
i = i->next;
}
- ei_encode_tuple_header(dst->buf, &dstpos, 3);
- ei_encode_longlong(dst->buf, &dstpos, notdeleted);
- ei_encode_longlong(dst->buf, &dstpos, deleted);
- ei_encode_longlong(dst->buf, &dstpos, size);
- dst->size = dstpos;
+ memset(dst, 0, 16);
+ set_bits(dst, 0, 40, notdeleted);
+ set_bits(dst + 5, 0, 40, deleted);
+ set_bits(dst + 10, 0, 48, size);
+ *size_r = 16;
}
-void by_id_reduce(sized_buf *dst, nodelist *leaflist, int count)
+void by_id_rereduce(char *dst, size_t *size_r, nodelist *ptrlist, int count)
{
(void)count;
- //Source term {Key, {Seq, Rev, Bp, Deleted, ContentMeta, Size}}
- //Result term {NotDeleted, Deleted, Size}
- dst->buf = (char *) malloc(30);
- if (!dst->buf) {
- return;
- }
- int dstpos = 0;
- int srcpos = 0;
- long notdeleted = 0, deleted = 0, size = 0;
- nodelist *i = leaflist;
+ uint64_t notdeleted = 0, deleted = 0, size = 0;
+
+ nodelist *i = ptrlist;
while (i != NULL) {
- srcpos = 0;
- long src_deleted = 0;
- long long src_size = 0;
- ei_decode_tuple_header(i->value.leaf->buf, &srcpos, NULL);
- ei_skip_term(i->value.leaf->buf, &srcpos); //skip key
- ei_decode_tuple_header(i->value.leaf->buf, &srcpos, NULL);
- ei_skip_term(i->value.leaf->buf, &srcpos); //skip seq
- ei_skip_term(i->value.leaf->buf, &srcpos); //skip rev
- ei_skip_term(i->value.leaf->buf, &srcpos); //skip bp
- ei_decode_long(i->value.leaf->buf, &srcpos, &src_deleted);
- ei_skip_term(i->value.leaf->buf, &srcpos); //skip ContentMeta
- ei_decode_longlong(i->value.leaf->buf, &srcpos, &src_size);
- if (src_deleted == 1) {
- deleted++;
- } else {
- notdeleted++;
- }
+ notdeleted += get_40(i->pointer->reduce_value.buf);
+ deleted += get_40(i->pointer->reduce_value.buf + 5);
+ size += get_48(i->pointer->reduce_value.buf + 10);
- size += src_size;
i = i->next;
}
- ei_encode_tuple_header(dst->buf, &dstpos, 3);
- ei_encode_long(dst->buf, &dstpos, notdeleted);
- ei_encode_long(dst->buf, &dstpos, deleted);
- ei_encode_longlong(dst->buf, &dstpos, size);
- dst->size = dstpos;
+ memset(dst, 0, 16);
+ set_bits(dst, 0, 40, notdeleted);
+ set_bits(dst + 5, 0, 40, deleted);
+ set_bits(dst + 10, 0, 48, size);
+ *size_r = 16;
}