Skip to content

Commit

Permalink
MB-11047 new api fdb_iterator_seek to fast forward to a key
Browse files Browse the repository at this point in the history 
If there are multiple iterators over a key range, this api can
be used to bring a slower iterator up to speed with the faster one.

Simple implementation that re-positions trie and avl tree cursors.

Feature test case added
Change-Id: If0ec438185a638faffd279c2dc3960f52484dd35
  • Loading branch information
@hisundar @greensky00
hisundar authored and greensky00 committed May 21, 2014
1 parent 4ab05a3 commit 6bb679c
Show file tree
Hide file tree
Showing 3 changed files with 210 additions and 40 deletions.
13 changes: 13 additions & 0 deletions include/libforestdb/forestdb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,19 @@ LIBFDB_API
fdb_status fdb_iterator_next_metaonly(fdb_iterator *iterator,
fdb_doc **doc);

/**
* Fast forward an iterator to return documents after the given seek_key
* If the key does not exist, seek forward to the next sorted key.
*
* @param iterator Pointer to the iterator.
* @param seek_key Pointer to the key to seek to.
* @param seek_keylen Length of the seek_key
* @return FDB_RESULT_SUCCESS on success.
*/
LIBFDB_API
fdb_status fdb_iterator_seek(fdb_iterator *iterator, const void *seek_key,
const size_t seek_keylen);

/**
* Close the iterator and free its associated resources.
*
Expand Down
131 changes: 91 additions & 40 deletions src/iterator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,22 @@ int _fdb_wal_cmp(struct avl_node *a, struct avl_node *b, void *aux)
}
}

int _fdb_key_cmp(fdb_iterator *iterator, void *key1, size_t keylen1,
void *key2, size_t keylen2) {
int cmp;
if (iterator->handle.config.cmp_fixed) {
// custom compare function for fixed size key
cmp = iterator->handle.config.cmp_fixed(key1, key2);
} else if (iterator->handle.config.cmp_variable) {
// custom compare function for variable length key
cmp = iterator->handle.config.cmp_variable(key1, keylen1,
key2, keylen2);
} else {
cmp = _fdb_keycmp(key1, keylen1, key2, keylen2);
}
return cmp;
}

fdb_status fdb_iterator_init(fdb_handle *handle,
fdb_iterator **ptr_iterator,
const void *start_key,
Expand Down Expand Up @@ -169,20 +185,9 @@ fdb_status fdb_iterator_init(fdb_handle *handle,
while(e) {
wal_item = _get_entry(e, struct wal_item, list_elem);
if (start_key) {
if (handle->config.cmp_fixed) {
// custom compare function for fixed size key
cmp = handle->config.cmp_fixed((void*)start_key,
wal_item->key);
} else if (handle->config.cmp_variable) {
// custom compare function for variable length key
cmp = handle->config.cmp_variable(
(void*)start_key, start_keylen,
wal_item->key, wal_item->keylen);
} else {
cmp = _fdb_keycmp((void *)start_key, start_keylen,
wal_item->key, wal_item->keylen);
}
}else{
cmp = _fdb_key_cmp(iterator, (void *)start_key, start_keylen,
wal_item->key, wal_item->keylen);
} else {
cmp = 0;
}

Expand Down Expand Up @@ -356,19 +361,9 @@ static fdb_status _fdb_iterator_next(fdb_iterator *iterator,
// get the current item of avl-tree
snap_item = _get_entry(iterator->tree_cursor, struct snap_wal_entry, avl);
if (hr != HBTRIE_RESULT_FAIL) {
if (iterator->handle.config.cmp_fixed) {
// custom compare function for fixed size key
cmp = iterator->handle.config.cmp_fixed(
snap_item->key, key);
} else if (iterator->handle.config.cmp_variable) {
// custom compare function for variable length key
cmp = iterator->handle.config.cmp_variable(
snap_item->key, snap_item->keylen,
key, keylen);
} else {
cmp = _fdb_keycmp(snap_item->key, snap_item->keylen, key, keylen);
}
}else{
cmp = _fdb_key_cmp(iterator, snap_item->key, snap_item->keylen,
key, keylen);
} else {
// no more docs in hb-trie
cmp = -1;
}
Expand Down Expand Up @@ -408,19 +403,8 @@ static fdb_status _fdb_iterator_next(fdb_iterator *iterator,
}

if (iterator->end_key) {

if (iterator->handle.config.cmp_fixed) {
// custom compare function for fixed size key
cmp = iterator->handle.config.cmp_fixed(
iterator->end_key, key);
} else if (iterator->handle.config.cmp_variable) {
// custom compare function for variable length key
cmp = iterator->handle.config.cmp_variable(
iterator->end_key, iterator->end_keylen,
key, keylen);
} else {
cmp = _fdb_keycmp(iterator->end_key, iterator->end_keylen, key, keylen);
}
cmp = _fdb_key_cmp(iterator, iterator->end_key, iterator->end_keylen,
key, keylen);

if (cmp < 0) {
// current key (KEY) is lexicographically greater than END_KEY
Expand Down Expand Up @@ -474,6 +458,73 @@ static fdb_status _fdb_iterator_next(fdb_iterator *iterator,
return FDB_RESULT_SUCCESS;
}

fdb_status fdb_iterator_seek(fdb_iterator *iterator, const void *seek_key,
const size_t seek_keylen) {
hbtrie_result hr = HBTRIE_RESULT_SUCCESS;
btree_result br;
struct snap_wal_entry *snap_item = NULL;
if (!iterator || !seek_key || !iterator->_key) {
return FDB_RESULT_INVALID_ARGS;
}

// disable seeking beyond the end key...
if (iterator->end_key && _fdb_key_cmp(iterator, (void *)iterator->end_key,
iterator->end_keylen,
(void *)seek_key, seek_keylen) <= 0) {
memcpy(iterator->_key, seek_key, seek_keylen);
iterator->_keylen = seek_keylen;
iterator->_offset = 0;
iterator->tree_cursor = NULL;
return FDB_RESULT_ITERATOR_FAIL;
}

// Roll the hb-trie/btree iterator forward to seek key
while (hr == HBTRIE_RESULT_SUCCESS) {
if (_fdb_key_cmp(iterator, (void *)seek_key, seek_keylen,
(void *)iterator->_key, iterator->_keylen) <= 0) {
// iterator->_key and iterator->_offset will help return seek_key
// on fdb_iterator_next() if we went past & pulled out a higher key
break;
}

// get next key from hb-trie (or idtree)
if (iterator->handle.config.cmp_variable) {
uint8_t *var_key = alca(uint8_t,
iterator->handle.config.chunksize);
memset(var_key, 0, iterator->handle.config.chunksize);

br = btree_next(iterator->idtree_iterator, var_key,
(void*)&iterator->_offset);
if (br == BTREE_RESULT_FAIL) {
hr = HBTRIE_RESULT_FAIL;
} else {
_get_var_key(var_key, iterator->_key, &iterator->_keylen);
_free_var_key(var_key);
}
} else {
hr = hbtrie_next(iterator->hbtrie_iterator, iterator->_key,
&iterator->_keylen, (void*)&iterator->_offset);
}
btreeblk_end(iterator->handle.bhandle);
iterator->_offset = _endian_decode(iterator->_offset);
}

// Roll the WAL iterator forward too to the seek_key
while (iterator->tree_cursor) {
int cmp;
snap_item = _get_entry(iterator->tree_cursor, struct snap_wal_entry,
avl);
cmp = _fdb_key_cmp(iterator, (void *)snap_item->key, snap_item->keylen,
(void *)seek_key, seek_keylen);
if ( cmp < 0) {
iterator->tree_cursor = avl_next(iterator->tree_cursor);
} else {
break;
}
}
return FDB_RESULT_SUCCESS;
}

// DOC returned by this function must be freed using 'fdb_doc_free'
static fdb_status _fdb_iterator_seq_next(fdb_iterator *iterator,
fdb_doc **doc)
Expand Down
106 changes: 106 additions & 0 deletions tests/forestdb_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1537,6 +1537,111 @@ void iterator_test()
TEST_RESULT("iterator test");
}

void iterator_seek_test()
{
TEST_INIT();

memleak_start();

int i, r;
int n = 10;
uint64_t offset;
fdb_handle *db;
fdb_doc **doc = alca(fdb_doc*, n);
fdb_doc *rdoc;
fdb_status status;
fdb_iterator *iterator;

char keybuf[256], metabuf[256], bodybuf[256], temp[256];

// remove previous dummy files
r = system(SHELL_DEL" dummy* > errorlog.txt");

fdb_config fconfig = fdb_get_default_config();
fconfig.buffercache_size = 0;
fconfig.wal_threshold = 1024;
fconfig.flags = FDB_OPEN_FLAG_CREATE;
fconfig.compaction_threshold = 0;

// open db
fdb_open(&db, "./dummy1", &fconfig);
status = fdb_set_log_callback(db, logCallbackFunc,
(void *) "iterator_seek_test");

// insert documents of odd number
for (i=1;i<n;i+=2){
sprintf(keybuf, "key%d", i);
sprintf(metabuf, "meta%d", i);
sprintf(bodybuf, "body%d", i);
fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
(void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
fdb_set(db, doc[i]);
}
// manually flush WAL & commit
fdb_commit(db, FDB_COMMIT_MANUAL_WAL_FLUSH);

// insert documents of even number
for (i=0;i<n;i+=2){
sprintf(keybuf, "key%d", i);
sprintf(metabuf, "meta%d", i);
sprintf(bodybuf, "body%d", i);
fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
(void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
fdb_set(db, doc[i]);
}
// commit without WAL flush
fdb_commit(db, FDB_COMMIT_NORMAL);

// now odd number docs are in hb-trie & even number docs are in WAL

// create an iterator for full range
fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);

// seek current iterator to inside the WAL's avl tree..
status = fdb_iterator_next(iterator, &rdoc);
TEST_CHK(status == FDB_RESULT_SUCCESS);

TEST_CHK(!memcmp(rdoc->key, doc[0]->key, rdoc->keylen));
TEST_CHK(!memcmp(rdoc->meta, doc[0]->meta, rdoc->metalen));
TEST_CHK(!memcmp(rdoc->body, doc[0]->body, rdoc->bodylen));
fdb_doc_free(rdoc);

// seek forward to 2nd key ..
status = fdb_iterator_seek(iterator, doc[2]->key, strlen(keybuf));
TEST_CHK(status == FDB_RESULT_SUCCESS);

// repeat until fail
i=2;
while(1){
status = fdb_iterator_next(iterator, &rdoc);
if (status == FDB_RESULT_ITERATOR_FAIL) break;

TEST_CHK(!memcmp(rdoc->key, doc[i]->key, rdoc->keylen));
TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));

fdb_doc_free(rdoc);
i++;
};
TEST_CHK(i==10);
fdb_iterator_close(iterator);

// close db file
fdb_close(db);

// free all documents
for (i=0;i<n;++i){
fdb_doc_free(doc[i]);
}

// free all resources
fdb_shutdown();

memleak_end();

TEST_RESULT("iterator seek test");
}

void sequence_iterator_test()
{
TEST_INIT();
Expand Down Expand Up @@ -3087,6 +3192,7 @@ int main(){
#endif
incomplete_block_test();
iterator_test();
iterator_seek_test();
sequence_iterator_test();
custom_compare_primitive_test();
custom_compare_variable_test();
Expand Down

0 comments on commit 6bb679c

Please sign in to comment.