Skip to content

Commit

Permalink
ForestDB optimizations
Browse files Browse the repository at this point in the history 
Avoid loading document bodies unless necessary.
  • Loading branch information
@snej
snej committed Apr 25, 2016
1 parent 33b4609 commit 94625e3
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 42 deletions.
3 changes: 3 additions & 0 deletions Source/CBLForestBridge.h
View file
Expand Up @@ -75,6 +75,9 @@ using namespace CBL;

@interface CBLForestBridge : NSObject

+ (CBL_MutableRevision*) revisionObjectFromForestDocInfo: (C4DocumentInfo&)docInfo
status: (CBLStatus*)outStatus;

+ (CBL_MutableRevision*) revisionObjectFromForestDoc: (C4Document*)doc
docID: (NSString*)docIDIfKnown
revID: (CBL_RevID*)revIDIfKnown
Expand Down
12 changes: 12 additions & 0 deletions Source/CBLForestBridge.mm
View file
Expand Up @@ -244,6 +244,18 @@ C4EncryptionKey symmetricKey2Forest(CBLSymmetricKey* key) {
@implementation CBLForestBridge


+ (CBL_MutableRevision*) revisionObjectFromForestDocInfo: (C4DocumentInfo&)docInfo
status: (CBLStatus*)outStatus
{
CBL_MutableRevision* result;
result = [[CBL_MutableRevision alloc] initWithDocID: slice2string(docInfo.docID)
revID: slice2revID(docInfo.revID)
deleted: (docInfo.flags & kRevDeleted) != 0];
result.sequence = docInfo.sequence;
return result;
}


+ (CBL_MutableRevision*) revisionObjectFromForestDoc: (C4Document*)doc
docID: (UU NSString*)docID
revID: (CBL_RevID*)revID
Expand Down
39 changes: 24 additions & 15 deletions Source/CBL_ForestDBDocEnumerator.mm
View file
Expand Up @@ -34,7 +34,7 @@ - (instancetype) initWithStorage: (CBL_ForestDBStorage*)storage
_storage = storage;
C4EnumeratorOptions c4options = {0, 0};
_includeDocs = (options->includeDocs || options.filter);
if (_includeDocs || options->allDocsMode >= kCBLShowConflicts)
if (_includeDocs || options->allDocsMode == kCBLOnlyConflicts)
c4options.flags |= kC4IncludeBodies;
if (options->descending)
c4options.flags |= kC4Descending;
Expand Down Expand Up @@ -94,11 +94,10 @@ - (CBLQueryRow*) generateNextRow {
return nil;
C4Error c4err;
while (c4enum_next(_enum, &c4err)) {
CLEANUP(C4Document)* doc = c4enum_getDocument(_enum, &c4err);
if (!doc)
break;
NSString* docID = slice2string(doc->docID);
if (!(doc->flags & kExists)) {
C4DocumentInfo docInfo;
c4enum_getDocumentInfo(_enum, &docInfo);
NSString* docID = slice2string(docInfo.docID);
if (!(docInfo.flags & kExists)) {
LogVerbose(Query, @"AllDocs: No such row with key=\"%@\"", docID);
return [[CBLQueryRow alloc] initWithDocID: nil
sequence: 0
Expand All @@ -107,17 +106,28 @@ - (CBLQueryRow*) generateNextRow {
docRevision: nil];
}

bool deleted = (doc->flags & kDeleted) != 0;
bool deleted = (docInfo.flags & kDeleted) != 0;
bool conflicted = (docInfo.flags & kConflicted) != 0;
if (deleted && _allDocsMode != kCBLIncludeDeleted && !_byKey)
continue; // skip deleted doc
if (!(doc->flags & kConflicted) && _allDocsMode == kCBLOnlyConflicts)
if (!conflicted && _allDocsMode == kCBLOnlyConflicts)
continue; // skip non-conflicted doc
if (_skip > 0) {
--_skip;
continue;
}

CBL_RevID* revID = slice2revID(doc->revID);
CBL_RevID* revID = slice2revID(docInfo.revID);

// We'll need the full document if we're including doc bodies or listing conflicts:
CLEANUP(C4Document)* doc = NULL;
if (_includeDocs || (_allDocsMode >= kCBLShowConflicts && conflicted)) {
doc = c4enum_getDocument(_enum, &c4err);
if (!doc)
break;
if (!c4doc_loadRevisionBody(doc, &c4err))
break;
}

CBL_Revision* docRevision = nil;
if (_includeDocs) {
Expand All @@ -133,12 +143,11 @@ - (CBLQueryRow*) generateNextRow {
}

NSMutableArray<NSString*>* conflicts = nil;
if (_allDocsMode >= kCBLShowConflicts && (doc->flags & kConflicted)) {
conflicts = [NSMutableArray array];
if (_allDocsMode >= kCBLShowConflicts && conflicted) {
conflicts = [NSMutableArray new];
[conflicts addObject: revID.asString];
while (c4doc_selectNextLeafRevision(doc, false, false, NULL)) {
NSString* conflictID = slice2string(doc->selectedRev.revID);
[conflicts addObject: conflictID];
[conflicts addObject: slice2string(doc->selectedRev.revID)];
}
if (conflicts.count == 1)
conflicts = nil;
Expand All @@ -149,7 +158,7 @@ - (CBLQueryRow*) generateNextRow {
{@"_conflicts", conflicts}); // (not found in CouchDB)
LogVerbose(Query, @"AllDocs: Found row with key=\"%@\", value=%@", docID, value);
CBLQueryRow *row = [[CBLQueryRow alloc] initWithDocID: docID
sequence: doc->sequence
sequence: docInfo.sequence
key: docID
value: value
docRevision: docRevision];
Expand All @@ -176,7 +185,7 @@ - (CBLQueryRow*) generateNextRow {

- (BOOL) rowPassesFilter: (CBLQueryRow*)row {
//FIX: I'm not supposed to know the delegates' real classes...
[row moveToDatabase: _storage.delegate view: nil];
[row moveToDatabase: (CBLDatabase*)_storage.delegate view: nil];
if (!_filter(row))
return NO;
[row _clearDatabase];
Expand Down
72 changes: 46 additions & 26 deletions Source/CBL_ForestDBStorage.mm
View file
Expand Up @@ -504,11 +504,14 @@ - (CBL_RevisionList*) changesSinceSequence: (SequenceNumber)lastSequence
return nil;
}

BOOL withBody = (options->includeDocs || filter != nil);
BOOL revsWithBodies = (options->includeDocs || filter != nil);
BOOL loadC4Doc = (revsWithBodies || options->includeConflicts);
unsigned limit = options->limit;

C4EnumeratorOptions c4opts = kC4DefaultEnumeratorOptions;
c4opts.flags |= kC4IncludeDeleted;
if (!loadC4Doc)
c4opts.flags &= ~kC4IncludeBodies;
C4Error c4err = {};
CLEANUP(C4DocEnumerator)* e = c4db_enumerateChanges(_forest, lastSequence, &c4opts, &c4err);
if (!e) {
Expand All @@ -518,28 +521,39 @@ - (CBL_RevisionList*) changesSinceSequence: (SequenceNumber)lastSequence
CBL_RevisionList* changes = [[CBL_RevisionList alloc] init];
while (limit-- > 0 && c4enum_next(e, &c4err)) {
@autoreleasepool {
CLEANUP(C4Document) *doc = c4enum_getDocument(e, &c4err);
if (!doc)
break;
NSString* docID = slice2string(doc->docID);
do {
if (loadC4Doc) {
CLEANUP(C4Document) *doc = c4enum_getDocument(e, &c4err);
if (!doc)
break;
NSString* docID = slice2string(doc->docID);
do {
CBL_MutableRevision* rev;
rev = [CBLForestBridge revisionObjectFromForestDoc: doc
docID: docID
revID: nil
withBody: revsWithBodies
status: outStatus];
if (!rev)
return nil;
if (!filter || filter(rev)) {
if (!options->includeDocs)
rev.body = nil;
[changes addRev: rev];
}
} while (options->includeConflicts && c4doc_selectNextLeafRevision(doc, true,
revsWithBodies,
&c4err));
if (c4err.code)
break;
} else {
C4DocumentInfo docInfo;
c4enum_getDocumentInfo(e, &docInfo);
CBL_MutableRevision* rev;
rev = [CBLForestBridge revisionObjectFromForestDoc: doc
docID: docID
revID: nil
withBody: withBody
status: outStatus];
rev = [CBLForestBridge revisionObjectFromForestDocInfo: docInfo status: outStatus];
if (!rev)
return nil;
if (!filter || filter(rev)) {
if (!options->includeDocs)
rev.body = nil;
[changes addRev: rev];
}
} while (options->includeConflicts && c4doc_selectNextLeafRevision(doc, true, withBody,
&c4err));
if (c4err.code)
break;
[changes addRev: rev];
}
}
}
if (c4err.code) {
Expand Down Expand Up @@ -593,25 +607,31 @@ - (BOOL) findMissingRevisions: (CBL_RevisionList*)revs
#pragma mark - PURGING / COMPACTING:


- (NSSet*) findAllAttachmentKeys: (NSError**)outError {
NSMutableSet* keys = [NSMutableSet setWithCapacity: 1000];
- (NSSet<NSData*>*) findAllAttachmentKeys: (NSError**)outError {
NSMutableSet<NSData*>* keys = [NSMutableSet setWithCapacity: 1000];
C4EnumeratorOptions c4opts = kC4DefaultEnumeratorOptions;
c4opts.flags &= ~kC4IncludeBodies;
c4opts.flags |= kC4IncludeDeleted;
C4Error c4err;
CLEANUP(C4DocEnumerator)* e = c4db_enumerateAllDocs(_forest, kC4SliceNull, kC4SliceNull,
NULL, &c4err);
&c4opts, &c4err);
if (!e) {
err2OutNSError(c4err, outError);
return nil;
}

while (c4enum_next(e, &c4err)) {
C4DocumentInfo info;
c4enum_getDocumentInfo(e, &info);
C4DocumentFlags flags = info.flags;
if (!(flags & kHasAttachments) || ((flags & kDeleted) && !(flags & kConflicted)))
continue;

CLEANUP(C4Document)* doc = c4enum_getDocument(e, &c4err);
if (!doc) {
err2OutNSError(c4err, outError);
return nil;
}
C4DocumentFlags flags = doc->flags;
if (!(flags & kHasAttachments) || ((flags & kDeleted) && !(flags & kConflicted)))
continue;

// Since db is assumed to have just been compacted, we know that non-current revisions
// won't have any bodies. So only scan the current revs.
Expand Down
2 changes: 1 addition & 1 deletion vendor/CBForest
Submodule CBForest updated 21 files
+1 −1 C/c4Database.cc
+1 −1 C/c4DocEnumerator.cc
+5 −5 C/c4Document.cc
+1 −1 C/c4Document.h
+1 −1 C/c4Impl.hh
+5 −0 C/tests/c4DatabaseTest.cc
+1 −1 CBForest Tests/Database_Test.mm
+1 −1 CBForest Tests/GeoIndex_Test.mm
+1 −1 CBForest Tests/MapReduce_Test.mm
+6 −13 CBForest/Database.cc
+4 −15 CBForest/Database.hh
+5 −8 CBForest/DocEnumerator.cc
+3 −0 CBForest/DocEnumerator.hh
+7 −13 CBForest/Document.cc
+1 −1 CBForest/Document.hh
+4 −23 CBForest/KeyStore.cc
+0 −6 CBForest/VersionedDocument.cc
+0 −1 CBForest/VersionedDocument.hh
+2 −0 CBForest/slice.hh
+1 −1 Java/src/com/couchbase/cbforest/Constants.java
+1 −1 vendor/forestdb

0 comments on commit 94625e3

Please sign in to comment.