Replication encoding/encryption API now supports attachments.

During a pull, the _attachments objects for large attachments will have a temporary "file" property that points to the local file path where the attachment data can be found. The transformation block can read this file, but must not modify it.
couchbase · Mar 15, 2014 · 0ec5e5d · 0ec5e5d
1 parent 6d4394e
commit 0ec5e5d
Show file tree

Hide file tree

Showing 12 changed files with 218 additions and 43 deletions.
diff --git a/CouchbaseLite.xcodeproj/project.pbxproj b/CouchbaseLite.xcodeproj/project.pbxproj
@@ -176,6 +176,9 @@
 		275A29031649A11A00B0D8EE /* CouchbaseLitePrivate.h in Headers */ = {isa = PBXBuildFile; fileRef = 27DA434D15914F5700F9E7B5 /* CouchbaseLitePrivate.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		275A29041649A12B00B0D8EE /* CBLAttachment.h in Headers */ = {isa = PBXBuildFile; fileRef = 27E2E5A1159383E9005B9234 /* CBLAttachment.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		275A29051649A13900B0D8EE /* CBLReplication.h in Headers */ = {isa = PBXBuildFile; fileRef = 27E2E5CB159533A5005B9234 /* CBLReplication.h */; settings = {ATTRIBUTES = (Public, ); }; };
+		2762AB5818D36F1B00649D47 /* CBLReplication+Transformation.h in Headers */ = {isa = PBXBuildFile; fileRef = 2762AB5618D36F1B00649D47 /* CBLReplication+Transformation.h */; settings = {ATTRIBUTES = (Private, ); }; };
+		2762AB5918D36F1B00649D47 /* CBLReplication+Transformation.m in Sources */ = {isa = PBXBuildFile; fileRef = 2762AB5718D36F1B00649D47 /* CBLReplication+Transformation.m */; };
+		2762AB5A18D36F1B00649D47 /* CBLReplication+Transformation.m in Sources */ = {isa = PBXBuildFile; fileRef = 2762AB5718D36F1B00649D47 /* CBLReplication+Transformation.m */; };
 		2766EFF814DB7F9F009ECCA8 /* CBLMultipartWriter.h in Headers */ = {isa = PBXBuildFile; fileRef = 2766EFF614DB7F9F009ECCA8 /* CBLMultipartWriter.h */; };
 		2766EFF914DB7F9F009ECCA8 /* CBLMultipartWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = 2766EFF714DB7F9F009ECCA8 /* CBLMultipartWriter.m */; };
 		2766EFFA14DB7F9F009ECCA8 /* CBLMultipartWriter.m in Sources */ = {isa = PBXBuildFile; fileRef = 2766EFF714DB7F9F009ECCA8 /* CBLMultipartWriter.m */; };
@@ -349,6 +352,7 @@
 		27846FBD15D475DF0030122F /* MYStreamUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = 27846FB815D475DF0030122F /* MYStreamUtils.m */; settings = {COMPILER_FLAGS = "-fno-objc-arc"; }; };
 		27846FBE15D475DF0030122F /* MYStreamUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = 27846FB815D475DF0030122F /* MYStreamUtils.m */; settings = {COMPILER_FLAGS = "-fno-objc-arc"; }; };
 		27846FF115D5C8250030122F /* APITests.m in Sources */ = {isa = PBXBuildFile; fileRef = 27846FF015D5C8250030122F /* APITests.m */; };
+		2785B48D18D3F11900CBB41A /* CBLReplication+Transformation.h in Headers */ = {isa = PBXBuildFile; fileRef = 2762AB5618D36F1B00649D47 /* CBLReplication+Transformation.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		2785BE9D16BC87EA00A3E881 /* CBL_URLProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C706461487584300F0F099 /* CBL_URLProtocol.h */; };
 		2785BE9E16BC87ED00A3E881 /* CBL_Router.h in Headers */ = {isa = PBXBuildFile; fileRef = 27C706431486BE7100F0F099 /* CBL_Router.h */; };
 		278B0CA0152A8B1900577747 /* CBLCanonicalJSON.h in Headers */ = {isa = PBXBuildFile; fileRef = 278B0C9E152A8B1900577747 /* CBLCanonicalJSON.h */; };
@@ -982,6 +986,8 @@
 		27538F9E17F1D9D6004C3BFD /* PrivilegedInstall.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = PrivilegedInstall.m; sourceTree = "<group>"; };
 		27538FA117F1ECE8004C3BFD /* LoggingMode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LoggingMode.h; sourceTree = "<group>"; };
 		27538FA217F1ECE8004C3BFD /* LoggingMode.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = LoggingMode.c; sourceTree = "<group>"; };
+		2762AB5618D36F1B00649D47 /* CBLReplication+Transformation.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "CBLReplication+Transformation.h"; sourceTree = "<group>"; };
+		2762AB5718D36F1B00649D47 /* CBLReplication+Transformation.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "CBLReplication+Transformation.m"; sourceTree = "<group>"; };
 		2766EFF614DB7F9F009ECCA8 /* CBLMultipartWriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CBLMultipartWriter.h; sourceTree = "<group>"; };
 		2766EFF714DB7F9F009ECCA8 /* CBLMultipartWriter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CBLMultipartWriter.m; sourceTree = "<group>"; };
 		2766EFFB14DC7B37009ECCA8 /* CBLMultiStreamWriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CBLMultiStreamWriter.h; sourceTree = "<group>"; };
@@ -2195,6 +2201,8 @@
 				277EF39C17F4DEDD00F7B7F7 /* CBLQuery+Geo.m */,
 				27E2E5CB159533A5005B9234 /* CBLReplication.h */,
 				27E2E5CC159533A5005B9234 /* CBLReplication.m */,
+				2762AB5618D36F1B00649D47 /* CBLReplication+Transformation.h */,
+				2762AB5718D36F1B00649D47 /* CBLReplication+Transformation.m */,
 				27726AAA1889835F00AE6931 /* CBLJSON.h */,
 				27726AAB1889835F00AE6931 /* CBLJSON.m */,
 				27726AA81889835F00AE6931 /* CBLGeometry.h */,
@@ -2444,6 +2452,7 @@
 				270B3E211489390F00E0A926 /* CBL_URLProtocol.h in Headers */,
 				279EB2CE149140DE00E74185 /* CBLView+Internal.h in Headers */,
 				2753157614ACEFC90065964D /* DDRange.h in Headers */,
+				2762AB5818D36F1B00649D47 /* CBLReplication+Transformation.h in Headers */,
 				2753157A14ACEFC90065964D /* HTTPAuthenticationRequest.h in Headers */,
 				272401BE1860CBE00080E082 /* GCDAsyncSocket.h in Headers */,
 				2753157E14ACEFC90065964D /* HTTPConnection.h in Headers */,
@@ -2562,6 +2571,7 @@
 				27B0B7FD1492BDE800A817AD /* CBL_Body.h in Headers */,
 				27B0B7FE1492BDE800A817AD /* CBL_Revision.h in Headers */,
 				27F08C8B15A7A31B003C3E2B /* CBL_Attachment.h in Headers */,
+				2785B48D18D3F11900CBB41A /* CBLReplication+Transformation.h in Headers */,
 				27B0B7FF1492BDE800A817AD /* CBL_Server.h in Headers */,
 				27B0B8001492BDE800A817AD /* CBL_Router.h in Headers */,
 				27B0B8011492BDE800A817AD /* CBL_URLProtocol.h in Headers */,
@@ -3136,6 +3146,7 @@
 				27A82E3814A1145000C0B850 /* FMDatabaseAdditions.m in Sources */,
 				270B3E181489382F00E0A926 /* FMResultSet.m in Sources */,
 				27F128B7156AC8C5008465C2 /* OAPlaintextSignatureProvider.m in Sources */,
+				2762AB5918D36F1B00649D47 /* CBLReplication+Transformation.m in Sources */,
 				27F128B9156AC8F0008465C2 /* OAMutableURLRequest.m in Sources */,
 				270F5702156AD215000FEB8F /* OARequestParameter.m in Sources */,
 				27F128B5156AC8B7008465C2 /* OAToken.m in Sources */,
@@ -3301,6 +3312,7 @@
 				27F128B3156AC1CA008465C2 /* CBLOAuth1Authorizer.m in Sources */,
 				270F5707156AE0BF000FEB8F /* CBLAuthorizer.m in Sources */,
 				27ED8632157D0FC600712B33 /* CBLDocument.m in Sources */,
+				2762AB5A18D36F1B00649D47 /* CBLReplication+Transformation.m in Sources */,
 				27DA42FA158E66DD00F9E7B5 /* CBLRevision.m in Sources */,
 				27DA42FF158E7D3500F9E7B5 /* CBLDatabase.m in Sources */,
 				27DA430A158FA35600F9E7B5 /* CBLCache.m in Sources */,

diff --git a/Source/API/CBLReplication+Transformation.h b/Source/API/CBLReplication+Transformation.h
@@ -0,0 +1,27 @@
+//
+//  CBLReplication+Transformation.h
+//  CouchbaseLite
+//
+//  Created by Jens Alfke on 3/14/14.
+//
+//
+
+#import <CouchbaseLite/CouchbaseLite.h>
+
+
+/** A callback block for transforming revision bodies during replication.
+    See CBLReplication.propertiesTransformationBlock's documentation for details. */
+typedef NSDictionary *(^CBLPropertiesTransformationBlock)(NSDictionary* doc);
+
+
+@interface CBLReplication (Transformation)
+
+/** Optional callback for transforming document bodies during replication; can be used to encrypt documents stored on the remote server, for example.
+    In a push replication, the block is called with document properties from the local database, and the transformed properties are what will be uploaded to the server.
+    In a pull replication, the block is called with document properties downloaded from the server, and the transformed properties are what will be stored in the local database.
+    The block takes an NSDictionary containing the document's properties (including the "_id" and "_rev" metadata), and returns a dictionary of transformed properties. It may return the input dictionary if it has no changes to make.
+    The transformation MUST preserve the values of any keys whose names begin with an underscore ("_")!
+    The block will be called on the background replicator thread, NOT on the CBLReplication's thread, so it shouldn't directly access any Couchbase Lite objects. */
+@property (strong) CBLPropertiesTransformationBlock propertiesTransformationBlock;
+
+@end
diff --git a/Source/API/CBLReplication+Transformation.m b/Source/API/CBLReplication+Transformation.m
@@ -0,0 +1,23 @@
+//
+//  CBLReplication+Transformation.m
+//  CouchbaseLite
+//
+//  Created by Jens Alfke on 3/14/14.
+//
+//
+
+#import "CBLReplication+Transformation.h"
+#import "CouchbaseLitePrivate.h"
+
+
+@implementation CBLReplication (Transformation)
+
+- (CBLPropertiesTransformationBlock) propertiesTransformationBlock {
+    return _propertiesTransformationBlock;
+}
+
+- (void) setPropertiesTransformationBlock:(CBLPropertiesTransformationBlock)block {
+    _propertiesTransformationBlock = block;
+}
+
+@end
diff --git a/Source/API/CBLReplication.h b/Source/API/CBLReplication.h
@@ -19,11 +19,6 @@ typedef enum {
 } CBLReplicationStatus;
 
 
-/** A callback block for transforming revision bodies during replication.
-    See CBLReplication.propertiesTransformationBlock's documentation for details. */
-typedef NSDictionary *(^CBLPropertiesTransformationBlock)(NSDictionary *);
-
-
 /** A 'push' or 'pull' replication between a local and a remote database.
     Replications can be one-shot or continuous. */
 @interface CBLReplication : NSObject

diff --git a/Source/API/CBLReplication.m b/Source/API/CBLReplication.m
@@ -58,7 +58,6 @@ @implementation CBLReplication
 @synthesize headers=_headers, OAuth=_OAuth, facebookEmailAddress=_facebookEmailAddress;
 @synthesize personaEmailAddress=_personaEmailAddress, customProperties=_customProperties;
 @synthesize running = _running, completedChangesCount=_completedChangesCount, changesCount=_changesCount, lastError=_lastError, status=_status;
-@synthesize propertiesTransformationBlock=_propertiesTransformationBlock;
 
 - (instancetype) initWithDatabase: (CBLDatabase*)database
                            remote: (NSURL*)remote

diff --git a/Source/API/CouchbaseLitePrivate.h b/Source/API/CouchbaseLitePrivate.h
@@ -9,6 +9,7 @@
 #import "CouchbaseLite.h"
 #import "CBLCache.h"
 #import "CBLDatabase.h"
+#import "CBLReplication+Transformation.h"
 #import "CBL_Revision.h"
 #import "CBLGeometry.h"
 @class CBLDatabaseChange, CBL_Revision, CBLManager, CBL_Server;
@@ -131,16 +132,11 @@
 
 
 @interface CBLReplication ()
+{
+    CBLPropertiesTransformationBlock _propertiesTransformationBlock;
+}
 - (instancetype) initWithDatabase: (CBLDatabase*)database
                            remote: (NSURL*)remote
                              pull: (BOOL)pull                       __attribute__((nonnull));
 @property (nonatomic, readonly) NSDictionary* properties;
-
-/** Optional callback for transforming document bodies during replication; can be used to encrypt documents stored on the remote server, for example.
-    In a push replication, the block is called with document properties from the local database, and the transformed properties are what will be uploaded to the server.
-    In a pull replication, the block is called with document properties downloaded from the server, and the transformed properties are what will be stored in the local database.
-    The block takes an NSDictionary containing the document's properties (including the "_id" and "_rev" metadata), and returns a dictionary of transformed properties. It may return the input dictionary if it has no changes to make.
-    The transformation MUST preserve the values of any keys whose names begin with an underscore ("_")!
-    The block will be called on the background replicator thread, NOT on the CBLReplication's thread, so it shouldn't directly access any Couchbase Lite objects. */
-@property (strong) CBLPropertiesTransformationBlock propertiesTransformationBlock;
 @end
diff --git a/Source/API/ReplicationAPITests.m b/Source/API/ReplicationAPITests.m
@@ -11,6 +11,7 @@
 #import "CouchbaseLitePrivate.h"
 #import "CBLInternal.h"
 #import "Test.h"
+#import <CommonCrypto/CommonCryptor.h>
 
 
 #if DEBUG
@@ -19,6 +20,8 @@
 // This db will get deleted and overwritten during every test.
 #define kPushThenPullDBName @"cbl_replicator_pushpull"
 #define kNDocuments 1000
+// This one too.
+#define kEncodedDBName @"cbl_replicator_encoding"
 
 
 @interface ReplicationObserverHelper : NSObject
@@ -262,9 +265,67 @@ static void runReplication(CBLReplication* repl, unsigned expectedChangesCount)
 }
 
 
+static UInt8 sEncryptionKey[kCCKeySizeAES256];
+static UInt8 sEncryptionIV[kCCBlockSizeAES128];
+
+
+// Tests the CBLReplication.propertiesTransformationBlock API, by encrypting the document's
+// "secret" property with AES-256 as it's pushed to the server. The encrypted data is stored in
+// an attachment named "(encrypted)".
+TestCase(ReplicationWithEncoding) {
+    RequireTestCase(RunPushReplication);
+    NSURL* remoteDbURL = RemoteTestDBURL(kEncodedDBName);
+    if (!remoteDbURL) {
+        Warn(@"Skipping test ReplicationWithEncoding (no remote test DB URL)");
+        return;
+    }
+    DeleteRemoteDB(remoteDbURL);
+
+    Log(@"Creating document...");
+    CBLDatabase* db = createEmptyManagerAndDb();
+    CBLDocument* doc = db[@"seekrit"];
+    [doc putProperties: @{@"secret": @"Attack at dawn"} error: NULL];
+
+    Log(@"Pushing...");
+    CBLReplication* repl = [db createPushReplication: remoteDbURL];
+    repl.createTarget = YES;
+
+    SecRandomCopyBytes(kSecRandomDefault, sizeof(sEncryptionKey), sEncryptionKey);
+    SecRandomCopyBytes(kSecRandomDefault, sizeof(sEncryptionIV), sEncryptionIV);
+
+    repl.propertiesTransformationBlock = ^NSDictionary*(NSDictionary* props) {
+        NSData* cleartext = [props[@"secret"] dataUsingEncoding: NSUTF8StringEncoding];
+        Assert(cleartext);
+        NSMutableData* ciphertext = [NSMutableData dataWithLength: cleartext.length + 128];
+        size_t encryptedLength;
+        CCCryptorStatus status = CCCrypt(kCCEncrypt, kCCAlgorithmAES, kCCOptionPKCS7Padding,
+                                         sEncryptionKey, sizeof(sEncryptionKey), sEncryptionIV,
+                                         cleartext.bytes, cleartext.length,
+                                         ciphertext.mutableBytes, ciphertext.length, &encryptedLength);
+        AssertEq(status, kCCSuccess);
+        Assert(encryptedLength > 0);
+        ciphertext.length = encryptedLength;
+        Log(@"Ciphertext = %@", ciphertext);
+
+        NSMutableDictionary* nuProps = [props mutableCopy];
+        [nuProps removeObjectForKey: @"secret"];
+        nuProps[@"_attachments"] = @{@"(encrypted)": @{@"data":[ciphertext base64Encoding]}};
+        Log(@"Encoded document = %@", nuProps);
+        return nuProps;
+    };
+
+    [repl start];
+    runReplication(repl, 1);
+    AssertNil(repl.lastError);
+    [db.manager close];
+}
+
+
+// Tests the CBLReplication.propertiesTransformationBlock API, by decrypting the encrypted
+// documents produced by ReplicationWithEncoding.
 TestCase(ReplicationWithDecoding) {
-    RequireTestCase(RunPullReplication);
-    NSURL* remoteDbURL = RemoteTestDBURL(kPushThenPullDBName);
+    RequireTestCase(ReplicationWithEncoding);
+    NSURL* remoteDbURL = RemoteTestDBURL(kEncodedDBName);
     if (!remoteDbURL) {
         Warn(@"Skipping test ReplicationWithDecoding (no remote test DB URL)");
         return;
@@ -276,23 +337,47 @@ static void runReplication(CBLReplication* repl, unsigned expectedChangesCount)
     repl.propertiesTransformationBlock = ^NSDictionary*(NSDictionary* props) {
         Assert(props.cbl_id);
         Assert(props.cbl_rev);
-        NSInteger index = [props[@"index"] integerValue];
-        if (index % 2)
+        NSDictionary* encrypted = [props[@"_attachments"] objectForKey: @"(encrypted)"];
+        if (!encrypted)
             return props;
+
+        NSData* ciphertext;
+        NSString* ciphertextStr = $castIf(NSString, encrypted[@"data"]);
+        if (ciphertextStr) {
+            // Attachment was inline:
+            ciphertext = [[NSData alloc] initWithBase64EncodedString: ciphertextStr options: 0];
+        } else {
+            // The replicator is kind enough to add a temporary "file" property that points to
+            // the downloaded attachment:
+            NSString* filePath = $castIf(NSString, encrypted[@"file"]);
+            Assert(filePath);
+            ciphertext = [NSData dataWithContentsOfFile: filePath];
+        }
+        Assert(ciphertext);
+        NSMutableData* cleartext = [NSMutableData dataWithLength: ciphertext.length];
+
+        size_t decryptedLength;
+        CCCryptorStatus status = CCCrypt(kCCDecrypt, kCCAlgorithmAES, kCCOptionPKCS7Padding,
+                                         sEncryptionKey, sizeof(sEncryptionKey), sEncryptionIV,
+                                         ciphertext.bytes, ciphertext.length,
+                                         cleartext.mutableBytes, cleartext.length, &decryptedLength);
+        AssertEq(status, kCCSuccess);
+        Assert(decryptedLength > 0);
+        cleartext.length = decryptedLength;
+        Log(@"Cleartext = %@", cleartext);
+        NSString* cleartextStr = [[NSString alloc] initWithData: cleartext encoding: NSUTF8StringEncoding];
+
         NSMutableDictionary* nuProps = [props mutableCopy];
-        nuProps[@"index"] = @(-index);
+        nuProps[@"secret"] = cleartextStr;
         return nuProps;
     };
-    runReplication(repl, kNDocuments);
+    runReplication(repl, 1);
     AssertNil(repl.lastError);
 
-    Log(@"Verifying documents...");
-    for (int i = 1; i <= kNDocuments; i++) {
-        CBLDocument* doc = db[ $sprintf(@"doc-%d", i) ];
-        int expectedIndex = (i%2) ? i : -i;
-        AssertEqual(doc[@"index"], @(expectedIndex));
-        AssertEqual(doc[@"bar"], $false);
-    }
+    // Finally, verify the decryption:
+    CBLDocument* doc = db[@"seekrit"];
+    NSString* plans = doc[@"secret"];
+    AssertEqual(plans, @"Attack at dawn");
     [db.manager close];
 }
 

diff --git a/Source/CBLDatabase+Attachments.m b/Source/CBLDatabase+Attachments.m
@@ -363,10 +363,26 @@ - (NSDictionary*) getAttachmentDictForSequence: (SequenceNumber)sequence
 
 - (NSURL*) fileForAttachmentDict: (NSDictionary*)attachmentDict
 {
-    CBLBlobKey key;
-    if (!digestToBlobKey(attachmentDict[@"digest"], &key))
+    NSString* digest = $castIf(NSString, attachmentDict[@"digest"]);
+    if (!digest)
         return nil;
-    return [NSURL fileURLWithPath: [_attachments pathForKey: key]];
+    NSString* path = nil;
+    id pending = _pendingAttachmentsByDigest[digest];
+    if (pending) {
+        if ([pending isKindOfClass: [CBL_BlobStoreWriter class]]) {
+            path = [pending filePath];
+        } else {
+            CBLBlobKey key = *(CBLBlobKey*)[pending bytes];
+            path = [_attachments pathForKey: key];
+        }
+    } else {
+        // If it's an installed attachment, ask the blob-store for it:
+        CBLBlobKey key;
+        if (digestToBlobKey(digest, &key))
+            path = [_attachments pathForKey: key];
+    }
+
+    return path ? [NSURL fileURLWithPath: path] : nil;
 }
 
 

diff --git a/Source/CBL_BlobStore.h b/Source/CBL_BlobStore.h
@@ -101,4 +101,8 @@ typedef struct {
 @property (readonly) NSString* MD5DigestString;
 @property (readonly) NSString* SHA1DigestString;
 
+/** The location of the temporary file containing the attachment contents.
+    Will be nil after -install is called. */
+@property (readonly) NSString* filePath;
+
 @end
diff --git a/Source/CBL_BlobStore.m b/Source/CBL_BlobStore.m
@@ -238,7 +238,7 @@ - (NSString*) tempDir {
 
 @implementation CBL_BlobStoreWriter
 
-@synthesize length=_length, blobKey=_blobKey;
+@synthesize length=_length, blobKey=_blobKey, filePath=_tempPath;
 
 - (instancetype) initWithStore: (CBL_BlobStore*)store {
     self = [super init];