CBL-957: multiple replicators/listeners test (#2658)

* when replicating from multiple replicators to same database.
* two listener on same database
* multiple replicators and listener test are failing intermittently: so disabled and linked the issue: CBL-1033

Objective-C/Tests/URLEndpointListenerTest.m

@@ -114,6 +114,93 @@ - (Listener*) listen: (Config*)config errorCode: (NSInteger)code errorDomain: (n
     return _listener;
 }
 
+// Two replicators, replicates docs to the listener; validates connection status
+- (void) validateMultipleReplicationsTo: (Listener*)listener replType: (CBLReplicatorType)type {
+    XCTestExpectation* exp1 = [self expectationWithDescription: @"replicator#1 stopped"];
+    XCTestExpectation* exp2 = [self expectationWithDescription: @"replicator#2 stopped"];
+
+    NSUInteger existingDocsInListener = listener.config.database.count;
+
+    // open DBs
+    NSError* err = nil;
+    Assert([self deleteDBNamed: @"db1" error: &err], @"Failed to delete db1 %@", err);
+    Assert([self deleteDBNamed: @"db2" error: &err], @"Failed to delete db2 %@", err);
+    CBLDatabase* db1 = [self openDBNamed: @"db1" error: &err];
+    AssertNil(err);
+    CBLDatabase* db2 = [self openDBNamed: @"db2" error: &err];
+    AssertNil(err);
+
+    // For keeping the replication long enough to validate connection status, we will use blob
+    NSData* content = [@"i am a blob" dataUsingEncoding: NSUTF8StringEncoding];
+
+    // DB#1
+    CBLBlob* blob1 = [[CBLBlob alloc] initWithContentType: @"text/plain" data: content];
+    CBLMutableDocument* doc1 = [self createDocument: @"doc-1"];
+    [doc1 setValue: blob1 forKey: @"blob"];
+    Assert([db1 saveDocument: doc1 error: &err], @"Fail to save db1 %@", err);
+
+    // DB#2
+    CBLBlob* blob2 = [[CBLBlob alloc] initWithContentType: @"text/plain" data: content];
+    CBLMutableDocument* doc2 = [self createDocument: @"doc-2"];
+    [doc2 setValue: blob2 forKey: @"blob"];
+    Assert([db2 saveDocument: doc2 error: &err], @"Fail to save db2 %@", err);
+
+    // replicators
+    CBLReplicatorConfiguration *config1, *config2;
+    config1 = [[CBLReplicatorConfiguration alloc] initWithDatabase: db1 target: listener.localEndpoint];
+    config2 = [[CBLReplicatorConfiguration alloc] initWithDatabase: db2 target: listener.localEndpoint];
+    config1.replicatorType = type;
+    config2.replicatorType = type;
+    config1.pinnedServerCertificate = (__bridge SecCertificateRef) listener.config.tlsIdentity.certs[0];
+    config2.pinnedServerCertificate = (__bridge SecCertificateRef) listener.config.tlsIdentity.certs[0];
+    CBLReplicator* repl1 = [[CBLReplicator alloc] initWithConfig: config1];
+    CBLReplicator* repl2 = [[CBLReplicator alloc] initWithConfig: config2];
+
+    // get listener status
+    __block Listener* weakListener = _listener;
+    __block uint64_t maxConnectionCount = 0, maxActiveCount = 0;
+    id changeListener = ^(CBLReplicatorChange * change) {
+        Listener* strongListener = weakListener;
+        if (change.status.activity == kCBLReplicatorBusy) {
+            maxConnectionCount = MAX(strongListener.status.connectionCount, maxConnectionCount);
+            maxActiveCount = MAX(strongListener.status.activeConnectionCount, maxActiveCount);
+        } else if (change.status.activity == kCBLReplicatorStopped) {
+            if (change.replicator == repl1)
+                [exp1 fulfill];
+            else
+                [exp2 fulfill];
+        }
+    };
+    id token1 = [repl1 addChangeListener: changeListener];
+    id token2 = [repl2 addChangeListener: changeListener];
+
+    // start & wait for replication
+    [repl1 start];
+    [repl2 start];
+    [self waitForExpectations: @[exp1, exp2] timeout: timeout];
+
+    // check both replicators access listener at same time
+    AssertEqual(maxConnectionCount, 2u);
+    AssertEqual(maxActiveCount, 2u);
+
+    // all data are transferred to/from
+    if (type < kCBLReplicatorTypePull)
+        AssertEqual(listener.config.database.count, existingDocsInListener + 2u);
+
+    AssertEqual(db1.count, existingDocsInListener + 1/* db2 doc*/);
+    AssertEqual(db2.count, existingDocsInListener + 1/* db1 doc*/);
+
+    // cleanup
+    [repl1 removeChangeListenerWithToken: token1];
+    [repl2 removeChangeListenerWithToken: token2];
+    repl1 = nil;
+    repl2 = nil;
+    Assert([db1 close: &err], @"Failed to close db1 %@", err);
+    Assert([db2 close: &err], @"Failed to close db2 %@", err);
+    db1 = nil;
+    db2 = nil;
+}
+
 - (void) tearDown {
     if (_listener) {
         [_listener stop];
@@ -499,4 +586,78 @@ - (void) testUnavailableNetworkInterface {
     [_listener stop];
 }
 
+- (void) testMultipleListenersOnSameDatabase {
+    if (!self.keyChainAccessAllowed) return;
+
+    Config* config = [[Config alloc] initWithDatabase: self.otherDB];
+    Listener* listener1 = [[Listener alloc] initWithConfig: config];
+    Listener* listener2 = [[Listener alloc] initWithConfig: config];
+
+    NSError* err = nil;
+    Assert([listener1 startWithError: &err]);
+    AssertNil(err);
+    Assert([listener2 startWithError: &err]);
+    AssertNil(err);
+
+    // replicate doc
+    [self generateDocumentWithID: @"doc-1"];
+    [self runWithTarget: listener1.localEndpoint
+                   type: kCBLReplicatorTypePushAndPull
+             continuous: NO
+          authenticator: nil
+             serverCert: (__bridge SecCertificateRef) listener1.config.tlsIdentity.certs[0]
+              errorCode: 0
+            errorDomain: nil];
+
+    [listener1 stop];
+    [listener2 stop];
+    [self ignoreException:^{
+        NSError* error = nil;
+        Assert([listener1.config.tlsIdentity deleteFromKeyChainWithError: &error],
+               @"Couldn't delete identity1: %@", error);
+        Assert([listener2.config.tlsIdentity deleteFromKeyChainWithError: &error],
+               @"Couldn't delete identity2: %@", error);
+    }];
+
+    AssertEqual(self.otherDB.count, 1);
+}
+
+// TODO: https://issues.couchbase.com/browse/CBL-1033
+- (void) _testMultipleReplicatorsToListener {
+    if (!self.keyChainAccessAllowed) return;
+
+    [self listen]; // writable listener
+
+    // save a doc on listenerDB
+    NSError* err = nil;
+    CBLMutableDocument* doc = [self createDocument: @"doc"];
+    [doc setValue: @"Tiger" forKey: @"species"];
+    Assert([self.otherDB saveDocument: doc error: &err], @"Failed to save listener DB %@", err);
+
+    [self validateMultipleReplicationsTo: _listener replType: kCBLReplicatorTypePushAndPull];
+
+    // cleanup
+    [_listener stop];
+}
+
+// TODO: https://issues.couchbase.com/browse/CBL-1033
+- (void) _testMultipleReplicatorsOnReadOnlyListener {
+    if (!self.keyChainAccessAllowed) return;
+
+    Config* config = [[Config alloc] initWithDatabase: self.otherDB];
+    config.readOnly = YES;
+    [self listen: config];
+
+    // save a doc on listenerDB
+    NSError* err = nil;
+    CBLMutableDocument* doc = [self createDocument: @"doc"];
+    [doc setValue: @"Tiger" forKey: @"species"];
+    Assert([self.otherDB saveDocument: doc error: &err], @"Failed to save listener DB %@", err);
+
+    [self validateMultipleReplicationsTo: _listener replType: kCBLReplicatorTypePull];
+
+    // cleanup
+    [_listener stop];
+}
+
 @end