Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 315 lines (254 sloc) 11.819 kB
56f4b33 @snej ToyPuller!
snej authored
1 //
9ae9ecf @snej The Great Renaming
snej authored
2 // TDPuller.m
3 // TouchDB
56f4b33 @snej ToyPuller!
snej authored
4 //
5 // Created by Jens Alfke on 12/2/11.
6 // Copyright (c) 2011 Couchbase, Inc. All rights reserved.
7 //
b976483 @snej Cleanup
snej authored
8 // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
9 // except in compliance with the License. You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software distributed under the
12 // License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13 // either express or implied. See the License for the specific language governing permissions
14 // and limitations under the License.
56f4b33 @snej ToyPuller!
snej authored
15
9ae9ecf @snej The Great Renaming
snej authored
16 #import "TDPuller.h"
0b50cda @snej Header cleanup
snej authored
17 #import "TDDatabase+Insertion.h"
18 #import "TDDatabase+Replication.h"
9ae9ecf @snej The Great Renaming
snej authored
19 #import "TDRevision.h"
42705e2 @snej Removing some external dependencies
snej authored
20 #import "TDChangeTracker.h"
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
21 #import "TDBatcher.h"
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
22 #import "TDInternal.h"
79f0fd5 @snej Added .error property to TDReplicator and TDChangeTracker.
snej authored
23 #import "TDMisc.h"
9abe203 @snej Better exception reporting in TDPuller.
snej authored
24 #import "ExceptionUtils.h"
56f4b33 @snej ToyPuller!
snej authored
25
26
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
27 // Maximum number of revisions to fetch simultaneously
28 #define kMaxOpenHTTPConnections 8
29
56f4b33 @snej ToyPuller!
snej authored
30
9ae9ecf @snej The Great Renaming
snej authored
31 @interface TDPuller () <TDChangeTrackerClient>
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
32 - (void) pullRemoteRevisions;
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
33 - (void) pullRemoteRevision: (TDRevision*)rev;
34 - (void) insertRevisions: (NSArray*)revs;
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
35 - (NSArray*) knownCurrentRevIDsOf: (TDRevision*)rev;
56f4b33 @snej ToyPuller!
snej authored
36 @end
37
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
38 static NSString* joinQuotedEscaped(NSArray* strings);
39
56f4b33 @snej ToyPuller!
snej authored
40
9ae9ecf @snej The Great Renaming
snej authored
41 @implementation TDPuller
56f4b33 @snej ToyPuller!
snej authored
42
43
21f6cf7 @chriskau Added support for filtered replication (push)
chriskau authored
44 @synthesize filterName=_filterName,filterParameters=_filterParameters;
e6b78eb @snej Add some support for filters
snej authored
45
46
56f4b33 @snej ToyPuller!
snej authored
47 - (void)dealloc {
48 [_changeTracker stop];
49 [_changeTracker release];
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
50 [_revsToPull release];
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
51 [_revsToInsert release];
e6b78eb @snej Add some support for filters
snej authored
52 [_filterName release];
21f6cf7 @chriskau Added support for filtered replication (push)
chriskau authored
53 [_filterParameters release];
56f4b33 @snej ToyPuller!
snej authored
54 [super dealloc];
55 }
56
57
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
58 - (void) beginReplicating {
56f4b33 @snej ToyPuller!
snej authored
59 Assert(!_changeTracker);
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
60 if (!_revsToInsert) {
61 _revsToInsert = [[TDBatcher alloc] initWithCapacity: 100 delay: 0.25
62 processor: ^(NSArray *revs) {
63 [self insertRevisions: revs];
64 }];
65 }
66
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
67 _nextFakeSequence = _maxInsertedFakeSequence = 0;
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
68 LogTo(SyncVerbose, @"%@ starting ChangeTracker with since=%@", self, _lastSequence);
42705e2 @snej Removing some external dependencies
snej authored
69 _changeTracker = [[TDChangeTracker alloc]
14d7b5e @snej ToyPusher!
snej authored
70 initWithDatabaseURL: _remote
71 mode: (_continuous ? kLongPoll :kOneShot)
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
72 lastSequence: _lastSequence
14d7b5e @snej ToyPusher!
snej authored
73 client: self];
e6b78eb @snej Add some support for filters
snej authored
74 _changeTracker.filterName = _filterName;
21f6cf7 @chriskau Added support for filtered replication (push)
chriskau authored
75 _changeTracker.filterParameters = _filterParameters;
56f4b33 @snej ToyPuller!
snej authored
76 [_changeTracker start];
7f5c5da @snej Replicator progress notification
snej authored
77 [self asyncTaskStarted];
56f4b33 @snej ToyPuller!
snej authored
78 }
79
80
81 - (void) stop {
7633394 @snej Clean up views/replicators when a TDDatabase is closed
snej authored
82 if (!_running)
83 return;
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
84 _changeTracker.client = nil; // stop it from calling my -changeTrackerStopped
56f4b33 @snej ToyPuller!
snej authored
85 [_changeTracker stop];
86 [_changeTracker release];
70e865f @snej Added revision-tree tests and a simple puller test.
snej authored
87 _changeTracker = nil;
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
88 [_revsToPull release];
89 _revsToPull = nil;
14d7b5e @snej ToyPusher!
snej authored
90 [super stop];
7f5c5da @snej Replicator progress notification
snej authored
91
92 if (_asyncTaskCount == 0)
93 [self stopped];
56f4b33 @snej ToyPuller!
snej authored
94 }
95
96
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
97 // Got a _changes feed entry from the TDChangeTracker.
56f4b33 @snej ToyPuller!
snej authored
98 - (void) changeTrackerReceivedChange: (NSDictionary*)change {
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
99 NSString* lastSequence = [[change objectForKey: @"seq"] description];
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
100 NSString* docID = [change objectForKey: @"id"];
101 if (!docID)
102 return;
03f9a66 @snej Fixed nasty bug causing TDPusher to push revs with no _id
snej authored
103 if (![TDDatabase isValidDocumentID: docID]) {
104 Warn(@"%@: Received invalid doc ID from _changes: %@", self, change);
105 return;
106 }
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
107 BOOL deleted = [[change objectForKey: @"deleted"] isEqual: (id)kCFBooleanTrue];
7f5c5da @snej Replicator progress notification
snej authored
108 NSArray* changes = $castIf(NSArray, [change objectForKey: @"changes"]);
109 for (NSDictionary* changeDict in changes) {
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
110 @autoreleasepool {
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
111 // Push each revision info to the inbox
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
112 NSString* revID = $castIf(NSString, [changeDict objectForKey: @"rev"]);
113 if (!revID)
114 continue;
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
115 TDPulledRevision* rev = [[TDPulledRevision alloc] initWithDocID: docID revID: revID
116 deleted: deleted];
117 rev.remoteSequenceID = lastSequence;
118 rev.sequence = ++_nextFakeSequence;
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
119 [self addToInbox: rev];
120 [rev release];
121 }
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
122 }
7f5c5da @snej Replicator progress notification
snej authored
123 self.changesTotal += changes.count;
56f4b33 @snej ToyPuller!
snej authored
124 }
125
126
42705e2 @snej Removing some external dependencies
snej authored
127 - (void) changeTrackerStopped:(TDChangeTracker *)tracker {
70e865f @snej Added revision-tree tests and a simple puller test.
snej authored
128 LogTo(Sync, @"%@: ChangeTracker stopped", self);
79f0fd5 @snej Added .error property to TDReplicator and TDChangeTracker.
snej authored
129
130 if (!_error && tracker.error)
131 self.error = tracker.error;
132
70e865f @snej Added revision-tree tests and a simple puller test.
snej authored
133 [_changeTracker release];
134 _changeTracker = nil;
135
7f5c5da @snej Replicator progress notification
snej authored
136 [_batcher flush];
137 [self asyncTasksFinished: 1];
70e865f @snej Added revision-tree tests and a simple puller test.
snej authored
138 }
139
140
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
141 // Process a bunch of remote revisions from the _changes feed at once
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
142 - (void) processInbox: (TDRevisionList*)inbox {
99bbf5a @snej DB now stores revision history tree.
snej authored
143 // Ask the local database which of the revs are not known to it:
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
144 LogTo(SyncVerbose, @"%@: Looking up %@", self, inbox);
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
145 NSString* lastInboxSequence = [inbox.allRevisions.lastObject remoteSequenceID];
7f5c5da @snej Replicator progress notification
snej authored
146 NSUInteger total = _changesTotal - inbox.count;
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
147 if (![_db findMissingRevisions: inbox]) {
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
148 Warn(@"%@ failed to look up local revs", self);
7f5c5da @snej Replicator progress notification
snej authored
149 inbox = nil;
56f4b33 @snej ToyPuller!
snej authored
150 }
7f5c5da @snej Replicator progress notification
snej authored
151 if (_changesTotal != total + inbox.count)
152 self.changesTotal = total + inbox.count;
153
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
154 if (inbox.count == 0) {
155 // Nothing to do. Just bump the lastSequence.
156 LogTo(SyncVerbose, @"%@ no new remote revisions to fetch", self);
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
157 self.lastSequence = lastInboxSequence;
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
158 return;
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
159 }
160
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
161 LogTo(Sync, @"%@ fetching %u remote revisions...", self, inbox.count);
3674722 @snej Added "One-Shot Sync" menu command to Mac demo
snej authored
162 LogTo(SyncVerbose, @"%@ fetching remote revisions %@", self, inbox.allRevisions);
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
163
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
164 // Dump the revs into the queue of revs to pull from the remote db:
165 if (!_revsToPull)
166 _revsToPull = [[NSMutableArray alloc] initWithCapacity: 100];
167 [_revsToPull addObjectsFromArray: inbox.allRevisions];
168
169 [self pullRemoteRevisions];
170 }
171
172
173 // Start up some HTTP GETs, within our limit on the maximum simultaneous number
174 - (void) pullRemoteRevisions {
175 while (_httpConnectionCount < kMaxOpenHTTPConnections && _revsToPull.count > 0) {
176 [self pullRemoteRevision: [_revsToPull objectAtIndex: 0]];
177 [_revsToPull removeObjectAtIndex: 0];
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
178 }
179 }
180
181
182 // Fetches the contents of a revision from the remote db, including its parent revision ID.
183 // The contents are stored into rev.properties.
184 - (void) pullRemoteRevision: (TDRevision*)rev
185 {
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
186 [self asyncTaskStarted];
187 ++_httpConnectionCount;
44d2903 @snej Miscellaneous fixes
snej authored
188
189 // Construct a query. We want the revision history, and the bodies of attachments that have
190 // been added since the latest revisions we have locally.
191 // See: http://wiki.apache.org/couchdb/HTTP_Document_API#Getting_Attachments_With_a_Document
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
192 NSString* path = $sprintf(@"/%@?rev=%@&revs=true&attachments=true",
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
193 TDEscapeURLParam(rev.docID), TDEscapeURLParam(rev.revID));
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
194 NSArray* knownRevs = [self knownCurrentRevIDsOf: rev];
195 if (knownRevs.count > 0)
196 path = [path stringByAppendingFormat: @"&atts_since=%@", joinQuotedEscaped(knownRevs)];
197
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
198 [self sendAsyncRequest: @"GET" path: path body: nil
199 onCompletion: ^(NSDictionary *properties, NSError *error) {
81b3f5a @snej TDPusher now sends attachments, non-optimally
snej authored
200 // OK, now we've got the response revision:
7f5c5da @snej Replicator progress notification
snej authored
201 if (properties) {
4c86648 @snej Partial compatibility with CouchDB 'all_docs.js' test suite
snej authored
202 NSArray* history = [TDDatabase parseCouchDBRevisionHistory: properties];
03f9a66 @snej Fixed nasty bug causing TDPusher to push revs with no _id
snej authored
203 if (history) {
204 rev.properties = properties;
205 // Add to batcher ... eventually it will be fed to -insertRevisions:.
206 [_revsToInsert queueObject: $array(rev, history)];
207 [self asyncTaskStarted];
208 } else {
47e70e1 @snej Replicator now stores state in _local doc in remote db
snej authored
209 Warn(@"%@: Missing revision history in response from %@", path, self);
03f9a66 @snej Fixed nasty bug causing TDPusher to push revs with no _id
snej authored
210 self.changesProcessed++;
211 }
7f5c5da @snej Replicator progress notification
snej authored
212 } else {
79f0fd5 @snej Added .error property to TDReplicator and TDChangeTracker.
snej authored
213 if (error)
214 self.error = error;
7f5c5da @snej Replicator progress notification
snej authored
215 self.changesProcessed++;
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
216 }
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
217
218 // Note that we've finished this task; then start another one if there
219 // are still revisions waiting to be pulled:
7f5c5da @snej Replicator progress notification
snej authored
220 [self asyncTasksFinished: 1];
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
221 --_httpConnectionCount;
222 [self pullRemoteRevisions];
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
223 }
224 ];
225 }
226
227
228 // This will be called when _revsToInsert fills up:
229 - (void) insertRevisions:(NSArray *)revs {
230 LogTo(Sync, @"%@ inserting %u revisions...", self, revs.count);
3674722 @snej Added "One-Shot Sync" menu command to Mac demo
snej authored
231 LogTo(SyncVerbose, @"%@ inserting %@", self, revs);
328678e @snej TDPuller now uses async GETs from the remote db.
snej authored
232
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
233 /* Updating self.lastSequence is tricky. It needs to be the received sequence ID of the revision for which we've successfully received and inserted (or rejected) it and all previous received revisions. That way, next time we can start tracking remote changes from that sequence ID and know we haven't missed anything. */
234 /* FIX: The current code below doesn't quite achieve that: it tracks the latest sequence ID we've successfully processed, but doesn't handle failures correctly across multiple calls to -insertRevisions. I think correct behavior will require keeping an NSMutableIndexSet to track the fake-sequences of all processed revisions; then we can find the first missing index in that set and not advance lastSequence past the revision with that fake-sequence. */
235
08ff087 @snej Oops, broke TDPuller in my last commit to it :(
snej authored
236 revs = [revs sortedArrayUsingComparator: ^(id array1, id array2) {
237 return TDSequenceCompare( [[array1 objectAtIndex: 0] sequence],
238 [[array2 objectAtIndex: 0] sequence]);
239 }];
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
240 BOOL allGood = YES;
241 TDPulledRevision* lastGoodRev = nil;
242
243 [_db beginTransaction];
244 BOOL success = NO;
245 @try{
246 for (NSArray* revAndHistory in revs) {
247 @autoreleasepool {
248 TDPulledRevision* rev = [revAndHistory objectAtIndex: 0];
249 NSArray* history = [revAndHistory objectAtIndex: 1];
250 // Insert the revision:
251 int status = [_db forceInsert: rev revisionHistory: history source: _remote];
252 if (status >= 300) {
253 if (status == 403)
254 LogTo(Sync, @"%@: Remote rev failed validation: %@", self, rev);
255 else {
256 Warn(@"%@ failed to write %@: status=%d", self, rev, status);
257 self.error = TDHTTPError(status, nil);
258 allGood = NO; // stop advancing lastGoodRev
259 }
79f0fd5 @snej Added .error property to TDReplicator and TDChangeTracker.
snej authored
260 }
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
261
262 if (allGood)
263 lastGoodRev = rev;
8022a88 @snej Limit number of HTTP connections used by TDPuller
snej authored
264 }
56f4b33 @snej ToyPuller!
snej authored
265 }
3111ad8 @snej More replication work. It's working in the iOS demo now!
snej authored
266
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
267 // Now update self.lastSequence from the latest consecutively inserted revision:
268 unsigned lastGoodFakeSequence = (unsigned) lastGoodRev.sequence;
269 if (lastGoodFakeSequence > _maxInsertedFakeSequence) {
270 _maxInsertedFakeSequence = lastGoodFakeSequence;
271 self.lastSequence = lastGoodRev.remoteSequenceID;
272 }
273
274 LogTo(Sync, @"%@ finished inserting %u revisions", self, revs.count);
275 success = YES;
276 } @catch (NSException *x) {
9abe203 @snej Better exception reporting in TDPuller.
snej authored
277 MYReportException(x, @"%@: Exception inserting revisions", self);
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
278 } @finally {
279 [_db endTransaction: success];
280 }
7f5c5da @snej Replicator progress notification
snej authored
281
282 [self asyncTasksFinished: revs.count];
283 self.changesProcessed += revs.count;
56f4b33 @snej ToyPuller!
snej authored
284 }
285
286
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
287 - (NSArray*) knownCurrentRevIDsOf: (TDRevision*)rev {
288 return [_db getAllRevisionsOfDocumentID: rev.docID onlyCurrent: YES].allRevIDs;
289 }
290
291
56f4b33 @snej ToyPuller!
snej authored
292 @end
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
293
294
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
295
296 @implementation TDPulledRevision
297
298 @synthesize remoteSequenceID=_remoteSequenceID;
299
300 - (void) dealloc {
301 [_remoteSequenceID release];
302 [super dealloc];
303 }
304
305 @end
306
307
308
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
309 static NSString* joinQuotedEscaped(NSArray* strings) {
310 if (strings.count == 0)
311 return @"[]";
312 NSData* json = [NSJSONSerialization dataWithJSONObject: strings options: 0 error: NULL];
1cfeaad @snej Support for BigCouch (Cloudant) replication
snej authored
313 return TDEscapeURLParam([json my_UTF8ToString]);
da73a20 @snej Checkpoint: More attachment-pulling work
snej authored
314 }
Something went wrong with that request. Please try again.