-
Notifications
You must be signed in to change notification settings - Fork 445
/
agent.js
591 lines (533 loc) · 17.6 KB
/
agent.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
var hat = require('hat');
var util = require('./util');
var types = require('./types');
var logger = require('./logger');
/**
* Agent deserializes the wire protocol messages received from the stream and
* calls the corresponding functions on its Agent. It uses the return values
* to send responses back. Agent also handles piping the operation streams
* provided by a Agent.
*
* @param {Backend} backend
* @param {Duplex} stream connection to a client
*/
function Agent(backend, stream) {
this.backend = backend;
this.stream = stream;
this.clientId = hat();
this.connectTime = Date.now();
// We need to track which documents are subscribed by the client. This is a
// map of collection -> id -> stream
this.subscribedDocs = {};
// Map from queryId -> emitter
this.subscribedQueries = {};
// We need to track this manually to make sure we don't reply to messages
// after the stream was closed.
this.closed = false;
// For custom use in middleware. The agent is a convenient place to cache
// session state in memory. It is in memory only as long as the session is
// active, and it is passed to each middleware call
this.custom = {};
// Initialize the remote client by sending it its agent Id.
this.send({
a: 'init',
protocol: 1,
id: this.clientId,
type: types.defaultType.uri
});
}
module.exports = Agent;
// Close the agent with the client.
Agent.prototype.close = function(err) {
if (err) {
logger.warn('Agent closed due to error', this.clientId, err.stack || err);
}
if (this.closed) return;
// This will end the writable stream and emit 'finish'
this.stream.end();
};
Agent.prototype._cleanup = function() {
this.closed = true;
// Clean up doc subscription streams
for (var collection in this.subscribedDocs) {
var docs = this.subscribedDocs[collection];
for (var id in docs) {
var stream = docs[id];
stream.destroy();
}
}
this.subscribedDocs = {};
// Clean up query subscription streams
for (var id in this.subscribedQueries) {
var emitter = this.subscribedQueries[id];
emitter.destroy();
}
this.subscribedQueries = {};
};
/**
* Passes operation data received on stream to the agent stream via
* _sendOp()
*/
Agent.prototype._subscribeToStream = function(collection, id, stream) {
if (this.closed) return stream.destroy();
var streams = this.subscribedDocs[collection] || (this.subscribedDocs[collection] = {});
// If already subscribed to this document, destroy the previously subscribed stream
var previous = streams[id];
if (previous) previous.destroy();
streams[id] = stream;
var agent = this;
stream.on('data', function(data) {
if (data.error) {
// Log then silently ignore errors in a subscription stream, since these
// may not be the client's fault, and they were not the result of a
// direct request by the client
logger.error('Doc subscription stream error', collection, id, data.error);
return;
}
if (agent._isOwnOp(collection, data)) return;
agent._sendOp(collection, id, data);
});
stream.on('end', function() {
// The op stream is done sending, so release its reference
var streams = agent.subscribedDocs[collection];
if (!streams) return;
delete streams[id];
if (util.hasKeys(streams)) return;
delete agent.subscribedDocs[collection];
});
};
Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query) {
var previous = this.subscribedQueries[queryId];
if (previous) previous.destroy();
this.subscribedQueries[queryId] = emitter;
var agent = this;
emitter.onExtra = function(extra) {
agent.send({a: 'q', id: queryId, extra: extra});
};
emitter.onDiff = function(diff) {
for (var i = 0; i < diff.length; i++) {
var item = diff[i];
if (item.type === 'insert') {
item.values = getResultsData(item.values);
}
}
// Consider stripping the collection out of the data we send here
// if it matches the query's collection.
agent.send({a: 'q', id: queryId, diff: diff});
};
emitter.onError = function(err) {
// Log then silently ignore errors in a subscription stream, since these
// may not be the client's fault, and they were not the result of a
// direct request by the client
logger.error('Query subscription stream error', collection, query, err);
};
emitter.onOp = function(op) {
var id = op.d;
if (agent._isOwnOp(collection, op)) return;
agent._sendOp(collection, id, op);
};
emitter._open();
};
Agent.prototype._isOwnOp = function(collection, op) {
// Detect ops from this client on the same projection. Since the client sent
// these in, the submit reply will be sufficient and we can silently ignore
// them in the streams for subscribed documents or queries
return (this.clientId === op.src) && (collection === (op.i || op.c));
};
Agent.prototype.send = function(message) {
// Quietly drop replies if the stream was closed
if (this.closed) return;
this.backend.emit('send', this, message);
this.stream.write(message);
};
Agent.prototype._sendOp = function(collection, id, op) {
var message = {
a: 'op',
c: collection,
d: id,
v: op.v,
src: op.src,
seq: op.seq
};
if (op.op) message.op = op.op;
if (op.create) message.create = op.create;
if (op.del) message.del = true;
this.send(message);
};
Agent.prototype._sendOps = function(collection, id, ops) {
for (var i = 0; i < ops.length; i++) {
this._sendOp(collection, id, ops[i]);
}
};
Agent.prototype._reply = function(request, err, message) {
if (err) {
if (typeof err === 'string') {
request.error = {
code: 4001,
message: err
};
} else {
if (err.stack) {
logger.warn(err.stack);
}
request.error = {
code: err.code,
message: err.message
};
}
this.send(request);
return;
}
if (!message) message = {};
message.a = request.a;
if (request.id) {
message.id = request.id;
} else {
if (request.c) message.c = request.c;
if (request.d) message.d = request.d;
if (request.b && !message.data) message.b = request.b;
}
this.send(message);
};
// Start processing events from the stream
Agent.prototype._open = function() {
if (this.closed) return;
this.backend.agentsCount++;
if (!this.stream.isServer) this.backend.remoteAgentsCount++;
var agent = this;
this.stream.on('data', function(chunk) {
if (agent.closed) return;
if (typeof chunk !== 'object') {
var err = {code: 4000, message: 'Received non-object message'};
return agent.close(err);
}
var request = {data: chunk};
agent.backend.trigger('receive', agent, request, function(err) {
var callback = function(err, message) {
agent._reply(request.data, err, message);
};
if (err) return callback(err);
agent._handleMessage(request.data, callback);
});
});
this.stream.on('end', function() {
agent.backend.agentsCount--;
if (!agent.stream.isServer) agent.backend.remoteAgentsCount--;
agent._cleanup();
});
};
// Check a request to see if its valid. Returns an error if there's a problem.
Agent.prototype._checkRequest = function(request) {
if (request.a === 'qf' || request.a === 'qs' || request.a === 'qu') {
// Query messages need an ID property.
if (typeof request.id !== 'number') return 'Missing query ID';
} else if (request.a === 'op' || request.a === 'f' || request.a === 's' || request.a === 'u') {
// Doc-based request.
if (request.c != null && typeof request.c !== 'string') return 'Invalid collection';
if (request.d != null && typeof request.d !== 'string') return 'Invalid id';
if (request.a === 'op') {
if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version';
}
} else if (request.a === 'bf' || request.a === 'bs' || request.a === 'bu') {
// Bulk request
if (request.c != null && typeof request.c !== 'string') return 'Invalid collection';
if (typeof request.b !== 'object') return 'Invalid bulk subscribe data';
}
};
// Handle an incoming message from the client
Agent.prototype._handleMessage = function(request, callback) {
try {
var errMessage = this._checkRequest(request);
if (errMessage) return callback({code: 4000, message: errMessage});
switch (request.a) {
case 'qf':
return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback);
case 'qs':
return this._querySubscribe(request.id, request.c, request.q, getQueryOptions(request), callback);
case 'qu':
return this._queryUnsubscribe(request.id, callback);
case 'bf':
return this._fetchBulk(request.c, request.b, callback);
case 'bs':
return this._subscribeBulk(request.c, request.b, callback);
case 'bu':
return this._unsubscribeBulk(request.c, request.b, callback);
case 'f':
return this._fetch(request.c, request.d, request.v, callback);
case 's':
return this._subscribe(request.c, request.d, request.v, callback);
case 'u':
return this._unsubscribe(request.c, request.d, callback);
case 'op':
var op = this._createOp(request);
if (!op) return callback({code: 4000, message: 'Invalid op message'});
return this._submit(request.c, request.d, op, callback);
case 'nf':
return this._fetchSnapshot(request.c, request.d, request.v, callback);
default:
callback({code: 4000, message: 'Invalid or unknown message'});
}
} catch (err) {
callback(err);
}
};
function getQueryOptions(request) {
var results = request.r;
var ids, fetch, fetchOps;
if (results) {
ids = [];
for (var i = 0; i < results.length; i++) {
var result = results[i];
var id = result[0];
var version = result[1];
ids.push(id);
if (version == null) {
if (fetch) {
fetch.push(id);
} else {
fetch = [id];
}
} else {
if (!fetchOps) fetchOps = {};
fetchOps[id] = version;
}
}
}
var options = request.o || {};
options.ids = ids;
options.fetch = fetch;
options.fetchOps = fetchOps;
return options;
}
Agent.prototype._queryFetch = function(queryId, collection, query, options, callback) {
// Fetch the results of a query once
var agent = this;
this.backend.queryFetch(this, collection, query, options, function(err, results, extra) {
if (err) return callback(err);
var message = {
data: getResultsData(results),
extra: extra
};
callback(null, message);
});
};
Agent.prototype._querySubscribe = function(queryId, collection, query, options, callback) {
// Subscribe to a query. The client is sent the query results and its
// notified whenever there's a change
var agent = this;
var wait = 1;
var message;
function finish(err) {
if (err) return callback(err);
if (--wait) return;
callback(null, message);
}
if (options.fetch) {
wait++;
this.backend.fetchBulk(this, collection, options.fetch, function(err, snapshotMap) {
if (err) return finish(err);
message = {data: getMapData(snapshotMap)};
finish();
});
}
if (options.fetchOps) {
wait++;
this._fetchBulkOps(collection, options.fetchOps, finish);
}
this.backend.querySubscribe(this, collection, query, options, function(err, emitter, results, extra) {
if (err) return finish(err);
if (this.closed) return emitter.destroy();
agent._subscribeToQuery(emitter, queryId, collection, query);
// No results are returned when ids are passed in as an option. Instead,
// want to re-poll the entire query once we've established listeners to
// emit any diff in results
if (!results) {
emitter.queryPoll(finish);
return;
}
message = {
data: getResultsData(results),
extra: extra
};
finish();
});
};
function getResultsData(results) {
var items = [];
for (var i = 0; i < results.length; i++) {
var result = results[i];
var item = getSnapshotData(result);
item.d = result.id;
items.push(item);
}
return items;
}
function getMapData(snapshotMap) {
var data = {};
for (var id in snapshotMap) {
data[id] = getSnapshotData(snapshotMap[id]);
}
return data;
}
function getSnapshotData(snapshot) {
var data = {
v: snapshot.v,
data: snapshot.data
};
if (types.defaultType !== types.map[snapshot.type]) {
data.type = snapshot.type;
}
return data;
}
Agent.prototype._queryUnsubscribe = function(queryId, callback) {
var emitter = this.subscribedQueries[queryId];
if (emitter) {
emitter.destroy();
delete this.subscribedQueries[queryId];
}
process.nextTick(callback);
};
Agent.prototype._fetch = function(collection, id, version, callback) {
if (version == null) {
// Fetch a snapshot
this.backend.fetch(this, collection, id, function(err, snapshot) {
if (err) return callback(err);
callback(null, {data: getSnapshotData(snapshot)});
});
} else {
// It says fetch on the tin, but if a version is specified the client
// actually wants me to fetch some ops
this._fetchOps(collection, id, version, callback);
}
};
Agent.prototype._fetchOps = function(collection, id, version, callback) {
var agent = this;
this.backend.getOps(this, collection, id, version, null, function(err, ops) {
if (err) return callback(err);
agent._sendOps(collection, id, ops);
callback();
});
};
Agent.prototype._fetchBulk = function(collection, versions, callback) {
if (Array.isArray(versions)) {
this.backend.fetchBulk(this, collection, versions, function(err, snapshotMap) {
if (err) return callback(err);
callback(null, {data: getMapData(snapshotMap)});
});
} else {
this._fetchBulkOps(collection, versions, callback);
}
};
Agent.prototype._fetchBulkOps = function(collection, versions, callback) {
var agent = this;
this.backend.getOpsBulk(this, collection, versions, null, function(err, opsMap) {
if (err) return callback(err);
for (var id in opsMap) {
var ops = opsMap[id];
agent._sendOps(collection, id, ops);
}
callback();
});
};
Agent.prototype._subscribe = function(collection, id, version, callback) {
// If the version is specified, catch the client up by sending all ops
// since the specified version
var agent = this;
this.backend.subscribe(this, collection, id, version, function(err, stream, snapshot) {
if (err) return callback(err);
agent._subscribeToStream(collection, id, stream);
// Snapshot is returned only when subscribing from a null version.
// Otherwise, ops will have been pushed into the stream
if (snapshot) {
callback(null, {data: getSnapshotData(snapshot)});
} else {
callback();
}
});
};
Agent.prototype._subscribeBulk = function(collection, versions, callback) {
var agent = this;
this.backend.subscribeBulk(this, collection, versions, function(err, streams, snapshotMap) {
if (err) return callback(err);
for (var id in streams) {
agent._subscribeToStream(collection, id, streams[id]);
}
if (snapshotMap) {
callback(null, {data: getMapData(snapshotMap)});
} else {
callback();
}
});
};
Agent.prototype._unsubscribe = function(collection, id, callback) {
// Unsubscribe from the specified document. This cancels the active
// stream or an inflight subscribing state
var docs = this.subscribedDocs[collection];
var stream = docs && docs[id];
if (stream) stream.destroy();
process.nextTick(callback);
};
Agent.prototype._unsubscribeBulk = function(collection, ids, callback) {
var docs = this.subscribedDocs[collection];
if (!docs) return process.nextTick(callback);
for (var i = 0; i < ids.length; i++) {
var id = ids[i];
var stream = docs[id];
if (stream) stream.destroy();
}
process.nextTick(callback);
};
Agent.prototype._submit = function(collection, id, op, callback) {
var agent = this;
this.backend.submit(this, collection, id, op, null, function(err, ops) {
// Message to acknowledge the op was successfully submitted
var ack = {src: op.src, seq: op.seq, v: op.v};
if (err) {
// Occassional 'Op already submitted' errors are expected to happen as
// part of normal operation, since inflight ops need to be resent after
// disconnect. In this case, ack the op so the client can proceed
if (err.code === 4001) return callback(null, ack);
return callback(err);
}
// Reply with any operations that the client is missing.
agent._sendOps(collection, id, ops);
callback(null, ack);
});
};
function CreateOp(src, seq, v, create) {
this.src = src;
this.seq = seq;
this.v = v;
this.create = create;
this.m = null;
}
function EditOp(src, seq, v, op) {
this.src = src;
this.seq = seq;
this.v = v;
this.op = op;
this.m = null;
}
function DeleteOp(src, seq, v, del) {
this.src = src;
this.seq = seq;
this.v = v;
this.del = del;
this.m = null;
}
// Normalize the properties submitted
Agent.prototype._createOp = function(request) {
// src can be provided if it is not the same as the current agent,
// such as a resubmission after a reconnect, but it usually isn't needed
var src = request.src || this.clientId;
if (request.op) {
return new EditOp(src, request.seq, request.v, request.op);
} else if (request.create) {
return new CreateOp(src, request.seq, request.v, request.create);
} else if (request.del) {
return new DeleteOp(src, request.seq, request.v, request.del);
}
};
Agent.prototype._fetchSnapshot = function (collection, id, version, callback) {
this.backend.fetchSnapshot(this, collection, id, version, callback);
};