forked from laurent22/joplin
/
file-api.ts
509 lines (413 loc) · 16.8 KB
/
file-api.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
import Logger from './Logger';
import shim from './shim';
import BaseItem from './models/BaseItem';
import time from './time';
const { isHidden } = require('./path-utils');
import JoplinError from './JoplinError';
const ArrayUtils = require('./ArrayUtils');
const { sprintf } = require('sprintf-js');
const Mutex = require('async-mutex').Mutex;
const logger = Logger.create('FileApi');
export interface MultiPutItem {
name: string;
body: string;
}
export interface RemoteItem {
id: string;
path?: string;
type_?: number;
isDeleted?: boolean;
// This the time when the file was created on the server. It is used for
// example for the locking mechanim or any file that's not an actual Joplin
// item.
updated_time?: number;
// This is the time that corresponds to the actual Joplin item updated_time
// value. A note is always uploaded with a delay so the server updated_time
// value will always be ahead. However for synchronising we need to know the
// exact Joplin item updated_time value.
jop_updated_time?: number;
}
export interface PaginatedList {
items: RemoteItem[];
has_more: boolean;
context: any;
}
function requestCanBeRepeated(error: any) {
const errorCode = typeof error === 'object' && error.code ? error.code : null;
// Unauthorized error - means username or password is incorrect or other
// permission issue, which won't be fixed by repeating the request.
if (errorCode === 403) return false;
// The target is explicitely rejecting the item so repeating wouldn't make a difference.
if (errorCode === 'rejectedByTarget') return false;
// We don't repeat failSafe errors because it's an indication of an issue at the
// server-level issue which usually cannot be fixed by repeating the request.
// Also we print the previous requests and responses to the log in this case,
// so not repeating means there will be less noise in the log.
if (errorCode === 'failSafe') return false;
return true;
}
async function tryAndRepeat(fn: Function, count: number) {
let retryCount = 0;
// Don't use internal fetch retry mechanim since we
// are already retrying here.
const shimFetchMaxRetryPrevious = shim.fetchMaxRetrySet(0);
const defer = () => {
shim.fetchMaxRetrySet(shimFetchMaxRetryPrevious);
};
while (true) {
try {
const result = await fn();
defer();
return result;
} catch (error) {
if (retryCount >= count || !requestCanBeRepeated(error)) {
defer();
throw error;
}
retryCount++;
await time.sleep(1 + retryCount * 3);
}
}
}
class FileApi {
private baseDir_: any;
private driver_: any;
private logger_: Logger = new Logger();
private syncTargetId_: number = null;
private tempDirName_: string = null;
public requestRepeatCount_: number = null; // For testing purpose only - normally this value should come from the driver
private remoteDateOffset_ = 0;
private remoteDateNextCheckTime_ = 0;
private remoteDateMutex_ = new Mutex();
private initialized_ = false;
constructor(baseDir: string | Function, driver: any) {
this.baseDir_ = baseDir;
this.driver_ = driver;
this.driver_.fileApi_ = this;
}
async initialize() {
if (this.initialized_) return;
this.initialized_ = true;
if (this.driver_.initialize) return this.driver_.initialize(this.fullPath(''));
}
// This can be true if the driver implements uploading items in batch. Will
// probably only be supported by Joplin Server.
public get supportsMultiPut(): boolean {
return !!this.driver().supportsMultiPut;
}
// This can be true when the sync target timestamps (updated_time) provided
// in the delta call are guaranteed to be accurate. That requires
// explicitely setting the timestamp, which is not done anymore on any sync
// target as it wasn't accurate (for example, the file system can't be
// relied on, and even OneDrive for some reason doesn't guarantee that the
// timestamp you set is what you get back).
//
// The only reliable one at the moment is Joplin Server since it reads the
// updated_time property directly from the item (it unserializes it
// server-side).
public get supportsAccurateTimestamp(): boolean {
return !!this.driver().supportsAccurateTimestamp;
}
async fetchRemoteDateOffset_() {
const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`;
const startTime = Date.now();
await this.put(tempFile, 'timeCheck');
// Normally it should be possible to read the file back immediately but
// just in case, read it in a loop.
const loopStartTime = Date.now();
let stat = null;
while (Date.now() - loopStartTime < 5000) {
stat = await this.stat(tempFile);
if (stat) break;
await time.msleep(200);
}
if (!stat) throw new Error('Timed out trying to get sync target clock time');
void this.delete(tempFile); // No need to await for this call
const endTime = Date.now();
const expectedTime = Math.round((endTime + startTime) / 2);
return stat.updated_time - expectedTime;
}
// Approximates the current time on the sync target. It caches the time offset to
// improve performance.
async remoteDate() {
const shouldSyncTime = () => {
return !this.remoteDateNextCheckTime_ || Date.now() > this.remoteDateNextCheckTime_;
};
if (shouldSyncTime()) {
const release = await this.remoteDateMutex_.acquire();
try {
// Another call might have refreshed the time while we were waiting for the mutex,
// so check again if we need to refresh.
if (shouldSyncTime()) {
this.remoteDateOffset_ = await this.fetchRemoteDateOffset_();
// The sync target clock should rarely change but the device one might,
// so we need to refresh relatively frequently.
this.remoteDateNextCheckTime_ = Date.now() + 10 * 60 * 1000;
}
} catch (error) {
logger.warn('Could not retrieve remote date - defaulting to device date:', error);
this.remoteDateOffset_ = 0;
this.remoteDateNextCheckTime_ = Date.now() + 60 * 1000;
} finally {
release();
}
}
return new Date(Date.now() + this.remoteDateOffset_);
}
// Ideally all requests repeating should be done at the FileApi level to remove duplicate code in the drivers, but
// historically some drivers (eg. OneDrive) are already handling request repeating, so this is optional, per driver,
// and it defaults to no repeating.
requestRepeatCount() {
if (this.requestRepeatCount_ !== null) return this.requestRepeatCount_;
if (this.driver_.requestRepeatCount) return this.driver_.requestRepeatCount();
return 0;
}
lastRequests() {
return this.driver_.lastRequests ? this.driver_.lastRequests() : [];
}
clearLastRequests() {
if (this.driver_.clearLastRequests) this.driver_.clearLastRequests();
}
baseDir() {
return typeof this.baseDir_ === 'function' ? this.baseDir_() : this.baseDir_;
}
tempDirName() {
if (this.tempDirName_ === null) throw Error('Temp dir not set!');
return this.tempDirName_;
}
setTempDirName(v: string) {
this.tempDirName_ = v;
}
fsDriver() {
return shim.fsDriver();
}
driver() {
return this.driver_;
}
setSyncTargetId(v: number) {
this.syncTargetId_ = v;
}
syncTargetId() {
if (this.syncTargetId_ === null) throw new Error('syncTargetId has not been set!!');
return this.syncTargetId_;
}
setLogger(l: Logger) {
if (!l) l = new Logger();
this.logger_ = l;
}
logger() {
return this.logger_;
}
fullPath(path: string) {
const output = [];
if (this.baseDir()) output.push(this.baseDir());
if (path) output.push(path);
return output.join('/');
}
// DRIVER MUST RETURN PATHS RELATIVE TO `path`
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
public async list(path = '', options: any = null): Promise<PaginatedList> {
if (!options) options = {};
if (!('includeHidden' in options)) options.includeHidden = false;
if (!('context' in options)) options.context = null;
if (!('includeDirs' in options)) options.includeDirs = true;
if (!('syncItemsOnly' in options)) options.syncItemsOnly = false;
logger.debug(`list ${this.baseDir()}`);
const result: PaginatedList = await tryAndRepeat(() => this.driver_.list(this.fullPath(path), options), this.requestRepeatCount());
if (!options.includeHidden) {
const temp = [];
for (let i = 0; i < result.items.length; i++) {
if (!isHidden(result.items[i].path)) temp.push(result.items[i]);
}
result.items = temp;
}
if (!options.includeDirs) {
result.items = result.items.filter((f: any) => !f.isDir);
}
if (options.syncItemsOnly) {
result.items = result.items.filter((f: any) => !f.isDir && BaseItem.isSystemPath(f.path));
}
return result;
}
// Deprectated
setTimestamp(path: string, timestampMs: number) {
logger.debug(`setTimestamp ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.setTimestamp(this.fullPath(path), timestampMs), this.requestRepeatCount());
// return this.driver_.setTimestamp(this.fullPath(path), timestampMs);
}
mkdir(path: string) {
logger.debug(`mkdir ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.mkdir(this.fullPath(path)), this.requestRepeatCount());
}
async stat(path: string) {
logger.debug(`stat ${this.fullPath(path)}`);
const output = await tryAndRepeat(() => this.driver_.stat(this.fullPath(path)), this.requestRepeatCount());
if (!output) return output;
output.path = path;
return output;
}
// Returns UTF-8 encoded string by default, or a Response if `options.target = 'file'`
get(path: string, options: any = null) {
if (!options) options = {};
if (!options.encoding) options.encoding = 'utf8';
logger.debug(`get ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.get(this.fullPath(path), options), this.requestRepeatCount());
}
async put(path: string, content: any, options: any = null) {
logger.debug(`put ${this.fullPath(path)}`, options);
if (options && options.source === 'file') {
if (!(await this.fsDriver().exists(options.path))) throw new JoplinError(`File not found: ${options.path}`, 'fileNotFound');
}
return tryAndRepeat(() => this.driver_.put(this.fullPath(path), content, options), this.requestRepeatCount());
}
public async multiPut(items: MultiPutItem[], options: any = null) {
if (!this.driver().supportsMultiPut) throw new Error('Multi PUT not supported');
return tryAndRepeat(() => this.driver_.multiPut(items, options), this.requestRepeatCount());
}
delete(path: string) {
logger.debug(`delete ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.delete(this.fullPath(path)), this.requestRepeatCount());
}
// Deprectated
move(oldPath: string, newPath: string) {
logger.debug(`move ${this.fullPath(oldPath)} => ${this.fullPath(newPath)}`);
return tryAndRepeat(() => this.driver_.move(this.fullPath(oldPath), this.fullPath(newPath)), this.requestRepeatCount());
}
// Deprectated
format() {
return tryAndRepeat(() => this.driver_.format(), this.requestRepeatCount());
}
clearRoot() {
return tryAndRepeat(() => this.driver_.clearRoot(this.baseDir()), this.requestRepeatCount());
}
delta(path: string, options: any = null) {
logger.debug(`delta ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.delta(this.fullPath(path), options), this.requestRepeatCount());
}
}
function basicDeltaContextFromOptions_(options: any) {
const output: any = {
timestamp: 0,
filesAtTimestamp: [],
statsCache: null,
statIdsCache: null,
deletedItemsProcessed: false,
};
if (!options || !options.context) return output;
const d = new Date(options.context.timestamp);
output.timestamp = isNaN(d.getTime()) ? 0 : options.context.timestamp;
output.filesAtTimestamp = Array.isArray(options.context.filesAtTimestamp) ? options.context.filesAtTimestamp.slice() : [];
output.statsCache = options.context && options.context.statsCache ? options.context.statsCache : null;
output.statIdsCache = options.context && options.context.statIdsCache ? options.context.statIdsCache : null;
output.deletedItemsProcessed = options.context && 'deletedItemsProcessed' in options.context ? options.context.deletedItemsProcessed : false;
return output;
}
// This is the basic delta algorithm, which can be used in case the cloud service does not have
// a built-in delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously
// the file system do not.
async function basicDelta(path: string, getDirStatFn: Function, options: any) {
const outputLimit = 50;
const itemIds = await options.allItemIdsHandler();
if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
const logger = options && options.logger ? options.logger : new Logger();
const context = basicDeltaContextFromOptions_(options);
if (context.timestamp > Date.now()) {
logger.warn(`BasicDelta: Context timestamp is greater than current time: ${context.timestamp}`);
logger.warn('BasicDelta: Sync will continue but it is likely that nothing will be synced');
}
const newContext = {
timestamp: context.timestamp,
filesAtTimestamp: context.filesAtTimestamp.slice(),
statsCache: context.statsCache,
statIdsCache: context.statIdsCache,
deletedItemsProcessed: context.deletedItemsProcessed,
};
// Stats are cached until all items have been processed (until hasMore is false)
if (newContext.statsCache === null) {
newContext.statsCache = await getDirStatFn(path);
newContext.statsCache.sort(function(a: any, b: any) {
return a.updated_time - b.updated_time;
});
newContext.statIdsCache = newContext.statsCache.filter((item: any) => BaseItem.isSystemPath(item.path)).map((item: any) => BaseItem.pathToId(item.path));
newContext.statIdsCache.sort(); // Items must be sorted to use binary search below
}
let output = [];
const updateReport = {
timestamp: context.timestamp,
older: 0,
newer: 0,
equal: 0,
};
// Find out which files have been changed since the last time. Note that we keep
// both the timestamp of the most recent change, *and* the items that exactly match
// this timestamp. This to handle cases where an item is modified while this delta
// function is running. For example:
// t0: Item 1 is changed
// t0: Sync items - run delta function
// t0: While delta() is running, modify Item 2
// Since item 2 was modified within the same millisecond, it would be skipped in the
// next sync if we relied exclusively on a timestamp.
for (let i = 0; i < newContext.statsCache.length; i++) {
const stat = newContext.statsCache[i];
if (stat.isDir) continue;
if (stat.updated_time < context.timestamp) {
updateReport.older++;
continue;
}
// Special case for items that exactly match the timestamp
if (stat.updated_time === context.timestamp) {
if (context.filesAtTimestamp.indexOf(stat.path) >= 0) {
updateReport.equal++;
continue;
}
}
if (stat.updated_time > newContext.timestamp) {
newContext.timestamp = stat.updated_time;
newContext.filesAtTimestamp = [];
updateReport.newer++;
}
newContext.filesAtTimestamp.push(stat.path);
output.push(stat);
if (output.length >= outputLimit) break;
}
logger.info(`BasicDelta: Report: ${JSON.stringify(updateReport)}`);
if (!newContext.deletedItemsProcessed) {
// Find out which items have been deleted on the sync target by comparing the items
// we have to the items on the target.
// Note that when deleted items are processed it might result in the output having
// more items than outputLimit. This is acceptable since delete operations are cheap.
const deletedItems = [];
for (let i = 0; i < itemIds.length; i++) {
const itemId = itemIds[i];
if (ArrayUtils.binarySearch(newContext.statIdsCache, itemId) < 0) {
deletedItems.push({
path: BaseItem.systemPath(itemId),
isDeleted: true,
});
}
}
const percentDeleted = itemIds.length ? deletedItems.length / itemIds.length : 0;
// If more than 90% of the notes are going to be deleted, it's most likely a
// configuration error or bug. For example, if the user moves their Nextcloud
// directory, or if a network drive gets disconnected and returns an empty dir
// instead of an error. In that case, we don't wipe out the user data, unless
// they have switched off the fail-safe.
if (options.wipeOutFailSafe && percentDeleted >= 0.90) throw new JoplinError(sprintf('Fail-safe: Sync was interrupted because %d%% of the data (%d items) is about to be deleted. To override this behaviour disable the fail-safe in the sync settings.', Math.round(percentDeleted * 100), deletedItems.length), 'failSafe');
output = output.concat(deletedItems);
}
newContext.deletedItemsProcessed = true;
const hasMore = output.length >= outputLimit;
if (!hasMore) {
// Clear temporary info from context. It's especially important to remove deletedItemsProcessed
// so that they are processed again on the next sync.
newContext.statsCache = null;
newContext.statIdsCache = null;
delete newContext.deletedItemsProcessed;
}
return {
hasMore: hasMore,
context: newContext,
items: output,
};
}
export { FileApi, basicDelta };