From f0bffbefcaad7f02af477726009e969162d819eb Mon Sep 17 00:00:00 2001 From: Karl Semich Date: Sat, 2 Nov 2019 09:42:09 -0400 Subject: [PATCH 1/2] backFile() implementation Provides for HTTP backing of files when there is no .superblock.txt file. --- README.md | 20 ++++++++++++---- src/CacheFS.js | 4 ++-- src/HttpBackend.js | 10 +++++++- src/PromisifiedFS.js | 43 ++++++++++++++++++++++++++++++----- src/__tests__/CacheFS.spec.js | 4 ++-- src/index.js | 5 ++++ 6 files changed, 71 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 22ade21..98c7b55 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ const fs = new FS("testfs") Options object: -| Param | Type [= default] | Description | -| ------ | ------------------ | --------------------------------------------------------------------- | -| `wipe` | boolean = false | Delete the database and start with an empty filesystem | -| `url` | string = undefined | Let `readFile` requests fall back to an HTTP request to this base URL | +| Param | Type [= default] | Description | +| --------- | ------------------ | --------------------------------------------------------------------- | +| `wipe` | boolean = false | Delete the database and start with an empty filesystem | +| `url` | string = undefined | Let `readFile` requests fall back to an HTTP request to this base URL | +| `urlauto` | boolean = false | Fall back to HTTP for every read of a missing file, even if unbacked | ### `fs.mkdir(filepath, opts?, cb)` @@ -149,6 +150,17 @@ Create a symlink at `filepath` that points to `target`. Read the target of a symlink. +### `fs.backFile(filepath, opts?, cb)` + +Create or change the stat data for a file backed by HTTP. Size is fetched with a HEAD request. Useful when using an HTTP backend without `urlauto` set, as then files will only be readable if they have stat data. +Note that stat data is made automatically from the file `/.superblock.txt` if found on the server. `/.superblock.txt` can be generated or updated with the [included standalone script](src/superblocktxt.js). + +Options object: + +| Param | Type [= default] | Description | +| ---------- | ------------------ | -------------------------------- | +| `mode` | number = 0o666 | Posix mode permissions | + ### `fs.promises` All the same functions as above, but instead of passing a callback they return a promise. diff --git a/src/CacheFS.js b/src/CacheFS.js index bc6b123..720fb57 100755 --- a/src/CacheFS.js +++ b/src/CacheFS.js @@ -160,7 +160,7 @@ module.exports = class CacheFS { let dir = this._lookup(filepath); return [...dir.keys()].filter(key => typeof key === "string"); } - writeFile(filepath, data, { mode }) { + writeStat(filepath, size, { mode }) { let ino; try { let oldStat = this.stat(filepath); @@ -180,7 +180,7 @@ module.exports = class CacheFS { let stat = { mode, type: "file", - size: data.length, + size, mtimeMs: Date.now(), ino, }; diff --git a/src/HttpBackend.js b/src/HttpBackend.js index 94d35f4..6106dcb 100644 --- a/src/HttpBackend.js +++ b/src/HttpBackend.js @@ -3,7 +3,7 @@ module.exports = class HttpBackend { this._url = url; } loadSuperblock() { - return fetch(this._url + '/.superblock.txt').then(res => res.text()) + return fetch(this._url + '/.superblock.txt').then(res => res.ok ? res.text() : null) } async readFile(filepath) { const res = await fetch(this._url + filepath) @@ -13,4 +13,12 @@ module.exports = class HttpBackend { throw new Error('ENOENT') } } + async sizeFile(filepath) { + const res = await fetch(this._url + filepath, { method: 'HEAD' }) + if (res.status === 200) { + return res.headers.get('content-length') + } else { + throw new Error('ENOENT') + } + } } diff --git a/src/PromisifiedFS.js b/src/PromisifiedFS.js index 86ad0ff..df0fd65 100644 --- a/src/PromisifiedFS.js +++ b/src/PromisifiedFS.js @@ -33,7 +33,7 @@ function cleanParams2(oldFilepath, newFilepath) { } module.exports = class PromisifiedFS { - constructor(name, { wipe, url } = {}) { + constructor(name, { wipe, url, urlauto } = {}) { this._name = name this._idb = new IdbBackend(name); this._mutex = new Mutex(name); @@ -45,6 +45,7 @@ module.exports = class PromisifiedFS { }, 500); if (url) { this._http = new HttpBackend(url) + this._urlauto = !!urlauto } this._operations = new Set() @@ -59,6 +60,7 @@ module.exports = class PromisifiedFS { this.lstat = this._wrap(this.lstat, false) this.readlink = this._wrap(this.readlink, false) this.symlink = this._wrap(this.symlink, true) + this.backFile = this._wrap(this.backFile, true) this._deactivationPromise = null this._deactivationTimeout = null @@ -143,18 +145,41 @@ module.exports = class PromisifiedFS { await this._idb.saveSuperblock(this._cache._root); } } + async _writeStat(filepath, size, opts) { + let dirparts = path.split(path.dirname(filepath)) + let dir = dirparts.shift() + for (let dirpart of dirparts) { + dir = path.join(dir, dirpart) + try { + this._cache.mkdir(dir, { mode: 0o777 }) + } catch (e) {} + } + return this._cache.writeStat(filepath, size, opts) + } async readFile(filepath, opts) { ;[filepath, opts] = cleanParams(filepath, opts); const { encoding } = opts; if (encoding && encoding !== 'utf8') throw new Error('Only "utf8" encoding is supported in readFile'); - const stat = this._cache.stat(filepath); - let data = await this._idb.readFile(stat.ino) + let data = null, stat = null + try { + stat = this._cache.stat(filepath); + data = await this._idb.readFile(stat.ino) + } catch (e) { + if (!this._urlauto) throw e + } if (!data && this._http) { data = await this._http.readFile(filepath) } - if (data && encoding === "utf8") { - data = decode(data); + if (data) { + if (!stat || stat.size != data.byteLength) { + stat = await this._writeStat(filepath, data.byteLength, { mode: stat ? stat.mode : 0o666 }) + this.saveSuperblock() // debounced + } + if (encoding === "utf8") { + data = decode(data); + } } + if (!stat) throw new ENOENT(filepath) return data; } async writeFile(filepath, data, opts) { @@ -166,7 +191,7 @@ module.exports = class PromisifiedFS { } data = encode(data); } - const stat = this._cache.writeFile(filepath, data, { mode }); + const stat = await this._cache.writeStat(filepath, data.byteLength, { mode }); await this._idb.writeFile(stat.ino, data) return null } @@ -222,4 +247,10 @@ module.exports = class PromisifiedFS { this._cache.symlink(target, filepath); return null; } + async backFile(filepath, opts) { + ;[filepath, opts] = cleanParams(filepath, opts); + let size = await this._http.sizeFile(filepath) + await this._writeStat(filepath, size, opts) + return null + } } diff --git a/src/__tests__/CacheFS.spec.js b/src/__tests__/CacheFS.spec.js index 4e5e396..2bd6115 100644 --- a/src/__tests__/CacheFS.spec.js +++ b/src/__tests__/CacheFS.spec.js @@ -21,7 +21,7 @@ describe("CacheFS module", () => { const fs = new CacheFS(); fs.activate() expect(fs.autoinc()).toEqual(1) - fs.writeFile('/foo', 'bar', {}) + fs.writeStat('/foo', 3, {}) expect(fs.autoinc()).toEqual(2) fs.mkdir('/bar', {}) expect(fs.autoinc()).toEqual(3) @@ -33,7 +33,7 @@ describe("CacheFS module", () => { expect(fs.autoinc()).toEqual(3) fs.mkdir('/bar/bar', {}) expect(fs.autoinc()).toEqual(4) - fs.writeFile('/bar/bar/boo', 'bar', {}) + fs.writeStat('/bar/bar/boo', 3, {}) expect(fs.autoinc()).toEqual(5) fs.unlink('/bar/bar/boo') expect(fs.autoinc()).toEqual(4) diff --git a/src/index.js b/src/index.js index 70e3c38..9082702 100755 --- a/src/index.js +++ b/src/index.js @@ -26,6 +26,7 @@ module.exports = class FS { this.lstat = this.lstat.bind(this) this.readlink = this.readlink.bind(this) this.symlink = this.symlink.bind(this) + this.backFile = this.backFile.bind(this) } readFile(filepath, opts, cb) { const [resolve, reject] = wrapCallback(opts, cb); @@ -71,4 +72,8 @@ module.exports = class FS { const [resolve, reject] = wrapCallback(cb); this.promises.symlink(target, filepath).then(resolve).catch(reject); } + backFile(filepath, size, opts, cb) { + const [resolve, reject] = wrapCallback(opts, cb); + this.promises.backFile(filepath, size, opts).then(resolve).catch(reject); + } } From fa2b11fc43459ae4ac3659331b40d8e24baa8c4a Mon Sep 17 00:00:00 2001 From: Karl Semich Date: Sat, 2 Nov 2019 19:06:48 -0400 Subject: [PATCH 2/2] add tests for backFile() --- .../test-folder/not-in-superblock.txt | 1 + src/__tests__/fallback.spec.js | 27 +++++++++++++++++++ src/index.js | 4 +-- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 src/__tests__/__fixtures__/test-folder/not-in-superblock.txt diff --git a/src/__tests__/__fixtures__/test-folder/not-in-superblock.txt b/src/__tests__/__fixtures__/test-folder/not-in-superblock.txt new file mode 100644 index 0000000..8e6d184 --- /dev/null +++ b/src/__tests__/__fixtures__/test-folder/not-in-superblock.txt @@ -0,0 +1 @@ +Hello from "not-in-superblock" \ No newline at end of file diff --git a/src/__tests__/fallback.spec.js b/src/__tests__/fallback.spec.js index b880175..b9b2d45 100644 --- a/src/__tests__/fallback.spec.js +++ b/src/__tests__/fallback.spec.js @@ -38,6 +38,12 @@ describe("http fallback", () => { done(); }); }); + it("read file not in superblock throws", done => { + fs.readFile("/not-in-superblock.txt", (err, data) => { + expect(err).not.toBe(null); + done(); + }); + }); it("read file /a.txt", done => { fs.readFile("/a.txt", 'utf8', (err, data) => { expect(err).toBe(null); @@ -82,4 +88,25 @@ describe("http fallback", () => { }); }); }); + describe("backFile", () => { + it("backing a nonexistant file throws", done => { + fs.backFile("/backFile/non-existant.txt", (err, data) => { + expect(err).not.toBe(null); + done(); + }); + }); + it("backing a file makes it readable", done => { + fs.backFile("/not-in-superblock.txt", (err, data) => { + expect(err).toBe(null) + fs.readFile("/not-in-superblock.txt", 'utf8', (err, data) => { + expect(err).toBe(null); + expect(data).toEqual('Hello from "not-in-superblock"'); + fs.unlink("/not-in-superblock.txt", (err, data) => { + expect(err).toBe(null); + done(); + }); + }); + }); + }); + }); }); diff --git a/src/index.js b/src/index.js index 9082702..1c15f1a 100755 --- a/src/index.js +++ b/src/index.js @@ -72,8 +72,8 @@ module.exports = class FS { const [resolve, reject] = wrapCallback(cb); this.promises.symlink(target, filepath).then(resolve).catch(reject); } - backFile(filepath, size, opts, cb) { + backFile(filepath, opts, cb) { const [resolve, reject] = wrapCallback(opts, cb); - this.promises.backFile(filepath, size, opts).then(resolve).catch(reject); + this.promises.backFile(filepath, opts).then(resolve).catch(reject); } }