backFile(): add http-backed files to superblock

README.md

@@ -59,10 +59,11 @@ const fs = new FS("testfs")
 
 Options object:
 
-| Param  | Type [= default]   | Description                                                           |
-| ------ | ------------------ | --------------------------------------------------------------------- |
-| `wipe` | boolean = false    | Delete the database and start with an empty filesystem                |
-| `url`  | string = undefined | Let `readFile` requests fall back to an HTTP request to this base URL |
+| Param     | Type [= default]   | Description                                                           |
+| --------- | ------------------ | --------------------------------------------------------------------- |
+| `wipe`    | boolean = false    | Delete the database and start with an empty filesystem                |
+| `url`     | string = undefined | Let `readFile` requests fall back to an HTTP request to this base URL |
+| `urlauto` | boolean = false    | Fall back to HTTP for every read of a missing file, even if unbacked  |
 
 ### `fs.mkdir(filepath, opts?, cb)`
 
@@ -149,6 +150,17 @@ Create a symlink at `filepath` that points to `target`.
 
 Read the target of a symlink.
 
+### `fs.backFile(filepath, opts?, cb)`
+
+Create or change the stat data for a file backed by HTTP.  Size is fetched with a HEAD request.  Useful when using an HTTP backend without `urlauto` set, as then files will only be readable if they have stat data.
+Note that stat data is made automatically from the file `/.superblock.txt` if found on the server.  `/.superblock.txt` can be generated or updated with the [included standalone script](src/superblocktxt.js).
+
+Options object:
+
+| Param      | Type [= default]   | Description                      |
+| ---------- | ------------------ | -------------------------------- |
+| `mode`     | number = 0o666     | Posix mode permissions           |
+
 ### `fs.promises`
 
 All the same functions as above, but instead of passing a callback they return a promise.

src/CacheFS.js

@@ -163,7 +163,7 @@ module.exports = class CacheFS {
     if (dir.get(STAT).type !== 'dir') throw new ENOTDIR();
     return [...dir.keys()].filter(key => typeof key === "string");
   }
-  writeFile(filepath, data, { mode }) {
+  writeStat(filepath, size, { mode }) {
     let ino;
     try {
       let oldStat = this.stat(filepath);
@@ -183,7 +183,7 @@ module.exports = class CacheFS {
     let stat = {
       mode,
       type: "file",
-      size: data.length,
+      size,
       mtimeMs: Date.now(),
       ino,
     };

src/HttpBackend.js

@@ -3,7 +3,7 @@ module.exports = class HttpBackend {
     this._url = url;
   }
   loadSuperblock() {
-    return fetch(this._url + '/.superblock.txt').then(res => res.text())
+    return fetch(this._url + '/.superblock.txt').then(res => res.ok ? res.text() : null)
   }
   async readFile(filepath) {
     const res = await fetch(this._url + filepath)
@@ -13,4 +13,12 @@ module.exports = class HttpBackend {
       throw new Error('ENOENT')
     }
   }
+  async sizeFile(filepath) {
+    const res = await fetch(this._url + filepath, { method: 'HEAD' })
+    if (res.status === 200) {
+      return res.headers.get('content-length')
+    } else {
+      throw new Error('ENOENT')
+    }
+  }
 }

src/PromisifiedFS.js

@@ -33,7 +33,7 @@ function cleanParams2(oldFilepath, newFilepath) {
 }
 
 module.exports = class PromisifiedFS {
-  constructor(name, { wipe, url } = {}) {
+  constructor(name, { wipe, url, urlauto } = {}) {
     this._name = name
     this._idb = new IdbBackend(name);
     this._mutex = new Mutex(name);
@@ -45,6 +45,7 @@ module.exports = class PromisifiedFS {
     }, 500);
     if (url) {
       this._http = new HttpBackend(url)
+      this._urlauto = !!urlauto
     }
     this._operations = new Set()
 
@@ -59,6 +60,7 @@ module.exports = class PromisifiedFS {
     this.lstat = this._wrap(this.lstat, false)
     this.readlink = this._wrap(this.readlink, false)
     this.symlink = this._wrap(this.symlink, true)
+    this.backFile = this._wrap(this.backFile, true)
 
     this._deactivationPromise = null
     this._deactivationTimeout = null
@@ -143,18 +145,41 @@ module.exports = class PromisifiedFS {
       await this._idb.saveSuperblock(this._cache._root);
     }
   }
+  async _writeStat(filepath, size, opts) {
+    let dirparts = path.split(path.dirname(filepath))
+    let dir = dirparts.shift()
+    for (let dirpart of dirparts) {
+      dir = path.join(dir, dirpart)
+      try {
+        this._cache.mkdir(dir, { mode: 0o777 })
+      } catch (e) {}
+    }
+    return this._cache.writeStat(filepath, size, opts)
+  }
   async readFile(filepath, opts) {
     ;[filepath, opts] = cleanParams(filepath, opts);
     const { encoding } = opts;
     if (encoding && encoding !== 'utf8') throw new Error('Only "utf8" encoding is supported in readFile');
-    const stat = this._cache.stat(filepath);
-    let data = await this._idb.readFile(stat.ino)
+    let data = null, stat = null
+    try {
+      stat = this._cache.stat(filepath);
+      data = await this._idb.readFile(stat.ino)
+    } catch (e) {
+      if (!this._urlauto) throw e
+    }
     if (!data && this._http) {
       data = await this._http.readFile(filepath)
     }
-    if (data && encoding === "utf8") {
-      data = decode(data);
+    if (data) {
+      if (!stat || stat.size != data.byteLength) {
+        stat = await this._writeStat(filepath, data.byteLength, { mode: stat ? stat.mode : 0o666 })
+        this.saveSuperblock() // debounced
+      }
+      if (encoding === "utf8") {
+        data = decode(data);
+      }
     }
+    if (!stat) throw new ENOENT(filepath)
     return data;
   }
   async writeFile(filepath, data, opts) {
@@ -166,7 +191,7 @@ module.exports = class PromisifiedFS {
       }
       data = encode(data);
     }
-    const stat = this._cache.writeFile(filepath, data, { mode });
+    const stat = await this._cache.writeStat(filepath, data.byteLength, { mode });
     await this._idb.writeFile(stat.ino, data)
     return null
   }
@@ -222,4 +247,10 @@ module.exports = class PromisifiedFS {
     this._cache.symlink(target, filepath);
     return null;
   }
+  async backFile(filepath, opts) {
+    ;[filepath, opts] = cleanParams(filepath, opts);
+    let size = await this._http.sizeFile(filepath)
+    await this._writeStat(filepath, size, opts)
+    return null
+  }
 }

src/__tests__/CacheFS.spec.js

@@ -21,7 +21,7 @@ describe("CacheFS module", () => {
     const fs = new CacheFS();
     fs.activate()
     expect(fs.autoinc()).toEqual(1)
-    fs.writeFile('/foo', 'bar', {})
+    fs.writeStat('/foo', 3, {})
     expect(fs.autoinc()).toEqual(2)
     fs.mkdir('/bar', {})
     expect(fs.autoinc()).toEqual(3)
@@ -33,7 +33,7 @@ describe("CacheFS module", () => {
     expect(fs.autoinc()).toEqual(3)
     fs.mkdir('/bar/bar', {})
     expect(fs.autoinc()).toEqual(4)
-    fs.writeFile('/bar/bar/boo', 'bar', {})
+    fs.writeStat('/bar/bar/boo', 3, {})
     expect(fs.autoinc()).toEqual(5)
     fs.unlink('/bar/bar/boo')
     expect(fs.autoinc()).toEqual(4)

src/__tests__/__fixtures__/test-folder/not-in-superblock.txt

@@ -0,0 +1 @@
+Hello from "not-in-superblock"

src/__tests__/fallback.spec.js

@@ -38,6 +38,12 @@ describe("http fallback", () => {
         done();
       });
     });
+    it("read file not in superblock throws", done => {
+      fs.readFile("/not-in-superblock.txt", (err, data) => {
+        expect(err).not.toBe(null);
+        done();
+      });
+    });
     it("read file /a.txt", done => {
       fs.readFile("/a.txt", 'utf8', (err, data) => {
         expect(err).toBe(null);
@@ -82,4 +88,25 @@ describe("http fallback", () => {
       });
     });
   });
+  describe("backFile", () => {
+    it("backing a nonexistant file throws", done => {
+      fs.backFile("/backFile/non-existant.txt", (err, data) => {
+        expect(err).not.toBe(null);
+        done();
+      });
+    });
+    it("backing a file makes it readable", done => {
+      fs.backFile("/not-in-superblock.txt", (err, data) => {
+        expect(err).toBe(null)
+        fs.readFile("/not-in-superblock.txt", 'utf8', (err, data) => {
+          expect(err).toBe(null);
+          expect(data).toEqual('Hello from "not-in-superblock"');
+          fs.unlink("/not-in-superblock.txt", (err, data) => {
+            expect(err).toBe(null);
+            done();
+          });
+	});
+      });
+    });
+  });
 });

src/index.js

@@ -26,6 +26,7 @@ module.exports = class FS {
     this.lstat = this.lstat.bind(this)
     this.readlink = this.readlink.bind(this)
     this.symlink = this.symlink.bind(this)
+    this.backFile = this.backFile.bind(this)
   }
   readFile(filepath, opts, cb) {
     const [resolve, reject] = wrapCallback(opts, cb);
@@ -71,4 +72,8 @@ module.exports = class FS {
     const [resolve, reject] = wrapCallback(cb);
     this.promises.symlink(target, filepath).then(resolve).catch(reject);
   }
+  backFile(filepath, opts, cb) {
+    const [resolve, reject] = wrapCallback(opts, cb);
+    this.promises.backFile(filepath, opts).then(resolve).catch(reject);
+  }
 }