From 906af59b67794af70bbeb4a16644af99d8f671c0 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Tue, 12 Apr 2022 13:34:29 -0700 Subject: [PATCH 01/14] [WasmFS] Initial OPFS/AccessHandles backend Add a backend that stores its underlying files in the Origin Private File System (OPFS) and reads and writes the files synchronously using `FileSystemSyncAccessHandle`. This initial implementation works correctly as long as there are no errors; better error handling and more robust edge case testing will come in a future PR. --- src/library_wasmfs_opfs.js | 239 +++++++++++++++++ src/modules.js | 1 + system/include/emscripten/wasmfs.h | 2 + system/lib/wasmfs/backends/opfs_backend.cpp | 274 ++++++++++++++++++++ tests/test_browser.py | 5 + tests/wasmfs/wasmfs_opfs.c | 110 ++++++++ tools/deps_info.py | 4 + tools/system_libs.py | 1 + 8 files changed, 636 insertions(+) create mode 100644 src/library_wasmfs_opfs.js create mode 100644 system/lib/wasmfs/backends/opfs_backend.cpp create mode 100644 tests/wasmfs/wasmfs_opfs.c diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js new file mode 100644 index 0000000000000..f44bcf0ca05a1 --- /dev/null +++ b/src/library_wasmfs_opfs.js @@ -0,0 +1,239 @@ +/** + * @license + * Copyright 2022 The Emscripten Authors + * SPDX-License-Identifier: MIT + */ + +mergeInto(LibraryManager.library, { + $wasmfsOPFSDirectories: { + allocated: [], + free: [], + get: function(i) { + return this.allocated[i]; + } + }, + + $wasmfsOPFSFiles: { + allocated: [], + free: [], + get: function(i) { + return this.allocated[i]; + } + }, + + $wasmfsOPFSAccesses: { + allocated: [], + free: [], + get: function(i) { + return this.allocated[i]; + } + }, + + $wasmfsOPFSAllocate: function(ids, handle) { + let id = ids.allocated.length; + if (ids.free.length > 0) { + id = ids.free.pop(); + } + assert(ids.allocated[id] === undefined); + ids.allocated[id] = handle; + return id; + }, + + $wasmfsOPFSFree: function(ids, id) { + delete ids.allocated[id]; + ids.free.push(id); + }, + + _wasmfs_opfs_init_root_directory__deps: ['$wasmfsOPFSDirectories'], + _wasmfs_opfs_init_root_directory: async function(ctx) { + if (wasmfsOPFSDirectories.allocated.length == 0) { + // Directory 0 is reserved as the root + let root = await navigator.storage.getDirectory(); + wasmfsOPFSDirectories.allocated.push(root); + } + _emscripten_proxy_finish(ctx); + }, + + // Return the file ID for the file with `name` under `parent`, creating it if + // it doesn't exist and `create` or otherwise return one of the following + // error codes: + // + // -1: file does not exist. + // -2: file exists but it is actually a directory. + // -3: file exists but an access handle cannot be created for it. + $wasmfsOPFSGetOrCreateFile__deps: ['$wasmfsOPFSAllocate', + '$wasmfsOPFSDirectories', + '$wasmfsOPFSFiles'], + $wasmfsOPFSGetOrCreateFile: async function(parent, name, create) { + let parent_handle = wasmfsOPFSDirectories.get(parent); + assert(parent_handle !== undefined); + let file_handle; + try { + file_handle = await parent_handle.getFileHandle(name, {create: create}); + } catch (err) { + if (err.name === "NotFoundError") { + return -1; + } + if (err.name === "TypeMismatchError") { + return -2; + } + abort("Unknown exception " + err.name); + } + return wasmfsOPFSAllocate(wasmfsOPFSFiles, file_handle); + }, + + // Return the file ID for the directory with `name` under `parent`, creating + // it if it doesn't exist and `create` or otherwise one of the following error + // codes: + // + // -1: directory does not exist. + // -2: directory exists but is actually a data file. + $wasmfsOPFSGetOrCreateDir__deps: ['$wasmfsOPFSAllocate', + '$wasmfsOPFSDirectories'], + $wasmfsOPFSGetOrCreateDir: async function(parent, name, create) { + let parent_handle = wasmfsOPFSDirectories.get(parent); + assert(parent_handle !== undefined); + let child_handle; + try { + child_handle = + await parent_handle.getDirectoryHandle(name, {create: create}); + } catch (err) { + if (err.name === "NotFoundError") { + return -1; + } + if (err.name === "TypeMismatchError") { + return -2; + } + abort("Unknown exception " + err.name); + } + return wasmfsOPFSAllocate(wasmfsOPFSDirectories, child_handle); + }, + + _wasmfs_opfs_get_child__deps: ['$wasmfsOPFSGetOrCreateFile', + '$wasmfsOPFSGetOrCreateDir'], + _wasmfs_opfs_get_child: + async function(ctx, parent, name_p, child_type_p, child_id_p) { + let name = UTF8ToString(name_p); + let child_type = 1; + let child_id = await wasmfsOPFSGetOrCreateFile(parent, name, false); + if (child_id == -2) { + child_type = 2; + child_id = await wasmfsOPFSGetOrCreateDir(parent, name, false); + } + {{{ makeSetValue('child_type_p', 0, 'child_type', 'i32') }}}; + {{{ makeSetValue('child_id_p', 0, 'child_id', 'i32') }}}; + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_get_entries__deps: [], + _wasmfs_opfs_get_entries: async function(ctx, dir_id, entries) { + let dir_handle = wasmfsOPFSDirectories.get(dir_id); + for await (const [name, child] of dir_handle.entries()) { + withStackSave(() => { + let name_p = allocateUTF8OnStack(name); + // TODO: Figure out how to use `cDefine` here + let type = child.kind == "file" ? 1 : 2; + __wasmfs_opfs_record_entry(entries, name_p, type) + }); + } + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_insert_file__deps: ['$wasmfsOPFSGetOrCreateFile'], + _wasmfs_opfs_insert_file: async function(ctx, parent, name_p, child_id_p) { + let name = UTF8ToString(name_p); + let child_id = await wasmfsOPFSGetOrCreateFile(parent, name, true); + {{{ makeSetValue('child_id_p', 0, 'child_id', 'i32') }}}; + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_insert_directory__deps: ['$wasmfsOPFSGetOrCreateDir'], + _wasmfs_opfs_insert_directory: async function(ctx, parent, name_p, child_id_p) { + let name = UTF8ToString(name_p); + let child_id = await wasmfsOPFSGetOrCreateDir(parent, name, true); + {{{ makeSetValue('child_id_p', 0, 'child_id', 'i32') }}}; + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_remove_child__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSDirectories'], + _wasmfs_opfs_remove_child: async function(ctx, dir_id, name_p) { + let name = UTF8ToString(name_p); + let dir_handle = wasmfsOPFSDirectories.get(dir_id); + await dir_handle.removeEntry(name); + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_free_file__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSFiles'], + _wasmfs_opfs_free_file: function(file_id) { + wasmfsOPFSFree(wasmfsOPFSFiles, file_id); + }, + + _wasmfs_opfs_free_directory__deps: ['$wasmfsOPFSFree', + '$wasmfsOPFSDirectories'], + _wasmfs_opfs_free_directory: function(dir_id) { + wasmfsOPFSFree(wasmfsOPFSDirectories, dir_id); + }, + + _wasmfs_opfs_open__deps: ['$wasmfsOPFSAllocate', + '$wasmfsOPFSFiles', + '$wasmfsOPFSAccesses'], + _wasmfs_opfs_open: async function(ctx, file_id, access_id_p) { + let file_handle = wasmfsOPFSFiles.get(file_id); + let access_id; + try { + let access_handle = await file_handle.createSyncAccessHandle(); + access_id = wasmfsOPFSAllocate(wasmfsOPFSAccesses, access_handle); + } catch (err) { + if (err.name === "InvalidStateError") { + access_id = -1; + } + abort("Unknown error opening file"); + } + {{{ makeSetValue('access_id_p', 0, 'access_id', 'i32') }}}; + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_close__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSAccesses'], + _wasmfs_opfs_close: async function(ctx, access_id) { + let access_handle = wasmfsOPFSAccesses.get(access_id); + await access_handle.close(); + wasmfsOPFSFree(wasmfsOPFSAccesses, access_id); + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_read__deps: ['$wasmfsOPFSAccesses'], + _wasmfs_opfs_read: function(access_id, buf_p, len, pos) { + let access_handle = wasmfsOPFSAccesses.get(access_id); + let data = HEAPU8.subarray(buf_p, buf_p + len); + return access_handle.read(data, {at: pos}); + }, + + _wasmfs_opfs_write__deps: ['$wasmfsOPFSAccesses'], + _wasmfs_opfs_write: function(access_id, buf_p, len, pos, nwritten_p) { + let access_handle = wasmfsOPFSAccesses.get(access_id); + let data = HEAPU8.subarray(buf_p, buf_p + len); + return access_handle.write(data, {at: pos}); + }, + + _wasmfs_opfs_get_size__deps: ['$wasmfsOPFSAccesses'], + _wasmfs_opfs_get_size: async function(ctx, access_id, size_p) { + let access_handle = wasmfsOPFSAccesses.get(access_id); + let size = await access_handle.getSize(); + {{{ makeSetValue('size_p', 0, 'size', 'i32') }}}; + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_set_size__deps: ['$wasmfsOPFSAccesses'], + _wasmfs_opfs_set_size: async function(ctx, access_id, size) { + let access_handle = wasmfsOPFSAccesses.get(access_id); + await access_handle.truncate(size); + _emscripten_proxy_finish(ctx); + }, + + _wasmfs_opfs_flush__deps: ['$wasmfsOPFSAccesses'], + _wasmfs_opfs_flush: async function(ctx, access_id) { + let access_handle = wasmfsOPFSAccesses.get(access_id); + await access_handle.flush(); + _emscripten_proxy_finish(ctx); + } +}); diff --git a/src/modules.js b/src/modules.js index 1a0bdab8bb8f4..c63478ecc9a66 100644 --- a/src/modules.js +++ b/src/modules.js @@ -93,6 +93,7 @@ global.LibraryManager = { libraries.push('library_wasmfs_js_file.js'); libraries.push('library_wasmfs_fetch.js'); libraries.push('library_wasmfs_node.js'); + libraries.push('library_wasmfs_opfs.js'); } // Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js) diff --git a/system/include/emscripten/wasmfs.h b/system/include/emscripten/wasmfs.h index 264fcdf22abab..b3401d5027b66 100644 --- a/system/include/emscripten/wasmfs.h +++ b/system/include/emscripten/wasmfs.h @@ -48,6 +48,8 @@ backend_t wasmfs_create_fetch_backend(const char* base_url); backend_t wasmfs_create_node_backend(const char* root); +backend_t wasmfs_create_opfs_backend(void); + #ifdef __cplusplus } #endif diff --git a/system/lib/wasmfs/backends/opfs_backend.cpp b/system/lib/wasmfs/backends/opfs_backend.cpp new file mode 100644 index 0000000000000..ef1ba4df65715 --- /dev/null +++ b/system/lib/wasmfs/backends/opfs_backend.cpp @@ -0,0 +1,274 @@ +// Copyright 2022 The Emscripten Authors. All rights reserved. +// Emscripten is available under two separate licenses, the MIT license and the +// University of Illinois/NCSA Open Source License. Both these licenses can be +// found in the LICENSE file. + +#include "backend.h" +#include "file.h" +#include "support.h" +#include "thread_utils.h" +#include "wasmfs.h" +#include + +namespace wasmfs { + +using ProxyWorker = emscripten::ProxyWorker; + +extern "C" { + +// Ensure that the root OPFS directory is initialized with ID 0. +void _wasmfs_opfs_init_root_directory(em_proxying_ctx* ctx); + +// Look up the child under `parent` with `name`. Write 1 to `child_type` if it's +// a regular file or 2 if it's a directory. Write the child's file or directory +// ID to `child_id`, or -1 if the child does not exist, or -2 if the child +// exists but cannot be opened. +void _wasmfs_opfs_get_child(em_proxying_ctx* ctx, + int parent, + const char* name, + int* child_type, + int* child_id); + +// Create a file under `parent` with `name` and store its ID in `child_id`. +void _wasmfs_opfs_insert_file(em_proxying_ctx* ctx, + int parent, + const char* name, + int* child_id); + +// Create a directory under `parent` with `name` and store its ID in `child_id`. +void _wasmfs_opfs_insert_directory(em_proxying_ctx* ctx, + int parent, + const char* name, + int* child_id); + +void _wasmfs_opfs_remove_child(em_proxying_ctx* ctx, + int dir_id, + const char* name); + +void _wasmfs_opfs_get_entries(em_proxying_ctx* ctx, + int dirID, + std::vector* entries); + +void _wasmfs_opfs_open(em_proxying_ctx* ctx, int file_id, int* access_id); + +void _wasmfs_opfs_close(em_proxying_ctx* ctx, int access_id); + +void _wasmfs_opfs_free_file(int file_id); + +void _wasmfs_opfs_free_directory(int dir_id); + +// Synchronous read. Return the number of bytes read. +int _wasmfs_opfs_read(int access_id, uint8_t* buf, uint32_t len, uint32_t pos); + +// Synchronous write. Return the number of bytes written. +int _wasmfs_opfs_write(int access_id, + const uint8_t* buf, + uint32_t len, + uint32_t pos); + +void _wasmfs_opfs_get_size(em_proxying_ctx* ctx, int access_id, uint32_t* size); + +void _wasmfs_opfs_set_size(em_proxying_ctx* ctx, int access_id, uint32_t size); + +void _wasmfs_opfs_flush(em_proxying_ctx* ctx, int access_id); + +} // extern "C" + +class OPFSFile : public DataFile { +public: + ProxyWorker& proxy; + + // The IDs of the corresponding file handle and, if the file is open, the + // corresponding access handle. + int fileID; + int accessID = -1; + + // The number of times this file has been opened. We only close its + // AccessHandle when this falls to zero. + size_t openCount = 0; + + OPFSFile(mode_t mode, backend_t backend, int fileID, ProxyWorker& proxy) + : DataFile(mode, backend), fileID(fileID), proxy(proxy) {} + + ~OPFSFile() override { + assert(openCount == 0); + assert(accessID == -1); + proxy([&]() { _wasmfs_opfs_free_file(fileID); }); + } + +private: + size_t getSize() override { + uint32_t size; + proxy([&](auto ctx) { _wasmfs_opfs_get_size(ctx.ctx, accessID, &size); }); + return size_t(size); + } + + void setSize(size_t size) override { + proxy([&](auto ctx) { _wasmfs_opfs_set_size(ctx.ctx, accessID, size); }); + } + + void open(oflags_t flags) override { + if (openCount == 0) { + proxy([&](auto ctx) { _wasmfs_opfs_open(ctx.ctx, fileID, &accessID); }); + ++openCount; + } + // TODO: proper error handling. + assert(accessID >= 0); + } + + void close() override { + assert(openCount >= 1); + if (--openCount == 0) { + proxy([&](auto ctx) { _wasmfs_opfs_close(ctx.ctx, accessID); }); + accessID = -1; + } + } + + __wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override { + uint32_t nread; + proxy([&]() { nread = _wasmfs_opfs_read(accessID, buf, len, offset); }); + // TODO: Add a way to report the actual bytes read. We currently assume the + // available bytes can't change under us. + return __WASI_ERRNO_SUCCESS; + } + + __wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override { + uint32_t nwritten; + proxy([&]() { nwritten = _wasmfs_opfs_write(accessID, buf, len, offset); }); + // TODO: Add a way to report the actual bytes written. We currently assume + // the write cannot be short. + return __WASI_ERRNO_SUCCESS; + } + + void flush() override { + proxy([&](auto ctx) { _wasmfs_opfs_flush(ctx.ctx, accessID); }); + } +}; + +class OPFSDirectory : public Directory { +public: + ProxyWorker& proxy; + + // The ID of this directory in the JS library. + int dirID = 0; + + OPFSDirectory(mode_t mode, backend_t backend, int dirID, ProxyWorker& proxy) + : Directory(mode, backend), dirID(dirID), proxy(proxy) {} + + ~OPFSDirectory() override { + // Never free the root directory ID. + if (dirID != 0) { + proxy([&]() { _wasmfs_opfs_free_directory(dirID); }); + } + } + +private: + std::shared_ptr getChild(const std::string& name) override { + int childType = 0, childID = 0; + proxy([&](auto ctx) { + _wasmfs_opfs_get_child( + ctx.ctx, dirID, name.c_str(), &childType, &childID); + }); + if (childID < 0) { + // TODO: More fine-grained error reporting. + return NULL; + } + if (childType == 1) { + return std::make_shared(0777, getBackend(), childID, proxy); + } else if (childType == 2) { + return std::make_shared( + 0777, getBackend(), childID, proxy); + } else { + WASMFS_UNREACHABLE("Unexpected child type"); + } + } + + std::shared_ptr insertDataFile(const std::string& name, + mode_t mode) override { + int childID = 0; + proxy([&](auto ctx) { + _wasmfs_opfs_insert_file(ctx.ctx, dirID, name.c_str(), &childID); + }); + // TODO: Handle errors gracefully. + assert(childID >= 0); + return std::make_shared(mode, getBackend(), childID, proxy); + } + + std::shared_ptr insertDirectory(const std::string& name, + mode_t mode) override { + int childID = 0; + proxy([&](auto ctx) { + _wasmfs_opfs_insert_directory(ctx.ctx, dirID, name.c_str(), &childID); + }); + // TODO: Handle errors gracefully. + assert(childID >= 0); + return std::make_shared(mode, getBackend(), childID, proxy); + } + + std::shared_ptr insertSymlink(const std::string& name, + const std::string& target) override { + // Symlinks not supported. + return nullptr; + } + + bool insertMove(const std::string& name, + std::shared_ptr file) override { + // OPFS does not support rename. + return false; + } + + bool removeChild( + const std::string& name /*, std::shared_ptr child*/) override { + proxy([&](auto ctx) { + _wasmfs_opfs_remove_child(ctx.ctx, dirID, name.c_str()); + }); + // if (child) { + // // TODO: call `close` on the handle then delete it and free the ID. + // } + return true; + } + + size_t getNumEntries() override { return getEntries().size(); } + + std::vector getEntries() override { + std::vector entries; + proxy( + [&](auto ctx) { _wasmfs_opfs_get_entries(ctx.ctx, dirID, &entries); }); + return entries; + } +}; + +class OPFSBackend : public Backend { +public: + ProxyWorker proxy; + + std::shared_ptr createFile(mode_t mode) override { + // No way to support a raw file without a parent directory. + return nullptr; + } + + std::shared_ptr createDirectory(mode_t mode) override { + proxy([](auto ctx) { _wasmfs_opfs_init_root_directory(ctx.ctx); }); + return std::make_shared(mode, this, 0, proxy); + } + + std::shared_ptr createSymlink(std::string target) override { + // Symlinks not supported. + return nullptr; + } +}; + +extern "C" { + +backend_t wasmfs_create_opfs_backend() { + return wasmFS.addBackend(std::make_unique()); +} + +void EMSCRIPTEN_KEEPALIVE _wasmfs_opfs_record_entry( + std::vector* entries, const char* name, int type) { + entries->push_back({name, File::FileKind(type), 0}); +} + +} // extern "C" + +} // namespace wasmfs diff --git a/tests/test_browser.py b/tests/test_browser.py index 10364f575e6e3..dde30d9f0d9e6 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -5294,6 +5294,11 @@ def test_wasmfs_fetch_backend(self, args): self.btest_exit(test_file('wasmfs/wasmfs_fetch.c'), args=['-sWASMFS', '-sUSE_PTHREADS'] + args) + @requires_threads + def test_wasmfs_opfs(self): + self.btest_exit(test_file('wasmfs/wasmfs_opfs.c'), + args=['-sWASMFS', '-pthread', '-sPROXY_TO_PTHREAD']) + @no_firefox('no 4GB support yet') def test_zzz_zzz_emmalloc_memgrowth(self, *args): self.btest(test_file('browser/emmalloc_memgrowth.cpp'), expected='0', args=['-sMALLOC=emmalloc', '-sALLOW_MEMORY_GROWTH=1', '-sABORTING_MALLOC=0', '-sASSERTIONS=2', '-sMINIMAL_RUNTIME=1', '-sMAXIMUM_MEMORY=4GB']) diff --git a/tests/wasmfs/wasmfs_opfs.c b/tests/wasmfs/wasmfs_opfs.c new file mode 100644 index 0000000000000..665b29242733e --- /dev/null +++ b/tests/wasmfs/wasmfs_opfs.c @@ -0,0 +1,110 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +int main(int argc, char* argv[]) { + emscripten_console_log("starting main"); + + backend_t opfs = wasmfs_create_opfs_backend(); + emscripten_console_log("created OPFS backend"); + + int err = wasmfs_create_directory("/opfs", 0777, opfs); + assert(err == 0); + emscripten_console_log("mounted OPFS root directory"); + + err = mkdir("/opfs/working", 0777); + assert(err == 0); + emscripten_console_log("created OPFS directory"); + + int fd = open("/opfs/working/foo.txt", O_RDWR | O_CREAT | O_EXCL, 0777); + assert(fd > 0); + emscripten_console_log("created OPFS file"); + + const char* msg = "Hello, OPFS!"; + int nwritten = write(fd, msg, strlen(msg)); + assert(nwritten == strlen(msg)); + emscripten_console_logf("wrote message: %s (%d)", msg, nwritten); + + int off = lseek(fd, 0, SEEK_SET); + assert(off == 0); + emscripten_console_log("seeked"); + + char buf[100] = {}; + int nread = read(fd, buf, 100); + assert(nread == strlen(msg)); + assert(strcmp(buf, msg) == 0); + emscripten_console_logf("read message: %s (%d)", buf, nread); + + fdatasync(fd); + emscripten_console_log("flushed"); + + struct stat stat; + err = fstat(fd, &stat); + assert(err == 0); + assert(stat.st_size == strlen(msg)); + emscripten_console_log("statted"); + + err = ftruncate(fd, 100); + assert(err == 0); + err = fstat(fd, &stat); + assert(err == 0); + assert(stat.st_size == 100); + emscripten_console_log("truncated to 100"); + + err = ftruncate(fd, 0); + assert(err == 0); + err = fstat(fd, &stat); + assert(err == 0); + assert(stat.st_size == 0); + emscripten_console_log("truncated to 0"); + + struct dirent** entries; + int nentries = scandir("/opfs/working", &entries, NULL, alphasort); + assert(nentries == 3); + assert(strcmp(entries[0]->d_name, ".") == 0); + assert(strcmp(entries[1]->d_name, "..") == 0); + assert(strcmp(entries[2]->d_name, "foo.txt") == 0); + assert(entries[2]->d_type == DT_REG); + for (int i = 0; i < nentries; i++) { + free(entries[i]); + } + free(entries); + emscripten_console_log("read /opfs/working entries"); + + nentries = scandir("/opfs", &entries, NULL, alphasort); + assert(nentries == 3); + assert(strcmp(entries[2]->d_name, "working") == 0); + assert(entries[2]->d_type == DT_DIR); + for (int i = 0; i < nentries; i++) { + free(entries[i]); + } + free(entries); + emscripten_console_log("read /opfs entries"); + + err = close(fd); + assert(err == 0); + emscripten_console_log("closed file"); + + err = unlink("/opfs/working/foo.txt"); + assert(err == 0); + err = access("/opfs/working/foo.txt", F_OK); + assert(err == -1); + emscripten_console_log("removed OPFS file"); + + err = rmdir("/opfs/working"); + assert(err == 0); + err = access("/opfs/working", F_OK); + assert(err == -1); + emscripten_console_log("removed OPFS directory"); + + emscripten_console_log("done"); +} diff --git a/tools/deps_info.py b/tools/deps_info.py index c931b0ecf6b9d..5877d3de47eda 100644 --- a/tools/deps_info.py +++ b/tools/deps_info.py @@ -218,4 +218,8 @@ def get_deps_info(): _deps_info['emscripten_set_offscreencanvas_size_on_target_thread_js'] = ['malloc'] if settings.USE_PTHREADS: _deps_info['emscripten_set_canvas_element_size_calling_thread'] = ['emscripten_dispatch_to_thread_'] + + if settings.WASMFS: + _deps_info['wasmfs_create_opfs_backend'] = ['emscripten_proxy_finish'] + return _deps_info diff --git a/tools/system_libs.py b/tools/system_libs.py index 74df1fe79e62d..0eadc8ad41430 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1493,6 +1493,7 @@ def get_files(self): 'js_file_backend.cpp', 'memory_backend.cpp', 'node_backend.cpp', + 'opfs_backend.cpp', 'proxied_file_backend.cpp']) return backends + files_in_path( path='system/lib/wasmfs', From 371a5792d8da1ff06b26581400eb99fd1de303ab Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 25 Apr 2022 17:11:01 -0700 Subject: [PATCH 02/14] Update comments --- system/lib/wasmfs/backends/opfs_backend.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/system/lib/wasmfs/backends/opfs_backend.cpp b/system/lib/wasmfs/backends/opfs_backend.cpp index ef1ba4df65715..1f694c7de8f6f 100644 --- a/system/lib/wasmfs/backends/opfs_backend.cpp +++ b/system/lib/wasmfs/backends/opfs_backend.cpp @@ -213,18 +213,14 @@ class OPFSDirectory : public Directory { bool insertMove(const std::string& name, std::shared_ptr file) override { - // OPFS does not support rename. + // OPFS does not support rename (or does it???) return false; } - bool removeChild( - const std::string& name /*, std::shared_ptr child*/) override { + bool removeChild(const std::string& name) override { proxy([&](auto ctx) { _wasmfs_opfs_remove_child(ctx.ctx, dirID, name.c_str()); }); - // if (child) { - // // TODO: call `close` on the handle then delete it and free the ID. - // } return true; } From df035c448e663020c0e4edd7004e5fda2595ec5f Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 25 Apr 2022 17:35:59 -0700 Subject: [PATCH 03/14] implement rename --- src/library_wasmfs_opfs.js | 9 +++++++++ system/lib/wasmfs/backends/opfs_backend.cpp | 13 +++++++++++-- tests/wasmfs/wasmfs_opfs.c | 11 ++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index f44bcf0ca05a1..d87e29ff85afb 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -155,6 +155,15 @@ mergeInto(LibraryManager.library, { _emscripten_proxy_finish(ctx); }, + _wasmfs_opfs_move__deps: ['$wasmfsOPFSFiles', '$wasmfsOPFSDirectories'], + _wasmfs_opfs_move: async function(ctx, file_id, new_dir_id, name_p) { + let name = UTF8ToString(name_p); + let file_handle = wasmfsOPFSFiles.get(file_id); + let new_dir_handle = wasmfsOPFSDirectories.get(new_dir_id); + await file_handle.move(new_dir_handle, name); + _emscripten_proxy_finish(ctx); + }, + _wasmfs_opfs_remove_child__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSDirectories'], _wasmfs_opfs_remove_child: async function(ctx, dir_id, name_p) { let name = UTF8ToString(name_p); diff --git a/system/lib/wasmfs/backends/opfs_backend.cpp b/system/lib/wasmfs/backends/opfs_backend.cpp index 1f694c7de8f6f..9716c2cef44f8 100644 --- a/system/lib/wasmfs/backends/opfs_backend.cpp +++ b/system/lib/wasmfs/backends/opfs_backend.cpp @@ -41,6 +41,11 @@ void _wasmfs_opfs_insert_directory(em_proxying_ctx* ctx, const char* name, int* child_id); +void _wasmfs_opfs_move(em_proxying_ctx* ctx, + int file_id, + int new_dir_id, + const char* name); + void _wasmfs_opfs_remove_child(em_proxying_ctx* ctx, int dir_id, const char* name); @@ -213,8 +218,12 @@ class OPFSDirectory : public Directory { bool insertMove(const std::string& name, std::shared_ptr file) override { - // OPFS does not support rename (or does it???) - return false; + auto old_file = std::static_pointer_cast(file); + proxy([&](auto ctx) { + _wasmfs_opfs_move(ctx.ctx, old_file->fileID, dirID, name.c_str()); + }); + // TODO: Handle errors. + return true; } bool removeChild(const std::string& name) override { diff --git a/tests/wasmfs/wasmfs_opfs.c b/tests/wasmfs/wasmfs_opfs.c index 665b29242733e..b2c08031146fe 100644 --- a/tests/wasmfs/wasmfs_opfs.c +++ b/tests/wasmfs/wasmfs_opfs.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -94,10 +95,18 @@ int main(int argc, char* argv[]) { assert(err == 0); emscripten_console_log("closed file"); - err = unlink("/opfs/working/foo.txt"); + err = rename("/opfs/working/foo.txt", "/opfs/foo.txt"); assert(err == 0); err = access("/opfs/working/foo.txt", F_OK); assert(err == -1); + err = access("/opfs/foo.txt", F_OK); + assert(err == 0); + emscripten_console_log("moved file"); + + err = unlink("/opfs/foo.txt"); + assert(err == 0); + err = access("/opfs/foo.txt", F_OK); + assert(err == -1); emscripten_console_log("removed OPFS file"); err = rmdir("/opfs/working"); From 837b249f0bd21abcd757ed59d36efc21458110c1 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Tue, 26 Apr 2022 15:46:44 -0700 Subject: [PATCH 04/14] address comments --- src/library_wasmfs_opfs.js | 178 ++++++++++---------- system/lib/wasmfs/backends/opfs_backend.cpp | 14 +- 2 files changed, 99 insertions(+), 93 deletions(-) diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index d87e29ff85afb..f39a8fef70314 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -30,12 +30,14 @@ mergeInto(LibraryManager.library, { }, $wasmfsOPFSAllocate: function(ids, handle) { - let id = ids.allocated.length; + let id; if (ids.free.length > 0) { id = ids.free.pop(); + ids.allocated[id] = handle; + } else { + id = ids.allocated.length; + ids.allocated.push(handle); } - assert(ids.allocated[id] === undefined); - ids.allocated[id] = handle; return id; }, @@ -65,21 +67,21 @@ mergeInto(LibraryManager.library, { '$wasmfsOPFSDirectories', '$wasmfsOPFSFiles'], $wasmfsOPFSGetOrCreateFile: async function(parent, name, create) { - let parent_handle = wasmfsOPFSDirectories.get(parent); - assert(parent_handle !== undefined); - let file_handle; + let parentHandle = wasmfsOPFSDirectories.get(parent); + assert(parentHandle !== undefined); + let fileHandle; try { - file_handle = await parent_handle.getFileHandle(name, {create: create}); - } catch (err) { - if (err.name === "NotFoundError") { + fileHandle = await parentHandle.getFileHandle(name, {create: create}); + } catch (e) { + if (e.name === "NotFoundError") { return -1; } - if (err.name === "TypeMismatchError") { + if (e.name === "TypeMismatchError") { return -2; } - abort("Unknown exception " + err.name); + throw e; } - return wasmfsOPFSAllocate(wasmfsOPFSFiles, file_handle); + return wasmfsOPFSAllocate(wasmfsOPFSFiles, fileHandle); }, // Return the file ID for the directory with `name` under `parent`, creating @@ -91,158 +93,158 @@ mergeInto(LibraryManager.library, { $wasmfsOPFSGetOrCreateDir__deps: ['$wasmfsOPFSAllocate', '$wasmfsOPFSDirectories'], $wasmfsOPFSGetOrCreateDir: async function(parent, name, create) { - let parent_handle = wasmfsOPFSDirectories.get(parent); - assert(parent_handle !== undefined); - let child_handle; + let parentHandle = wasmfsOPFSDirectories.get(parent); + assert(parentHandle !== undefined); + let childHandle; try { - child_handle = - await parent_handle.getDirectoryHandle(name, {create: create}); - } catch (err) { - if (err.name === "NotFoundError") { + childHandle = + await parentHandle.getDirectoryHandle(name, {create: create}); + } catch (e) { + if (e.name === "NotFoundError") { return -1; } - if (err.name === "TypeMismatchError") { + if (e.name === "TypeMismatchError") { return -2; } - abort("Unknown exception " + err.name); + abort("Unknown exception " + e.name); } - return wasmfsOPFSAllocate(wasmfsOPFSDirectories, child_handle); + return wasmfsOPFSAllocate(wasmfsOPFSDirectories, childHandle); }, _wasmfs_opfs_get_child__deps: ['$wasmfsOPFSGetOrCreateFile', '$wasmfsOPFSGetOrCreateDir'], _wasmfs_opfs_get_child: - async function(ctx, parent, name_p, child_type_p, child_id_p) { - let name = UTF8ToString(name_p); - let child_type = 1; - let child_id = await wasmfsOPFSGetOrCreateFile(parent, name, false); - if (child_id == -2) { - child_type = 2; - child_id = await wasmfsOPFSGetOrCreateDir(parent, name, false); + async function(ctx, parent, namePtr, childTypePtr, childIDPtr) { + let name = UTF8ToString(namePtr); + let childType = 1; + let childID = await wasmfsOPFSGetOrCreateFile(parent, name, false); + if (childID == -2) { + childType = 2; + childID = await wasmfsOPFSGetOrCreateDir(parent, name, false); } - {{{ makeSetValue('child_type_p', 0, 'child_type', 'i32') }}}; - {{{ makeSetValue('child_id_p', 0, 'child_id', 'i32') }}}; + {{{ makeSetValue('childTypePtr', 0, 'childType', 'i32') }}}; + {{{ makeSetValue('childIDPtr', 0, 'childID', 'i32') }}}; _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_get_entries__deps: [], - _wasmfs_opfs_get_entries: async function(ctx, dir_id, entries) { - let dir_handle = wasmfsOPFSDirectories.get(dir_id); - for await (const [name, child] of dir_handle.entries()) { + _wasmfs_opfs_get_entries: async function(ctx, dirID, entries) { + let dirHandle = wasmfsOPFSDirectories.get(dirID); + for await (const [name, child] of dirHandle.entries()) { withStackSave(() => { - let name_p = allocateUTF8OnStack(name); + let namePtr = allocateUTF8OnStack(name); // TODO: Figure out how to use `cDefine` here let type = child.kind == "file" ? 1 : 2; - __wasmfs_opfs_record_entry(entries, name_p, type) + __wasmfs_opfs_record_entry(entries, namePtr, type) }); } _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_insert_file__deps: ['$wasmfsOPFSGetOrCreateFile'], - _wasmfs_opfs_insert_file: async function(ctx, parent, name_p, child_id_p) { - let name = UTF8ToString(name_p); - let child_id = await wasmfsOPFSGetOrCreateFile(parent, name, true); - {{{ makeSetValue('child_id_p', 0, 'child_id', 'i32') }}}; + _wasmfs_opfs_insert_file: async function(ctx, parent, namePtr, childIDPtr) { + let name = UTF8ToString(namePtr); + let childID = await wasmfsOPFSGetOrCreateFile(parent, name, true); + {{{ makeSetValue('childIDPtr', 0, 'childID', 'i32') }}}; _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_insert_directory__deps: ['$wasmfsOPFSGetOrCreateDir'], - _wasmfs_opfs_insert_directory: async function(ctx, parent, name_p, child_id_p) { - let name = UTF8ToString(name_p); - let child_id = await wasmfsOPFSGetOrCreateDir(parent, name, true); - {{{ makeSetValue('child_id_p', 0, 'child_id', 'i32') }}}; + _wasmfs_opfs_insert_directory: async function(ctx, parent, namePtr, childIDPtr) { + let name = UTF8ToString(namePtr); + let childID = await wasmfsOPFSGetOrCreateDir(parent, name, true); + {{{ makeSetValue('childIDPtr', 0, 'childID', 'i32') }}}; _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_move__deps: ['$wasmfsOPFSFiles', '$wasmfsOPFSDirectories'], - _wasmfs_opfs_move: async function(ctx, file_id, new_dir_id, name_p) { - let name = UTF8ToString(name_p); - let file_handle = wasmfsOPFSFiles.get(file_id); - let new_dir_handle = wasmfsOPFSDirectories.get(new_dir_id); - await file_handle.move(new_dir_handle, name); + _wasmfs_opfs_move: async function(ctx, fileID, newDirID, namePtr) { + let name = UTF8ToString(namePtr); + let fileHandle = wasmfsOPFSFiles.get(fileID); + let newDirHandle = wasmfsOPFSDirectories.get(newDirID); + await fileHandle.move(newDirHandle, name); _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_remove_child__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSDirectories'], - _wasmfs_opfs_remove_child: async function(ctx, dir_id, name_p) { - let name = UTF8ToString(name_p); - let dir_handle = wasmfsOPFSDirectories.get(dir_id); - await dir_handle.removeEntry(name); + _wasmfs_opfs_remove_child: async function(ctx, dirID, namePtr) { + let name = UTF8ToString(namePtr); + let dirHandle = wasmfsOPFSDirectories.get(dirID); + await dirHandle.removeEntry(name); _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_free_file__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSFiles'], - _wasmfs_opfs_free_file: function(file_id) { - wasmfsOPFSFree(wasmfsOPFSFiles, file_id); + _wasmfs_opfs_free_file: function(fileID) { + wasmfsOPFSFree(wasmfsOPFSFiles, fileID); }, _wasmfs_opfs_free_directory__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSDirectories'], - _wasmfs_opfs_free_directory: function(dir_id) { - wasmfsOPFSFree(wasmfsOPFSDirectories, dir_id); + _wasmfs_opfs_free_directory: function(dirID) { + wasmfsOPFSFree(wasmfsOPFSDirectories, dirID); }, _wasmfs_opfs_open__deps: ['$wasmfsOPFSAllocate', '$wasmfsOPFSFiles', '$wasmfsOPFSAccesses'], - _wasmfs_opfs_open: async function(ctx, file_id, access_id_p) { - let file_handle = wasmfsOPFSFiles.get(file_id); - let access_id; + _wasmfs_opfs_open: async function(ctx, fileID, accessIDPtr) { + let fileHandle = wasmfsOPFSFiles.get(fileID); + let accessID; try { - let access_handle = await file_handle.createSyncAccessHandle(); - access_id = wasmfsOPFSAllocate(wasmfsOPFSAccesses, access_handle); - } catch (err) { - if (err.name === "InvalidStateError") { - access_id = -1; + let accessHandle = await fileHandle.createSyncAccessHandle(); + accessID = wasmfsOPFSAllocate(wasmfsOPFSAccesses, accessHandle); + } catch (e) { + if (e.name === "InvalidStateError") { + accessID = -1; } abort("Unknown error opening file"); } - {{{ makeSetValue('access_id_p', 0, 'access_id', 'i32') }}}; + {{{ makeSetValue('accessIDPtr', 0, 'accessID', 'i32') }}}; _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_close__deps: ['$wasmfsOPFSFree', '$wasmfsOPFSAccesses'], - _wasmfs_opfs_close: async function(ctx, access_id) { - let access_handle = wasmfsOPFSAccesses.get(access_id); - await access_handle.close(); - wasmfsOPFSFree(wasmfsOPFSAccesses, access_id); + _wasmfs_opfs_close: async function(ctx, accessID) { + let accessHandle = wasmfsOPFSAccesses.get(accessID); + await accessHandle.close(); + wasmfsOPFSFree(wasmfsOPFSAccesses, accessID); _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_read__deps: ['$wasmfsOPFSAccesses'], - _wasmfs_opfs_read: function(access_id, buf_p, len, pos) { - let access_handle = wasmfsOPFSAccesses.get(access_id); - let data = HEAPU8.subarray(buf_p, buf_p + len); - return access_handle.read(data, {at: pos}); + _wasmfs_opfs_read: function(accessID, bufPtr, len, pos) { + let accessHandle = wasmfsOPFSAccesses.get(accessID); + let data = HEAPU8.subarray(bufPtr, bufPtr + len); + return accessHandle.read(data, {at: pos}); }, _wasmfs_opfs_write__deps: ['$wasmfsOPFSAccesses'], - _wasmfs_opfs_write: function(access_id, buf_p, len, pos, nwritten_p) { - let access_handle = wasmfsOPFSAccesses.get(access_id); - let data = HEAPU8.subarray(buf_p, buf_p + len); - return access_handle.write(data, {at: pos}); + _wasmfs_opfs_write: function(accessID, bufPtr, len, pos, nwrittenPtr) { + let accessHandle = wasmfsOPFSAccesses.get(accessID); + let data = HEAPU8.subarray(bufPtr, bufPtr + len); + return accessHandle.write(data, {at: pos}); }, _wasmfs_opfs_get_size__deps: ['$wasmfsOPFSAccesses'], - _wasmfs_opfs_get_size: async function(ctx, access_id, size_p) { - let access_handle = wasmfsOPFSAccesses.get(access_id); - let size = await access_handle.getSize(); - {{{ makeSetValue('size_p', 0, 'size', 'i32') }}}; + _wasmfs_opfs_get_size: async function(ctx, accessID, sizePtr) { + let accessHandle = wasmfsOPFSAccesses.get(accessID); + let size = await accessHandle.getSize(); + {{{ makeSetValue('sizePtr', 0, 'size', 'i32') }}}; _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_set_size__deps: ['$wasmfsOPFSAccesses'], - _wasmfs_opfs_set_size: async function(ctx, access_id, size) { - let access_handle = wasmfsOPFSAccesses.get(access_id); - await access_handle.truncate(size); + _wasmfs_opfs_set_size: async function(ctx, accessID, size) { + let accessHandle = wasmfsOPFSAccesses.get(accessID); + await accessHandle.truncate(size); _emscripten_proxy_finish(ctx); }, _wasmfs_opfs_flush__deps: ['$wasmfsOPFSAccesses'], - _wasmfs_opfs_flush: async function(ctx, access_id) { - let access_handle = wasmfsOPFSAccesses.get(access_id); - await access_handle.flush(); + _wasmfs_opfs_flush: async function(ctx, accessID) { + let accessHandle = wasmfsOPFSAccesses.get(accessID); + await accessHandle.flush(); _emscripten_proxy_finish(ctx); } }); diff --git a/system/lib/wasmfs/backends/opfs_backend.cpp b/system/lib/wasmfs/backends/opfs_backend.cpp index 9716c2cef44f8..f4946af18c9cc 100644 --- a/system/lib/wasmfs/backends/opfs_backend.cpp +++ b/system/lib/wasmfs/backends/opfs_backend.cpp @@ -10,9 +10,7 @@ #include "wasmfs.h" #include -namespace wasmfs { - -using ProxyWorker = emscripten::ProxyWorker; +using namespace wasmfs; extern "C" { @@ -79,6 +77,10 @@ void _wasmfs_opfs_flush(em_proxying_ctx* ctx, int access_id); } // extern "C" +namespace { + +using ProxyWorker = emscripten::ProxyWorker; + class OPFSFile : public DataFile { public: ProxyWorker& proxy; @@ -249,6 +251,8 @@ class OPFSBackend : public Backend { std::shared_ptr createFile(mode_t mode) override { // No way to support a raw file without a parent directory. + // TODO: update the core system to document this as a possible result of + // `createFile` and to handle it gracefully. return nullptr; } @@ -263,6 +267,8 @@ class OPFSBackend : public Backend { } }; +} // anonymous namespace + extern "C" { backend_t wasmfs_create_opfs_backend() { @@ -275,5 +281,3 @@ void EMSCRIPTEN_KEEPALIVE _wasmfs_opfs_record_entry( } } // extern "C" - -} // namespace wasmfs From 7a2a66deccda9041059a1637940a39746841505d Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 29 Apr 2022 11:12:53 -0700 Subject: [PATCH 05/14] TODO about id pool utility --- src/library_wasmfs_opfs.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index f39a8fef70314..868a9bc436a4c 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -5,6 +5,7 @@ */ mergeInto(LibraryManager.library, { + // TODO: Generate these ID pools from a common utility. $wasmfsOPFSDirectories: { allocated: [], free: [], From d677978e07d1dff47eb7eca4b53982a3b6664fe9 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 29 Apr 2022 11:39:20 -0700 Subject: [PATCH 06/14] cDefine --- src/library_wasmfs_opfs.js | 5 +++-- src/struct_info_cxx.json | 21 +++++++++++++++++++++ src/struct_info_internal.json | 9 --------- system/lib/wasmfs/file.h | 7 ++++++- tools/gen_struct_info.py | 3 ++- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index 868a9bc436a4c..9dd53bd8ebf71 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -134,8 +134,9 @@ mergeInto(LibraryManager.library, { for await (const [name, child] of dirHandle.entries()) { withStackSave(() => { let namePtr = allocateUTF8OnStack(name); - // TODO: Figure out how to use `cDefine` here - let type = child.kind == "file" ? 1 : 2; + let type = child.kind == "file" ? + {{{ cDefine('File::DataFileKind') }}} : + {{{ cDefine('File::DirectoryKind') }}}; __wasmfs_opfs_record_entry(entries, namePtr, type) }); } diff --git a/src/struct_info_cxx.json b/src/struct_info_cxx.json index 2b73cbca0ee62..c3770bc129e24 100644 --- a/src/struct_info_cxx.json +++ b/src/struct_info_cxx.json @@ -11,5 +11,26 @@ "adjustedPtr" ] } + }, + // =========================================== + // WasmFS + // =========================================== + { + "file": "file.h", + "defines": [ + "wasmfs::File::UnknownKind", + "wasmfs::File::DataFileKind", + "wasmfs::File::DirectoryKind", + "wasmfs::File::SymlinkKind" + ] + }, + { + "file": "async_callback.h", + "structs": { + "CallbackState": [ + "result", + "offset" + ] + } } ] diff --git a/src/struct_info_internal.json b/src/struct_info_internal.json index 9061907eea27f..9d46fa7c19463 100644 --- a/src/struct_info_internal.json +++ b/src/struct_info_internal.json @@ -48,15 +48,6 @@ ] } }, - { - "file": "async_callback.h", - "structs": { - "CallbackState": [ - "result", - "offset" - ] - } - }, { "file": "proxying_notification_state.h", "defines": [ diff --git a/system/lib/wasmfs/file.h b/system/lib/wasmfs/file.h index 8b0b8198494ca..953414d350f98 100644 --- a/system/lib/wasmfs/file.h +++ b/system/lib/wasmfs/file.h @@ -44,7 +44,12 @@ using oflags_t = uint32_t; // to implement the mapping from `File` objects to their underlying files. class File : public std::enable_shared_from_this { public: - enum FileKind { UnknownKind, DataFileKind, DirectoryKind, SymlinkKind }; + enum FileKind { + UnknownKind = 0, + DataFileKind = 1, + DirectoryKind = 2, + SymlinkKind = 3 + }; const FileKind kind; diff --git a/tools/gen_struct_info.py b/tools/gen_struct_info.py index 5ec984554cda1..bcd8107c2d625 100755 --- a/tools/gen_struct_info.py +++ b/tools/gen_struct_info.py @@ -434,12 +434,13 @@ def main(args): '-I' + utils.path_from_root('system/lib/libc/musl/src/internal'), '-I' + utils.path_from_root('system/lib/libc/musl/src/include'), '-I' + utils.path_from_root('system/lib/pthread/'), - '-I' + utils.path_from_root('system/lib/wasmfs/'), ] cxxflags = [ '-I' + utils.path_from_root('system/lib/libcxxabi/src'), '-D__USING_EMSCRIPTEN_EXCEPTIONS__', + '-I' + utils.path_from_root('system/lib/wasmfs/'), + '-std=c++17', ] # Look for structs in all passed headers. From 4cc379bb12071ad0ddfbe1850f373a5925f20e3d Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 29 Apr 2022 14:29:14 -0700 Subject: [PATCH 07/14] Address comments in JS --- src/library_wasmfs_opfs.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index 9dd53bd8ebf71..82b41228e84b3 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -10,6 +10,7 @@ mergeInto(LibraryManager.library, { allocated: [], free: [], get: function(i) { + assert(this.allocated[i] !== undefined); return this.allocated[i]; } }, @@ -18,6 +19,7 @@ mergeInto(LibraryManager.library, { allocated: [], free: [], get: function(i) { + assert(this.allocated[i] !== undefined); return this.allocated[i]; } }, @@ -26,6 +28,7 @@ mergeInto(LibraryManager.library, { allocated: [], free: [], get: function(i) { + assert(this.allocated[i] !== undefined); return this.allocated[i]; } }, @@ -69,7 +72,6 @@ mergeInto(LibraryManager.library, { '$wasmfsOPFSFiles'], $wasmfsOPFSGetOrCreateFile: async function(parent, name, create) { let parentHandle = wasmfsOPFSDirectories.get(parent); - assert(parentHandle !== undefined); let fileHandle; try { fileHandle = await parentHandle.getFileHandle(name, {create: create}); @@ -95,7 +97,6 @@ mergeInto(LibraryManager.library, { '$wasmfsOPFSDirectories'], $wasmfsOPFSGetOrCreateDir: async function(parent, name, create) { let parentHandle = wasmfsOPFSDirectories.get(parent); - assert(parentHandle !== undefined); let childHandle; try { childHandle = @@ -131,7 +132,7 @@ mergeInto(LibraryManager.library, { _wasmfs_opfs_get_entries__deps: [], _wasmfs_opfs_get_entries: async function(ctx, dirID, entries) { let dirHandle = wasmfsOPFSDirectories.get(dirID); - for await (const [name, child] of dirHandle.entries()) { + for await (let [name, child] of dirHandle.entries()) { withStackSave(() => { let namePtr = allocateUTF8OnStack(name); let type = child.kind == "file" ? From 44bd93d3018396625008bba0d2d631138b3aa814 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 29 Apr 2022 14:33:28 -0700 Subject: [PATCH 08/14] Test persistence --- tests/test_browser.py | 6 ++++-- tests/wasmfs/wasmfs_opfs.c | 26 +++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index dde30d9f0d9e6..75529a77d4650 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -5296,8 +5296,10 @@ def test_wasmfs_fetch_backend(self, args): @requires_threads def test_wasmfs_opfs(self): - self.btest_exit(test_file('wasmfs/wasmfs_opfs.c'), - args=['-sWASMFS', '-pthread', '-sPROXY_TO_PTHREAD']) + trest = test_file('wasmfs/wasmfs_opfs.c') + args = ['-sWASMFS', '-pthread', '-sPROXY_TO_PTHREAD'] + self.btest_exit(test, args=args + ['-DWASMFS_SETUP']) + self.btest_exit(test, args=args + ['-DWASMFS_RESUME']) @no_firefox('no 4GB support yet') def test_zzz_zzz_emmalloc_memgrowth(self, *args): diff --git a/tests/wasmfs/wasmfs_opfs.c b/tests/wasmfs/wasmfs_opfs.c index b2c08031146fe..68d1df422c660 100644 --- a/tests/wasmfs/wasmfs_opfs.c +++ b/tests/wasmfs/wasmfs_opfs.c @@ -12,25 +12,39 @@ #include #include +// Define WASMFS_SETUP then WASMFS_RESUME to run the test as two separate +// programs to test persistence. Alternatively, define neither and run the full +// test as a single program. + int main(int argc, char* argv[]) { + int err, fd; + const char* msg = "Hello, OPFS!"; + emscripten_console_log("starting main"); backend_t opfs = wasmfs_create_opfs_backend(); emscripten_console_log("created OPFS backend"); - int err = wasmfs_create_directory("/opfs", 0777, opfs); + err = wasmfs_create_directory("/opfs", 0777, opfs); assert(err == 0); emscripten_console_log("mounted OPFS root directory"); +#ifdef WASMFS_RESUME + + fd = open("/opfs/working/foo.txt", O_RDWR); + assert(fd > 0); + emscripten_console_log("opened existing OPFS file"); + +#else // !WASMFS_RESUME + err = mkdir("/opfs/working", 0777); assert(err == 0); emscripten_console_log("created OPFS directory"); - int fd = open("/opfs/working/foo.txt", O_RDWR | O_CREAT | O_EXCL, 0777); + fd = open("/opfs/working/foo.txt", O_RDWR | O_CREAT | O_EXCL, 0777); assert(fd > 0); emscripten_console_log("created OPFS file"); - const char* msg = "Hello, OPFS!"; int nwritten = write(fd, msg, strlen(msg)); assert(nwritten == strlen(msg)); emscripten_console_logf("wrote message: %s (%d)", msg, nwritten); @@ -39,6 +53,8 @@ int main(int argc, char* argv[]) { assert(off == 0); emscripten_console_log("seeked"); +#endif // !WASMFS_RESUME + char buf[100] = {}; int nread = read(fd, buf, 100); assert(nread == strlen(msg)); @@ -54,6 +70,8 @@ int main(int argc, char* argv[]) { assert(stat.st_size == strlen(msg)); emscripten_console_log("statted"); +#ifndef WASMFS_SETUP + err = ftruncate(fd, 100); assert(err == 0); err = fstat(fd, &stat); @@ -116,4 +134,6 @@ int main(int argc, char* argv[]) { emscripten_console_log("removed OPFS directory"); emscripten_console_log("done"); + +#endif // !WASMFS_SETUP } From df5a15e813c5c689bd16ddc12db91a49586ed11a Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 29 Apr 2022 14:38:06 -0700 Subject: [PATCH 09/14] fix --- tests/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 75529a77d4650..f49da1ecae962 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -5296,7 +5296,7 @@ def test_wasmfs_fetch_backend(self, args): @requires_threads def test_wasmfs_opfs(self): - trest = test_file('wasmfs/wasmfs_opfs.c') + test = test_file('wasmfs/wasmfs_opfs.c') args = ['-sWASMFS', '-pthread', '-sPROXY_TO_PTHREAD'] self.btest_exit(test, args=args + ['-DWASMFS_SETUP']) self.btest_exit(test, args=args + ['-DWASMFS_RESUME']) From 9a5306c8cb0e6135e5cdca040f913f51a97594c4 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 2 May 2022 09:32:00 -0700 Subject: [PATCH 10/14] update struct info test --- tests/reference_struct_info.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/reference_struct_info.json b/tests/reference_struct_info.json index acd86d9438ebb..64a28381a4dc2 100644 --- a/tests/reference_struct_info.json +++ b/tests/reference_struct_info.json @@ -274,6 +274,10 @@ "F_SETLKW64": 7, "F_SETOWN": 8, "F_UNLCK": 2, + "File::DataFileKind": 1, + "File::DirectoryKind": 2, + "File::SymlinkKind": 3, + "File::UnknownKind": 0, "INADDR_LOOPBACK": 2130706433, "INT_MAX": 2147483647, "IPPROTO_TCP": 6, From 2eb90f1210ccd97c37705f9508e462b1f82141ac Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 2 May 2022 13:26:16 -0700 Subject: [PATCH 11/14] fix --- src/library_wasmfs_opfs.js | 13 ++++++++++--- tests/wasmfs/wasmfs_opfs.c | 11 +++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index 82b41228e84b3..b03fc723057c8 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -108,7 +108,7 @@ mergeInto(LibraryManager.library, { if (e.name === "TypeMismatchError") { return -2; } - abort("Unknown exception " + e.name); + throw e; } return wasmfsOPFSAllocate(wasmfsOPFSDirectories, childHandle); }, @@ -195,13 +195,20 @@ mergeInto(LibraryManager.library, { let fileHandle = wasmfsOPFSFiles.get(fileID); let accessID; try { - let accessHandle = await fileHandle.createSyncAccessHandle(); + let accessHandle; + // TODO: Remove this once the Access Handles API has settled. + if (FileSystemFileHandle.prototype.createSyncAccessHandle.length == 0) { + accessHandle = await fileHandle.createSyncAccessHandle(); + } else { + accessHandle = await fileHandle.createSyncAccessHandle( + {mode: "in-place"}); + } accessID = wasmfsOPFSAllocate(wasmfsOPFSAccesses, accessHandle); } catch (e) { if (e.name === "InvalidStateError") { accessID = -1; } - abort("Unknown error opening file"); + throw e; } {{{ makeSetValue('accessIDPtr', 0, 'accessID', 'i32') }}}; _emscripten_proxy_finish(ctx); diff --git a/tests/wasmfs/wasmfs_opfs.c b/tests/wasmfs/wasmfs_opfs.c index 68d1df422c660..b6089be3ade4a 100644 --- a/tests/wasmfs/wasmfs_opfs.c +++ b/tests/wasmfs/wasmfs_opfs.c @@ -16,6 +16,8 @@ // programs to test persistence. Alternatively, define neither and run the full // test as a single program. +void cleanup(void); + int main(int argc, char* argv[]) { int err, fd; const char* msg = "Hello, OPFS!"; @@ -37,6 +39,9 @@ int main(int argc, char* argv[]) { #else // !WASMFS_RESUME + // Remove old files if they exist. + cleanup(); + err = mkdir("/opfs/working", 0777); assert(err == 0); emscripten_console_log("created OPFS directory"); @@ -137,3 +142,9 @@ int main(int argc, char* argv[]) { #endif // !WASMFS_SETUP } + +void cleanup(void) { + unlink("/opfs/working/foo.txt"); + rmdir("/opfs/working"); + unlink("/opfs/foo.txt"); +} From d90b3f96da2711fdab8c3536bb68bf9e62160604 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 2 May 2022 16:37:16 -0700 Subject: [PATCH 12/14] Add chrome flag --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d75521667cad..850a427f9585a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -241,7 +241,7 @@ commands: EMTEST_DETECT_TEMPFILE_LEAKS: "0" # --no-sandbox becasue we are running as root and chrome requires # this flag for now: https://crbug.com/638180 - CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile" + CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile --enable-experimental-web-platform-features" CHROME_FLAGS_HEADLESS: "--headless --remote-debugging-port=1234" CHROME_FLAGS_WASM: "--enable-features=WebAssembly --enable-features=SharedArrayBuffer --disable-features=WebAssemblyTrapHandler --js-flags=\"--experimental-wasm-threads --harmony-sharedarraybuffer\"" CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito" From 2153e7362d1136aee5be1598217a564385c4377f Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 2 May 2022 18:17:28 -0700 Subject: [PATCH 13/14] no firefox --- tests/test_browser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_browser.py b/tests/test_browser.py index f49da1ecae962..af0365e0bd8c6 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -5295,6 +5295,7 @@ def test_wasmfs_fetch_backend(self, args): args=['-sWASMFS', '-sUSE_PTHREADS'] + args) @requires_threads + @no_firefox('no OPFS support yet') def test_wasmfs_opfs(self): test = test_file('wasmfs/wasmfs_opfs.c') args = ['-sWASMFS', '-pthread', '-sPROXY_TO_PTHREAD'] From 3a2c8ed74fd9fed3d659500e56b5b0c4fbcb2981 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Tue, 3 May 2022 13:59:29 -0700 Subject: [PATCH 14/14] Download new chrome --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 850a427f9585a..01ffc72c55ebc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ commands: # Currently downloading form our own buckets due to: # https://github.com/emscripten-core/emscripten/issues/14987 #wget -O ~/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - wget -O ~/chrome.deb https://storage.googleapis.com/webassembly/chrome/google-chrome-stable_current_amd64.deb + wget -O ~/chrome.deb https://storage.googleapis.com/webassembly/chrome/google-chrome-stable_current_amd64_new.deb dpkg -i ~/chrome.deb emsdk-env: description: "emsdk_env.sh"