Compile with WASI

kddnewton · kddnewton · commit 73c44b0b9c39 · 2023-10-25T21:48:50.000-04:00
diff --git a/.github/workflows/javascript-bindings.yml b/.github/workflows/javascript-bindings.yml
@@ -0,0 +1,48 @@
+name: JavaScript Bindings
+
+on:
+  push:
+    paths:
+      - ".github/workflows/javascript-bindings.yml"
+      - "include/"
+      - "src/"
+      - "*akefile*"
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: head
+          bundler-cache: true
+
+      - name: rake templates
+        run: bundle exec rake templates
+
+      - name: Set up WASI-SDK
+        run: |
+          wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz
+          tar xvf wasi-sdk-20.0-linux.tar.gz
+
+      - name: Build the project
+        run: make wasm WASI_SDK_PATH=$(pwd)/wasi-sdk-20.0
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+
+      - name: Run the tests
+        run: npm test
+        working-directory: javascript
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: prism.wasm
+          path: javascript/src/prism.wasm
diff --git a/.gitignore b/.gitignore
@@ -25,6 +25,12 @@ a.out
 /ext/prism/api_node.c
 /fuzz/output/
 /include/prism/ast.h
+/javascript/node_modules/
+/javascript/package-lock.json
+/javascript/src/deserialize.js
+/javascript/src/nodes.js
+/javascript/src/prism.wasm
+/javascript/types/
 /java/org/prism/AbstractNodeVisitor.java
 /java/org/prism/Loader.java
 /java/org/prism/Nodes.java
diff --git a/Makefile b/Makefile
@@ -13,6 +13,7 @@ SOEXT := $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 CPPFLAGS := -Iinclude
 CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
 CC := cc
+WASI_SDK_PATH := /opt/wasi-sdk
 
 HEADERS := $(shell find include -name '*.h')
 SOURCES := $(shell find src -name '*.c')
@@ -23,6 +24,7 @@ all: shared static
 
 shared: build/librubyparser.$(SOEXT)
 static: build/librubyparser.a
+wasm: javascript/src/prism.wasm
 
 build/librubyparser.$(SOEXT): $(SHARED_OBJECTS)
 	$(ECHO) "linking $@"
@@ -32,6 +34,10 @@ build/librubyparser.a: $(STATIC_OBJECTS)
 	$(ECHO) "building $@"
 	$(Q) $(AR) $(ARFLAGS) $@ $(STATIC_OBJECTS) $(Q1:0=>/dev/null)
 
+javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
+	$(ECHO) "building $@"
+	$(Q) $(WASI_SDK_PATH)/bin/clang --sysroot=$(WASI_SDK_PATH)/share/wasi-sysroot/ $(DEBUG_FLAGS) -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -o $@ $(SOURCES)
+
 build/shared/%.o: src/%.c Makefile $(HEADERS)
 	$(ECHO) "compiling $@"
 	$(Q) mkdir -p $(@D)
diff --git a/README.md b/README.md
@@ -85,6 +85,7 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additio
 * [Encoding](docs/encoding.md)
 * [Fuzzing](docs/fuzzing.md)
 * [Heredocs](docs/heredocs.md)
+* [JavaScript](docs/javascript.md)
 * [Mapping](docs/mapping.md)
 * [Ripper](docs/ripper.md)
 * [Ruby API](docs/ruby_api.md)
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -4,6 +4,8 @@ A lot of code in prism's repository is templated from a single configuration fil
 
 * `ext/prism/api_node.c` - for defining how to build Ruby objects for the nodes out of C structs
 * `include/prism/ast.h` - for defining the C structs that represent the nodes
+* `javascript/src/deserialize.js` - for defining how to deserialize the nodes in JavaScript
+* `javascript/src/nodes.js` - for defining the nodes in JavaScript
 * `java/org/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java
 * `java/org/prism/Loader.java` - for defining how to deserialize the nodes in Java
 * `java/org/prism/Nodes.java` - for defining the nodes in Java
diff --git a/docs/javascript.md b/docs/javascript.md
@@ -0,0 +1,90 @@
+# JavaScript
+
+Prism provides bindings to JavaScript out of the box.
+
+## Node
+
+To use the package from node, install the `@ruby/prism` dependency:
+
+```sh
+npm install @ruby/prism
+```
+
+Then import the package:
+
+```js
+import { loadPrism } from "@ruby/prism";
+```
+
+Then call the load function to get a parse function:
+
+```js
+const parse = await loadPrism();
+```
+
+## Browser
+
+To use the package from the browser, you will need to do some additional work. The [javascript/example.html](javascript/example.html) file shows an example of running Prism in the browser. You will need to instantiate the WebAssembly module yourself and then pass it to the `parsePrism` function.
+
+First, get a shim for WASI since not all browsers support it yet.
+
+```js
+import { WASI } from "https://unpkg.com/@bjorn3/browser_wasi_shim@latest/dist/index.js";
+```
+
+Next, import the `parsePrism` function from `@ruby/prism`, either through a CDN or by bundling it with your application.
+
+```js
+import { parsePrism } from "https://unpkg.com/@ruby/prism@latest/src/parsePrism.js";
+```
+
+Next, fetch and instantiate the WebAssembly module. You can access it through a CDN or by bundling it with your application.
+
+```js
+const wasm = await WebAssembly.compileStreaming(fetch("https://unpkg.com/@ruby/prism@latest/src/prism.wasm"));
+```
+
+Next, instantiate the module and initialize WASI.
+
+```js
+const wasi = new WASI([], [], []);
+const instance = await WebAssembly.instantiate(wasm, { wasi_snapshot_preview1: wasi.wasiImport });
+wasi.initialize(instance);
+```
+
+Finally, you can create a function that will parse a string of Ruby code.
+
+```js
+function parse(source) {
+  return parsePrism(instance.exports, source);
+}
+```
+
+## API
+
+Now that we have access to a `parse` function, we can use it to parse Ruby code:
+
+```js
+const parseResult = parse("1 + 2");
+```
+
+A ParseResult object is very similar to the Prism::ParseResult object from Ruby. It has the same properties: `value`, `comments`, `magicComments`, `errors`, and `warnings`. Here we can serialize the AST to JSON.
+
+```js
+console.log(JSON.stringify(parseResult.value, null, 2));
+```
+
+## Building
+
+To build the WASM package yourself, first obtain a copy of `wasi-sdk`. You can retrieve this here: <https://github.com/WebAssembly/wasi-sdk>. Next, run:
+
+```sh
+make wasm WASI_SDK_PATH=path/to/wasi-sdk
+```
+
+This will generate `javascript/src/prism.wasm`. From there, you can run the tests to verify everything was generated correctly.
+
+```sh
+cd javascript
+node test
+```
diff --git a/javascript/README.md b/javascript/README.md
@@ -0,0 +1,3 @@
+# @ruby/prism
+
+JavaScript bindings for Ruby's [prism](https://github.com/ruby/prism) parser.
diff --git a/javascript/example.html b/javascript/example.html
@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>@ruby/prism</title>
+  </head>
+  <body style="margin: 0;">
+    <div style="display: grid; grid-template-columns: 1fr 1fr;">
+      <div>
+        <textarea id="input" style="box-sizing: border-box; width: 100%; height: 100vh; resize: none; vertical-align: top;"></textarea>
+      </div>
+      <div style="height: 100vh; overflow-y: scroll;">
+        <code><pre id="output" style="margin: 0; padding: 1em;"></pre></code>  
+      </div>
+    </div>
+    <script type="module">
+import { WASI } from "https://unpkg.com/@bjorn3/browser_wasi_shim@latest/dist/index.js";
+import { parsePrism } from "https://unpkg.com/@ruby/prism@latest/src/parsePrism.js";
+
+const wasm = await WebAssembly.compileStreaming(fetch("https://unpkg.com/@ruby/prism@latest/src/prism.wasm"));
+const wasi = new WASI([], [], []);
+
+const instance = await WebAssembly.instantiate(wasm, { wasi_snapshot_preview1: wasi.wasiImport });
+wasi.initialize(instance);
+
+let timeout = null;
+const input = document.getElementById("input");
+const output = document.getElementById("output");
+
+input.addEventListener("input", function (event) {
+  if (timeout) clearTimeout(timeout);
+
+  timeout = setTimeout(function () {
+    const result = parsePrism(instance.exports, event.target.value);
+    output.textContent = JSON.stringify(result, null, 2);
+  }, 250);
+});
+    </script>
+  </body>
+</html>
diff --git a/javascript/package.json b/javascript/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "@ruby/prism",
+  "version": "0.15.2",
+  "description": "Prism Ruby parser",
+  "type": "module",
+  "main": "src/index.js",
+  "types": "types/index.d.ts",
+  "scripts": {
+    "prepublishOnly": "npm run type",
+    "test": "node test.js",
+    "type": "tsc --allowJs -d --emitDeclarationOnly --outDir types src/index.js"
+  },
+  "author": "Shopify <ruby@shopify.com>",
+  "license": "MIT"
+}
diff --git a/javascript/src/index.js b/javascript/src/index.js
@@ -0,0 +1,23 @@
+import { WASI } from "wasi";
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+
+import { ParseResult } from "./deserialize.js";
+import { parsePrism } from "./parsePrism.js";
+
+/**
+ * Load the prism wasm module and return a parse function.
+ *
+ * @returns {Promise<(source: string) => ParseResult>}
+ */
+export async function loadPrism() {
+  const wasm = await WebAssembly.compile(await readFile(fileURLToPath(new URL("prism.wasm", import.meta.url))));
+  const wasi = new WASI({ version: "preview1" });
+
+  const instance = await WebAssembly.instantiate(wasm, wasi.getImportObject());
+  wasi.initialize(instance);
+
+  return function (source) {
+    return parsePrism(instance.exports, source);
+  }
+}
diff --git a/javascript/src/parsePrism.js b/javascript/src/parsePrism.js
@@ -0,0 +1,28 @@
+import { ParseResult, deserialize } from "./deserialize.js";
+
+/**
+ * Parse the given source code.
+ *
+ * @param {WebAssembly.Exports} prism
+ * @param {string} source
+ * @returns {ParseResult}
+ */
+export function parsePrism(prism, source) {
+  const sourceArray = new TextEncoder().encode(source);
+  const sourcePointer = prism.calloc(1, sourceArray.length);
+
+  const bufferPointer = prism.calloc(prism.pm_buffer_sizeof(), 1);
+  prism.pm_buffer_init(bufferPointer);
+
+  const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, sourceArray.length);
+  sourceView.set(sourceArray);
+
+  prism.pm_parse_serialize(sourcePointer, sourceArray.length, bufferPointer);
+  const serializedView = new Uint8Array(prism.memory.buffer, prism.pm_buffer_value(bufferPointer), prism.pm_buffer_length(bufferPointer));
+  const result = deserialize(sourceArray, serializedView);
+
+  prism.pm_buffer_free(bufferPointer);
+  prism.free(sourcePointer);
+  prism.free(bufferPointer);
+  return result;
+}
diff --git a/javascript/test.js b/javascript/test.js
@@ -0,0 +1,80 @@
+import test from "node:test";
+import assert from "node:assert";
+import { loadPrism } from "./src/index.js";
+import * as nodes from "./src/nodes.js";
+
+const parse = await loadPrism();
+
+test("node", () => {
+  const result = parse("foo");
+  assert(result.value instanceof nodes.ProgramNode);
+});
+
+test("node? present", () => {
+  const result = parse("foo.bar");
+  assert(result.value.statements.body[0].receiver instanceof nodes.CallNode);
+});
+
+test("node? absent", () => {
+  const result = parse("foo");
+  assert(result.value.statements.body[0].receiver === null);
+});
+
+test("node[]", () => {
+  const result = parse("foo.bar");
+  assert(result.value.statements.body instanceof Array);
+});
+
+test("string", () => {
+  const result = parse('"foo"');
+  assert(result.value.statements.body[0].unescaped === "foo");
+});
+
+test("constant", () => {
+  const result = parse("foo = 1");
+  assert(result.value.locals[0] === "foo");
+});
+
+test("constant? present", () => {
+  const result = parse("def foo(*bar); end");
+  assert(result.value.statements.body[0].parameters.rest.name === "bar");
+});
+
+test("constant? absent", () => {
+  const result = parse("def foo(*); end");
+  assert(result.value.statements.body[0].parameters.rest.name === null);
+});
+
+test("constant[]", async() => {
+  const result = parse("foo = 1");
+  assert(result.value.locals instanceof Array);
+});
+
+test("location", () => {
+  const result = parse("foo = 1");
+  assert(typeof result.value.location.startOffset === "number");
+});
+
+test("location? present", () => {
+  const result = parse("def foo = bar");
+  assert(result.value.statements.body[0].equalLoc !== null);
+});
+
+test("location? absent", () => {
+  const result = parse("def foo; bar; end");
+  assert(result.value.statements.body[0].equalLoc === null);
+});
+
+test("uint32", () => {
+  const result = parse("foo = 1");
+  assert(result.value.statements.body[0].depth === 0);
+});
+
+test("flags", () => {
+  const result = parse("/foo/mi");
+  const regexp = result.value.statements.body[0];
+
+  assert(regexp.isIgnoreCase());
+  assert(regexp.isMultiLine());
+  assert(!regexp.isExtended());
+});
diff --git a/prism.gemspec b/prism.gemspec
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
     "docs/encoding.md",
     "docs/fuzzing.md",
     "docs/heredocs.md",
+    "docs/javascript.md",
     "docs/mapping.md",
     "docs/prism.png",
     "docs/ripper.md",
diff --git a/rakelib/check_manifest.rake b/rakelib/check_manifest.rake
diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb
diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb
diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb
diff --git a/templates/template.rb b/templates/template.rb

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# @ruby/prism`
	`2`	`+`
	`3`	`+JavaScript bindings for Ruby's [prism](https://github.com/ruby/prism) parser.`