Initial commit

Author: kddnewton

.gitignore

@@ -0,0 +1,8 @@
+/dist
+/head-wasm32-unknown-wasi-full-js/
+/lib
+/node_modules/
+/ruby.wasm
+/src/app.wasm
+/wasi-vfs
+/yarn-error.log

Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "bundler"
+gem "rake"
+
+gem "syntax_tree"
+gem "syntax_tree-haml"
+gem "syntax_tree-rbs"

Gemfile.lock

@@ -0,0 +1,34 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    haml (5.2.2)
+      temple (>= 0.8.0)
+      tilt
+    prettier_print (0.1.0)
+    rake (13.0.6)
+    rbs (2.6.0)
+    syntax_tree (3.3.0)
+      prettier_print
+    syntax_tree-haml (1.3.1)
+      haml (>= 5.2)
+      prettier_print
+      syntax_tree (>= 2.0.1)
+    syntax_tree-rbs (0.5.0)
+      prettier_print
+      rbs
+      syntax_tree (>= 2.0.1)
+    temple (0.8.2)
+    tilt (2.0.11)
+
+PLATFORMS
+  x86_64-darwin-21
+
+DEPENDENCIES
+  bundler
+  rake
+  syntax_tree
+  syntax_tree-haml
+  syntax_tree-rbs
+
+BUNDLED WITH
+   2.3.14

Rakefile

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "rake/clean"
+
+desc "Download and unzip the wasi-vfs executable"
+file "wasi-vfs" do
+  version = "0.1.1"
+  filename =
+    if ENV["CI"]
+      "wasi-vfs-cli-x86_64-unknown-linux-gnu.zip"
+    else
+      "wasi-vfs-cli-x86_64-apple-darwin.zip"
+    end
+
+  `curl -LO "https://github.com/kateinoigakukun/wasi-vfs/releases/download/v#{version}/#{filename}"`
+  `unzip #{filename}`
+  rm filename
+end
+
+desc "Download and untar the Ruby WASI directory"
+directory "head-wasm32-unknown-wasi-full-js" do
+  require "json"
+  version = JSON.parse(File.read("package.json"))["dependencies"]["ruby-head-wasm-wasi"][1..]
+  filename = "ruby-head-wasm32-unknown-wasi-full-js.tar.gz"
+
+  `curl -LO https://github.com/ruby/ruby.wasm/releases/download/ruby-head-wasm-wasi-#{version}/#{filename}`
+  `tar xfz #{filename}`
+  rm filename
+end
+
+desc "Extract the Ruby executable from the WASI directory"
+file "ruby.wasm" => ["head-wasm32-unknown-wasi-full-js"] do
+  cp "head-wasm32-unknown-wasi-full-js/usr/local/bin/ruby", "ruby.wasm"
+end
+
+desc "Build the app.wasm file with additional files built in"
+file "src/app.wasm" => ["Gemfile.lock", "wasi-vfs", "ruby.wasm"] do
+  require "bundler/setup"
+
+  loaded_before = $LOADED_FEATURES.dup
+  require "syntax_tree"
+  require "syntax_tree/haml"
+  # require "syntax_tree/rbs"
+  loaded_after = $LOADED_FEATURES - loaded_before
+
+  filepaths = loaded_after.group_by { |filepath| filepath[%r{/gems/(.+?)-\d}, 1] }
+  gem_names = filepaths.keys.tap { |names| names.delete(nil) }.sort
+  puts "Gems loaded: #{gem_names.join(", ")}"
+
+  mkdir "lib"
+  gem_names.each do |gem_name|
+    libdir = $:.find { _1.match?(%r{/#{gem_name}-\d}) }
+
+    Dir["#{libdir}/**/*"].sort.each do |filepath|
+      relative = "lib#{filepath.delete_prefix(libdir)}"
+
+      if File.directory?(filepath)
+        mkdir_p relative
+      else
+        cp filepath, relative
+      end
+    end
+  end
+
+  `./wasi-vfs pack ruby.wasm --mapdir /lib::./lib --mapdir /usr::./head-wasm32-unknown-wasi-full-js/usr -o src/app.wasm`
+  rm_rf "lib"
+end
+
+CLOBBER.concat(%w[wasi-vfs head-wasm32-unknown-wasi-full-js ruby.wasm src/app.wasm lib])
+
+task default: "src/app.wasm"

bin/build

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+const esbuild = require("esbuild");
+const path = require("path");
+
+const wasmPlugin = {
+  name: "wasm",
+  setup(build) {
+    let path = require("path")
+    let fs = require("fs")
+
+    // Resolve ".wasm" files to a path with a namespace
+    build.onResolve({ filter: /\.wasm$/ }, (args) => {
+      // If this is the import inside the stub module, import the
+      // binary itself. Put the path in the "wasm-binary" namespace
+      // to tell our binary load callback to load the binary file.
+      if (args.namespace === "wasm-stub") {
+        return { path: args.path, namespace: "wasm-binary" };
+      }
+
+      // Otherwise, generate the JavaScript stub module for this
+      // ".wasm" file. Put it in the "wasm-stub" namespace to tell
+      // our stub load callback to fill it with JavaScript.
+      //
+      // Resolve relative paths to absolute paths here since this
+      // resolve callback is given "resolveDir", the directory to
+      // resolve imports against.
+      if (args.resolveDir === "") {
+        return; // Ignore unresolvable paths
+      }
+
+      return {
+        path: path.isAbsolute(args.path) ? args.path : path.join(args.resolveDir, args.path),
+        namespace: "wasm-stub"
+      };
+    });
+
+    // Virtual modules in the "wasm-stub" namespace are filled with
+    // the JavaScript code for compiling the WebAssembly binary. The
+    // binary itself is imported from a second virtual module.
+    build.onLoad({ filter: /.*/, namespace: "wasm-stub" }, async (args) => ({
+      contents: `import wasm from ${JSON.stringify(args.path)};
+        export default (imports) => WebAssembly.instantiate(wasm, imports).then((result) => result.instance);`,
+    }));
+
+    // Virtual modules in the "wasm-binary" namespace contain the
+    // actual bytes of the WebAssembly file. This uses esbuild's
+    // built-in "binary" loader instead of manually embedding the
+    // binary data inside JavaScript code ourselves.
+    build.onLoad({ filter: /.*/, namespace: "wasm-binary" }, async (args) => ({
+      contents: await fs.promises.readFile(args.path),
+      loader: "binary",
+    }));
+  }
+};
+
+esbuild.build({
+  bundle: true,
+  entryPoints: [path.join(__dirname, "../src/index.ts")],
+  format: "cjs",
+  minify: true,
+  outdir: path.join(__dirname, "../dist"),
+  platform: "node",
+  plugins: [wasmPlugin],
+  sourcemap: false,
+  target: "es6"
+});

package.json

@@ -0,0 +1,28 @@
+{
+  "name": "node-syntax-tree",
+  "version": "0.1.0",
+  "description": "A node package for accessing Syntax Tree through WASM",
+  "main": "dist/index.js",
+  "scripts": {},
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ruby-syntax-tree/node-syntax-tree.git"
+  },
+  "author": "Kevin Newton",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ruby-syntax-tree/node-syntax-tree/issues"
+  },
+  "homepage": "https://github.com/ruby-syntax-tree/node-syntax-tree#readme",
+  "dependencies": {
+    "@wasmer/wasi": "^0.12.0",
+    "@wasmer/wasmfs": "^0.12.0",
+    "path-browserify": "^1.0.1",
+    "ruby-head-wasm-wasi": "^0.3.0"
+  },
+  "devDependencies": {
+    "@types/node": "^17.0.33",
+    "@types/path-browserify": "^1.0.0",
+    "esbuild": "^0.14.39"
+  }
+}

src/app.d.ts

@@ -0,0 +1,9 @@
+declare module "*.wasm" {
+  type Exports = {
+    _initialize: () => void;
+    memory: WebAssembly.Memory;
+  };
+
+  type Instance = WebAssembly.Instance & { exports: Exports };
+  export default function load(imports: any): Promise<Instance>;
+}

src/index.ts

@@ -0,0 +1,104 @@
+import crypto from "crypto";
+import { WASI } from "@wasmer/wasi";
+import { WasmFs } from "@wasmer/wasmfs";
+import path from "path-browserify";
+import { RubyVM } from "ruby-head-wasm-wasi/dist/index";
+
+import load from "./app.wasm";
+
+// This overwrites the default writeSync function used by the WasmFs to instead
+// pipe it out to the console.
+function createWriter(originalWriter: Function) {
+  return function () {
+    let text: string;
+
+    if (arguments.length === 4) {
+      text = arguments[1];
+    } else {
+      text = new TextDecoder("utf-8").decode(arguments[1]);
+    }
+
+    switch (arguments[0]) {
+      case 1:
+        console.log(text);
+        break;
+      case 2:
+        console.warn(text);
+        break;
+    }
+
+    return originalWriter.call(arguments);
+  }
+}
+
+export default async function createRuby() {
+  // First, create a new file system that we can use internally within the Ruby
+  // WASM VM.
+  const wasmFs = new WasmFs();
+  wasmFs.fs.mkdirSync("/tmp", 0o777);
+  wasmFs.fs.writeSync = createWriter(wasmFs.fs.writeSync.bind(wasmFs.fs));
+
+  // Next, create a new WASI instance with the correct options overridden from
+  // the defaults.
+  const wasi = new WASI({
+    bindings: {
+      ...WASI.defaultBindings,
+      fs: wasmFs.fs,
+      path: path,
+      randomFillSync: crypto.randomFillSync as typeof WASI.defaultBindings.randomFillSync,
+    },
+    preopens: { "/": "/tmp" }
+  });
+
+  // Then, create a new Ruby VM instance that we can use to store the memory for
+  // our application.
+  const rubyVM = new RubyVM();
+  const imports = { wasi_snapshot_preview1: wasi.wasiImport };
+  rubyVM.addToImports(imports);
+
+  // Set the WASI memory to use the memory for our application.
+  const instance = await load(imports);
+  wasi.setMemory(instance.exports.memory);
+
+  // Load our application into the virtual machine.
+  instance.exports._initialize();
+  await rubyVM.setInstance(instance);
+
+  // Initial our virtual machine and return it. It should now be able to
+  // evaluate and execute Ruby code.
+  rubyVM.initialize();
+
+  // Once our virtual machine is booted, we're going to require the necessary
+  // files to make it work. I'm not sure why I need to explicitly require
+  // did_you_mean here, but it doesn't work without it.
+  rubyVM.eval(`
+    require "did_you_mean"
+    require "json"
+
+    $:.unshift("/lib")
+    require_relative "/lib/syntax_tree"
+    require_relative "/lib/syntax_tree/haml"
+    # require_relative "/lib/syntax_tree/rbs"
+  `);
+
+  return {
+    rubyVM,
+    formatHAML(source: string) {
+      return format(source, ".haml");
+    },
+    formatRuby(source: string) {
+      return format(source, ".rb");
+    },
+    // formatRBS(source: string) {
+    //   return format(source, ".rbs");
+    // }
+  };
+
+  function format(source: string, kind: string) {
+    const jsonSource = JSON.stringify(JSON.stringify(source));
+    const rubySource = `SyntaxTree::HANDLERS.fetch("${kind}").format(JSON.parse(${jsonSource}))`;
+    return rubyVM.eval(rubySource).toString();
+  }
+};
+
+export type Ruby = Awaited<ReturnType<typeof createRuby>>;

tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "noImplicitAny": true,
+    "strict": true
+  },
+  "include": ["src"]
+}