Process source maps in Rust instead of JS (denoland#1280)

- Improves speed and binary size significantly.
- Makes deno_last_exception() output a JSON structure.
- Isolate::execute and Isolate::event_loop now return
  structured, mapped JSError objects on errors.
- Removes libdeno functions:
  libdeno.setGlobalErrorHandler()
  libdeno.setPromiseRejectHandler()
  libdeno.setPromiseErrorExaminer()

In collaboration with Ryan Dahl.

BUILD.gn

@@ -39,6 +39,8 @@ main_extern = [
   "$rust_build:remove_dir_all",
   "$rust_build:ring",
   "$rust_build:rustyline",
+  "$rust_build:serde_json",
+  "$rust_build:source_map_mappings",
   "$rust_build:tempfile",
   "$rust_build:tokio",
   "$rust_build:tokio_executor",
@@ -84,7 +86,6 @@ ts_sources = [
   "js/platform.ts",
   "js/plugins.d.ts",
   "js/process.ts",
-  "js/promise_util.ts",
   "js/read_dir.ts",
   "js/read_file.ts",
   "js/read_link.ts",
@@ -101,7 +102,6 @@ ts_sources = [
   "js/types.ts",
   "js/url_search_params.ts",
   "js/util.ts",
-  "js/v8_source_maps.ts",
   "js/write_file.ts",
   "tsconfig.json",
 

Cargo.toml

@@ -24,6 +24,8 @@ rand = "=0.6.1"
 remove_dir_all = "=0.5.1"
 rustyline = "=2.1.0"
 ring = "=0.13.5"
+serde_json = "1.0.33"
+source-map-mappings = "0.5.0"
 tempfile = "=3.0.5"
 tokio = "=0.1.13"
 tokio-executor = "=0.1.5"

build_extra/rust/BUILD.gn

@@ -1101,3 +1101,45 @@ rust_crate("tokio_process") {
     ]
   }
 }
+
+rust_crate("vlq") {
+  source_root = "$registry_github/vlq-0.5.1/src/lib.rs"
+}
+
+rust_crate("source_map_mappings") {
+  source_root = "$registry_github/source-map-mappings-0.5.0/src/lib.rs"
+  extern = [
+    ":rand",
+    ":vlq",
+  ]
+}
+
+rust_crate("ryu") {
+  source_root = "$registry_github/ryu-0.2.7/src/lib.rs"
+  features = [ "small" ]
+}
+
+rust_crate("serde") {
+  source_root = "$registry_github/serde-1.0.80/src/lib.rs"
+  features = [
+    "default",
+    "std",
+  ]
+}
+
+rust_crate("serde_json") {
+  source_root = "$registry_github/serde_json-1.0.33/src/lib.rs"
+  features = [
+    "arbitrary_precision",
+    "default",
+    "preserve_order",
+    "indexmap",
+    "raw_value",
+  ]
+  extern = [
+    ":indexmap",
+    ":itoa",
+    ":ryu",
+    ":serde",
+  ]
+}

js/compiler.ts

@@ -3,10 +3,8 @@ import * as ts from "typescript";
 import { MediaType } from "gen/msg_generated";
 
 import { assetSourceCode } from "./assets";
-import { libdeno } from "./libdeno";
 import * as os from "./os";
 import { CodeProvider } from "./runner";
-import { RawSourceMap } from "./types";
 import { assert, log, notImplemented } from "./util";
 
 const EOL = "\n";
@@ -71,7 +69,7 @@ export class ModuleMetaData implements ts.IScriptSnapshot {
     public readonly mediaType: MediaType,
     public readonly sourceCode: SourceCode = "",
     public outputCode: OutputCode = "",
-    public sourceMap: SourceMap | RawSourceMap = ""
+    public sourceMap: SourceMap = ""
   ) {
     if (outputCode !== "" || fileName.endsWith(".d.ts")) {
       this.scriptVersion = "1";
@@ -131,8 +129,6 @@ export class Compiler
     ContainingFile,
     Map<ModuleSpecifier, ModuleFileName>
   >();
-  // Keep track of state of the last module requested via `getGeneratedContents`
-  private _lastModule: ModuleMetaData | undefined;
   // A reference to the log utility, so it can be monkey patched during testing
   private _log = log;
   // A map of module file names to module meta data
@@ -369,19 +365,15 @@ export class Compiler
       moduleMetaData.outputCode = `${
         outputFile.text
       }\n//# sourceURL=${fileName}`;
-      moduleMetaData.sourceMap = JSON.parse(sourceMapFile.text);
+      moduleMetaData.sourceMap = sourceMapFile.text;
     }
 
     moduleMetaData.scriptVersion = "1";
-    const sourceMap =
-      moduleMetaData.sourceMap === "string"
-        ? moduleMetaData.sourceMap
-        : JSON.stringify(moduleMetaData.sourceMap);
     this._os.codeCache(
       fileName,
       sourceCode,
       moduleMetaData.outputCode,
-      sourceMap
+      moduleMetaData.sourceMap
     );
     return moduleMetaData.outputCode;
   }
@@ -398,36 +390,10 @@ export class Compiler
   }
 
   /** Given a fileName, return what was generated by the compiler. */
-  getGeneratedContents = (fileName: string): string | RawSourceMap => {
-    this._log("compiler.getGeneratedContents", fileName);
-    if (fileName === "gen/bundle/main.js") {
-      assert(libdeno.mainSource.length > 0);
-      return libdeno.mainSource;
-    } else if (fileName === "main.js.map") {
-      return libdeno.mainSourceMap;
-    } else if (fileName === "deno_main.js") {
-      return "";
-    } else if (!fileName.endsWith(".map")) {
-      const moduleMetaData = this._moduleMetaDataMap.get(fileName);
-      if (!moduleMetaData) {
-        this._lastModule = undefined;
-        return "";
-      }
-      this._lastModule = moduleMetaData;
-      return moduleMetaData.outputCode;
-    } else {
-      if (this._lastModule && this._lastModule.sourceMap) {
-        // Assuming the the map will always be asked for after the source
-        // code.
-        const { sourceMap } = this._lastModule;
-        this._lastModule = undefined;
-        return sourceMap;
-      } else {
-        // Errors thrown here are caught by source-map.
-        throw new Error(`Unable to find source map: "${fileName}"`);
-      }
-    }
-  };
+  getGeneratedSourceMap(fileName: string): string {
+    const moduleMetaData = this._moduleMetaDataMap.get(fileName);
+    return moduleMetaData ? moduleMetaData.sourceMap : "";
+  }
 
   /** Get the output code for a module based on its filename. A call to
    * `.getFilename()` should occur before attempting to get the output code as

js/libdeno.ts

@@ -1,5 +1,4 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
-import { RawSourceMap } from "./types";
 import { globalEval } from "./global_eval";
 
 // The libdeno functions are moved so that users can't access them.
@@ -39,9 +38,6 @@ interface Libdeno {
   ) => void;
 
   setPromiseErrorExaminer: (handler: () => boolean) => void;
-
-  mainSource: string;
-  mainSourceMap: RawSourceMap;
 }
 
 const window = globalEval("this");

js/main.ts

@@ -11,22 +11,9 @@ import { Runner } from "./runner";
 import { libdeno } from "./libdeno";
 import { args } from "./deno";
 import { sendSync, handleAsyncMsgFromRust } from "./dispatch";
-import { promiseErrorExaminer, promiseRejectHandler } from "./promise_util";
 import { replLoop } from "./repl";
-import * as sourceMaps from "./v8_source_maps";
 import { version } from "typescript";
 
-// Install the source maps handler and do some pre-calculations so all of it is
-// available in the snapshot
-const compiler = Compiler.instance();
-sourceMaps.install({
-  installPrepareStackTrace: true,
-  getGeneratedContents: compiler.getGeneratedContents
-});
-const consumer = sourceMaps.loadConsumer("gen/bundle/main.js");
-assert(consumer != null);
-consumer!.computeColumnSpans();
-
 function sendStart(): msg.StartRes {
   const builder = flatbuffers.createBuilder();
   msg.Start.startStart(builder);
@@ -39,27 +26,9 @@ function sendStart(): msg.StartRes {
   return startRes;
 }
 
-function onGlobalError(
-  message: string,
-  source: string,
-  lineno: number,
-  colno: number,
-  error: any // tslint:disable-line:no-any
-) {
-  if (error instanceof Error) {
-    console.log(error.stack);
-  } else {
-    console.log(`Thrown: ${String(error)}`);
-  }
-  os.exit(1);
-}
-
 /* tslint:disable-next-line:no-default-export */
 export default function denoMain() {
   libdeno.recv(handleAsyncMsgFromRust);
-  libdeno.setGlobalErrorHandler(onGlobalError);
-  libdeno.setPromiseRejectHandler(promiseRejectHandler);
-  libdeno.setPromiseErrorExaminer(promiseErrorExaminer);
 
   // First we send an empty "Start" message to let the privileged side know we
   // are ready. The response should be a "StartRes" message containing the CLI
@@ -68,6 +37,8 @@ export default function denoMain() {
 
   setLogDebug(startResMsg.debugFlag());
 
+  const compiler = Compiler.instance();
+
   // handle `--types`
   if (startResMsg.typesFlag()) {
     const defaultLibFileName = compiler.getDefaultLibFileName();

js/unit_tests.ts

@@ -35,5 +35,4 @@ import "./text_encoding_test.ts";
 import "./timers_test.ts";
 import "./truncate_test.ts";
 import "./url_search_params_test.ts";
-import "./v8_source_maps_test.ts";
 import "./write_file_test.ts";