[js] Support source maps when writing code to file

Support reading them back in
Support #line 67 foo in Perl 6 code.

Author: pmurias

src/vm/js/Chunk.nqp

@@ -68,7 +68,8 @@ class Chunk {
     }
 
     method collect_with_source_map_info($offset, @strs, @mapping) {
-        if nqp::defined($!node) && $!node.node {
+        # HACK sometimes a QAST::Op sneaks into $!node.node, so call nqp::can
+        if nqp::defined($!node) && $!node.node && nqp::can($!node.node, 'from') {
             nqp::push_i(@mapping, $!node.node.from());
             nqp::push_i(@mapping, $offset);
         }

src/vm/js/HLL/Backend.nqp

@@ -73,10 +73,12 @@ my class JSWithSourceMap {
     has $!mapping;
     has $!p6-source;
     has $!file;
+    has $!comp_line_directives;
     method js() {$!js}
     method mapping() {$!mapping}
     method p6-source() {$!p6-source}
     method file() {$!file}
+    method comp_line_directives() {$!comp_line_directives}
 }
 
 class QASTWithMatch {
@@ -221,14 +223,35 @@ class JavaScriptBackend {
             my $file := nqp::ifnull(nqp::getlexdyn('$?FILES'), "<unknown file>");
             $backend.emit_with_source_map($parsed.ast, @js, @mapping, :$instant, :$shebang, :$nqp-runtime);
 
-            JSWithSourceMap.new(js => nqp::join('', @js), mapping => @mapping, p6-source => $parsed.match.orig, file => $file);
+            my $sourcemap_and_js := JSWithSourceMap.new(js => nqp::join('', @js), mapping => @mapping, p6-source => $parsed.match.orig, file => $file, comp_line_directives => @*comp_line_directives);
+
+            %adverbs<output> ?? self.sourcemap_to_file($sourcemap_and_js, %adverbs<output>) !! $sourcemap_and_js;
         } else {
             my $code := $backend.emit(self.get_ast($parsed), :$instant, :$substagestats, :$shebang, :$nqp-runtime);
             $code := self.beautify($code) if %adverbs<beautify>;
             $code;
         }
     }
 
+    method sourcemap_to_file($sourcemap_and_js, $output) {
+        my str $sourcemap_file := "$output.map";
+
+        my $sourcemap := nqp::getcomp('JavaScript').eval('nqp.buildSourceMap')(
+            $sourcemap_and_js.js,
+            $sourcemap_and_js.p6-source,
+            $sourcemap_and_js.mapping,
+            $output,
+            $sourcemap_and_js.file,
+            $sourcemap_and_js.comp_line_directives,
+            $sourcemap_file
+        );
+
+        spurt($sourcemap_file, $sourcemap<content>);
+
+
+        $sourcemap_and_js.js ~ "//# sourceMappingURL={$sourcemap<url>}";
+    }
+
     method beautify($code) {
         my $tmp_file := self.tmp_file();
 
@@ -252,11 +275,10 @@ class JavaScriptBackend {
     }
 
     method run($js, *%adverbs) {
-        # TODO source map support
 
         if !self.spawn_new_node {
             if nqp::istype($js, JSWithSourceMap) {
-                return nqp::getcomp('JavaScript').eval($js.js, :mapping($js.mapping), :p6-source($js.p6-source), :file($js.file));
+                return nqp::getcomp('JavaScript').eval($js.js, :mapping($js.mapping), :p6-source($js.p6-source), :file($js.file), :comp_line_directives($js.comp_line_directives));
             } else {
                 return nqp::getcomp('JavaScript').eval($js);
             }
@@ -269,6 +291,7 @@ class JavaScriptBackend {
         close($code);
 
         sub (*@args) {
+            # TODO source map support
             my @cmd := ["node",$tmp_file];
 
             my $i := 1;

src/vm/js/nqp-runtime/core.js

@@ -57,6 +57,10 @@ const unicodeCollationAlgorithm = require('unicode-collation-algorithm');
 
 const unicodeData = require('nqp-unicode-data');
 
+const resolveSourceMap = require('./resolve-sourcemap.js');
+
+const path = require('path');
+
 exports.CodeRef = CodeRef;
 
 op.isinvokable = function(obj) {
@@ -646,30 +650,74 @@ const SourceNode = require('source-map').SourceNode;
 
 const charProps = require('char-props');
 
-function createSourceMap(js, p6, mapping, jsFile, p6File) {
+function applyLineDirectives(file, line, directives) {
+  if (directives) {
+    for (let i = directives.length - 1; i >= 0; i--) {
+      if (line > directives[i].originalLine) {
+        return {
+          line: line - directives[i].originalLine + directives[i].line - 1,
+          file: directives[i].file
+        };
+      }
+    }
+  }
+  return {file: file, line: line};
+}
+
+function processCompLineDirectives(ctx, compLineDirectives) {
+  if (compLineDirectives === undefined) return undefined;
+  return compLineDirectives.array.map(directive => ({
+    originalLine: nqp.toInt(directive.array[0], ctx),
+    file: nqp.toStr(directive.array[2], ctx),
+    line: nqp.toInt(directive.array[1], ctx)
+  }));
+}
+
+function createSourceMap(js, p6, mapping, jsFile, p6File, lineDirectives) {
   const generator = new SourceMapGenerator({file: jsFile});
 
   const jsProps = charProps(js);
   const p6Props = charProps(p6);
 
-
   for (let i=0; i < mapping.length; i += 2) {
+    const {file: generatedFile, line: generatedLine} = applyLineDirectives(p6File, p6Props.lineAt(mapping[i])+1, lineDirectives);
     generator.addMapping({
       generated: {
         line: jsProps.lineAt(mapping[i+1])+1,
         column: jsProps.columnAt(mapping[i+1])+1,
       },
       original: {
-        line: p6Props.lineAt(mapping[i])+1,
+        line: generatedLine,
         column: p6Props.columnAt(mapping[i])+1,
       },
-      source: p6File,
+      source: generatedFile,
     });
   }
 
-  return new SourceMapConsumer(generator.toString());
+  return generator.toString();
 }
 
+
+class BuildSourceMap extends NQPObject {
+  $$call(ctx, NAMED, js, p6, mapping, jsFile, p6File, compLineDirectives, sourcemapFile) {
+    js = nqp.arg_s(ctx, js);
+    p6 = nqp.arg_s(ctx, p6);
+    jsFile = nqp.arg_s(ctx, jsFile);
+    mapping = mapping.array;
+    p6File = nqp.arg_s(ctx, p6File)
+    compLineDirectives = processCompLineDirectives(ctx, compLineDirectives);
+    sourcemapFile = nqp.arg_s(ctx, sourcemapFile);
+
+    const ret = new Hash();
+    ret.content.set('content', new NQPStr(createSourceMap(js, p6, mapping, path.relative(path.dirname(sourcemapFile), jsFile), p6File, compLineDirectives)));
+    ret.content.set('url', new NQPStr(path.relative(path.dirname(jsFile), sourcemapFile)));
+    return ret;
+  }
+}
+
+exports.buildSourceMap = new BuildSourceMap();
+
+
 class JavaScriptCompiler extends NQPObject {
   eval(ctx, _NAMED, self, code) {
     if (!(_NAMED !== null && _NAMED.hasOwnProperty('mapping'))) {
@@ -686,7 +734,15 @@ class JavaScriptCompiler extends NQPObject {
     global.nqp = nqp;
 
     if (_NAMED !== null && _NAMED.hasOwnProperty('mapping')) {
-      sourceMaps[fakeFilename] = createSourceMap(codeStr, nqp.toStr(_NAMED['p6-source'], ctx), _NAMED.mapping.array, fakeFilename, nqp.toStr(_NAMED.file, ctx));
+      sourceMaps[fakeFilename] = new SourceMapConsumer(
+        createSourceMap(
+          codeStr,
+          nqp.toStr(_NAMED['p6-source'], ctx),
+          _NAMED.mapping.array,
+          fakeFilename,
+          nqp.toStr(_NAMED.file, ctx),
+          processCompLineDirectives(ctx, _NAMED.comp_line_directives)));
+
       const node = SourceNode.fromStringWithSourceMap(codeStr, sourceMaps[fakeFilename]);
 
       // HACK
@@ -1366,6 +1422,18 @@ function backtrace(exception) {
                   line = original.line;
                   column = original.column;
                 }
+              } else {
+                if (file) {
+                  const resolved = resolveSourceMap(file);
+                  if (resolved !== null) {
+                    const original = resolved.originalPositionFor({line: line, column: column});
+                    if (original.source) {
+                      file = original.source;
+                      line = original.line;
+                      column = original.column;
+                    }
+                  }
+                }
               }
               if (file === undefined) file = '?';
               break;

src/vm/js/nqp-runtime/package.json

@@ -30,6 +30,7 @@
     "shortid": "2.2.8",
     "sleep": "^5.1.1",
     "source-map": "0.5.7",
+    "source-map-resolve": "^0.5.1",
     "stack-trace": "0.0.10",
     "unicharadata": "*",
     "unicode-collation-algorithm": "0.0.2",

src/vm/js/nqp-runtime/resolve-sourcemap.js

@@ -1,3 +1,13 @@
-
 const fs = require('fs');
 const sourceMapResolve = require('source-map-resolve');
+
+const SourceMapConsumer = require('source-map').SourceMapConsumer;
+
+const cache = new Map();
+module.exports = function(filename) {
+  if (cache.get(filename) !== undefined) return cache.get(filename);
+  const got = sourceMapResolve.resolveSourceMapSync(fs.readFileSync(filename).toString('utf8'), filename,  fs.readFileSync);
+  const ret = got !== null ? new SourceMapConsumer(got.map) : got;
+  cache.set(filename, ret);
+  return ret;
+};

src/vm/js/nqp-runtime/runtime.js

@@ -672,3 +672,5 @@ const props = require('./unicode-props.js');
 for (const prop in props) {
   exports[prop] = props[prop];
 }
+
+exports.buildSourceMap = core.buildSourceMap;