Add simple source map creator for preprocess_if_expr.py

Add a simple wrapper around mozilla/source-map which takes a ts or js
file with lines-removal comments from preprocess_if_expr.py and turns
them into source maps.

This is useful for at least two projects:
  1. code coverage: We want to be able to display code coverage for
     tests, and right now the code coverage doesn't include JavaScript
  2. WebUI JavaScript Error Reporting: We want to map the stacks we get
     in error reports back to the original line & column numbers.

BUG=b:186885186,chromium:1303956

Change-Id: I31cdac606efed7c7f463b7266ad7bdc567dcc0dc
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2950922
Reviewed-by: Ben Reich <benreich@chromium.org>
Reviewed-by: Yuke Liao <liaoyuke@chromium.org>
Reviewed-by: Demetrios Papadopoulos <dpapad@chromium.org>
Reviewed-by: Rebekah Potter <rbpotter@chromium.org>
Commit-Queue: Ian Barkley-Yeung <iby@chromium.org>
Cr-Commit-Position: refs/heads/main@{#984527}

tools/code_coverage/create_js_source_maps/OWNERS

@@ -0,0 +1,3 @@
+benreich@chromium.org
+iby@chromium.org
+tiborg@chromium.org

tools/code_coverage/create_js_source_maps/PRESUBMIT.py

@@ -0,0 +1,38 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""create_js_source_maps presubmit script.
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for
+details on the presubmit API built into gcl.
+"""
+USE_PYTHON3 = True
+PRESUBMIT_VERSION = '2.0.0'
+
+
+def CheckLint(input_api, output_api):
+  results = input_api.canned_checks.RunPylint(input_api, output_api)
+  results += input_api.canned_checks.CheckPatchFormatted(input_api,
+                                                         output_api,
+                                                         check_js=True)
+  try:
+    import sys
+    old_sys_path = sys.path[:]
+    cwd = input_api.PresubmitLocalPath()
+    sys.path += [input_api.os_path.join(cwd, '..', '..')]
+    from web_dev_style import presubmit_support
+    results += presubmit_support.CheckStyleESLint(input_api, output_api)
+  finally:
+    sys.path = old_sys_path
+  return results
+
+
+def CheckUnittests(input_api, output_api):
+  results = input_api.canned_checks.RunUnitTests(
+      input_api,
+      output_api, [
+          input_api.os_path.join(input_api.PresubmitLocalPath(), 'test',
+                                 'create_js_source_maps_test.py')
+      ],
+      run_on_python2=False)
+  return results

tools/code_coverage/create_js_source_maps/create_js_source_maps.gni

@@ -0,0 +1,30 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# NOTE: The "create_js_source_maps" build rule must come after the
+# "preprocess_if_expr" build rule(s) in the BUILD.gn file. If you are getting
+# "Target not found in this context" errors, check that the deps
+# names are correct and that they are defined earlier in the same BUILD.gn file.
+template("create_js_source_maps") {
+  action_foreach(target_name) {
+    script =
+        "//tools/code_coverage/create_js_source_maps/create_js_source_maps.py"
+    args = []
+    inputs = [
+      "//tools/code_coverage/create_js_source_maps/create_js_source_maps.js",
+    ]
+    sources = []
+    deps = invoker.deps
+    foreach(dependency, deps) {
+      foreach(preprocess_output, get_target_outputs(dependency)) {
+        if (get_path_info(preprocess_output, "extension") == "ts" ||
+            get_path_info(preprocess_output, "extension") == "js") {
+          sources += [ preprocess_output ]
+        }
+      }
+    }
+    outputs = [ "$target_gen_dir/{{source}}.map" ]
+    args = [ "{{source}}" ]
+  }
+}

tools/code_coverage/create_js_source_maps/create_js_source_maps.js

@@ -0,0 +1,109 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * @fileoverview A simple wrapper around mozilla/source-map. Scans a file
+ * processed by preprocess_if_expr.py looking for erasure comments. It creates
+ * a sourcemap mapping the post-processed TypeScript or JavaScript back to the
+ * original TypeScript or JavaScript.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import {ArgumentParser} from '../../../third_party/js_code_coverage/node_modules/argparse/argparse.js';
+import {SourceMapGenerator} from '../../../third_party/js_code_coverage/node_modules/source-map/source-map.js';
+
+// Regex matching the comment indicating that preprocess_if_expr removed lines.
+// The capture group contains the number of lines removed. Must match the
+// comment added by tools/grit/grit/format/html_inline.py
+const GRIT_REMOVED_LINES_REGEX = /grit-removed-lines:(\d+)/g;
+
+/**
+ * Adds a mapping for a line. We only map lines, not columns -- we don't have
+ * enough information to map columns within a line. (And the usual usage of
+ * preprocess_if_expr means we don't expect to see partial line removals with
+ * code after the removal.)
+ *
+ * @param {SourceMapGenerator} map The SourceMapGenerator.
+ * @param {string} sourceFileName The name of the original file.
+ * @param {number} originalLine The current line in the original source file.
+ * @param {number} generatedLine The current line in the generated (processed)
+ *                               source file.
+ * @param {boolean} verbose If true, print detailed information about the
+ *                          mappings as they are added.
+ */
+function addMapping(map, sourceFileName, originalLine, generatedLine, verbose) {
+  const mapping = {
+    source: sourceFileName,
+    original: {
+      line: originalLine,
+      column: 0,
+    },
+    generated: {
+      line: generatedLine,
+      column: 0,
+    },
+  };
+  if (verbose) {
+    console.log(mapping);
+  }
+  map.addMapping(mapping);
+}
+
+/**
+ * Processes one processed TypeScript or JavaScript file and produces one
+ * source map file.
+ *
+ * @param {string} inputFileName The TypeScript or JavaScript file to read from.
+ * @param {boolean} verbose If true, print detailed information about the
+ *                          mappings as they are added.
+ */
+function processOneFile(inputFileName, verbose) {
+  const inputFile = fs.readFileSync(inputFileName, 'utf8');
+  const inputLines = inputFile.split('\n');
+  const inputFileBaseName = path.basename(inputFileName);
+  const map = new SourceMapGenerator();
+
+  let originalLine = 0;
+  let generatedLine = 0;
+
+  for (const line of inputLines) {
+    generatedLine++;
+    originalLine++;
+
+    // Add to sourcemap before looking for removal comments. The beginning of
+    // the generated line came from the parts before the removal comment.
+    addMapping(map, inputFileBaseName, originalLine, generatedLine, verbose);
+
+    for (const removal of line.matchAll(GRIT_REMOVED_LINES_REGEX)) {
+      const removedLines = Number.parseInt(removal[1], 10);
+      if (verbose) {
+        console.log(`Found grit-removed-lines:${removedLines} on line ${
+            generatedLine}`);
+      }
+      originalLine += removedLines;
+    }
+  }
+
+  fs.writeFileSync(inputFileName + '.map', map.toString());
+}
+
+function main() {
+  const parser = new ArgumentParser({
+    description:
+        'Creates source maps for files preprocessed by preprocess_if_expr'
+  });
+
+  parser.addArgument(
+      ['-v', '--verbose'],
+      {help: 'Print each mapping & removed-line comment', action: 'storeTrue'});
+  parser.addArgument('input', {help: 'Input file name', action: 'store'});
+
+  const argv = parser.parseArgs();
+
+  processOneFile(argv.input, argv.verbose);
+}
+
+main();

tools/code_coverage/create_js_source_maps/create_js_source_maps.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env vpython3
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+from pathlib import Path
+
+_HERE_DIR = Path(__file__).parent
+_SOURCE_MAP_CREATOR = (_HERE_DIR / 'create_js_source_maps.js').resolve()
+
+_NODE_PATH = (_HERE_DIR.parent.parent.parent / 'third_party' / 'node').resolve()
+sys.path.append(str(_NODE_PATH))
+import node
+
+# Invokes "node create_js_source_maps.js (args)""
+# We can't use third_party/node/node.py directly from the gni template because
+# we don't have a good way to specify the path to create_js_source_maps.js in a
+# gni template.
+node.RunNode([str(_SOURCE_MAP_CREATOR)] + sys.argv[1:])

tools/code_coverage/create_js_source_maps/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "create_js_source_maps",
+  "version": "0.1",
+  "description": "Create JavaScript source maps after preprocess_if_expr",
+  "main": "create_js_source_maps.js",
+  "files": [ "create_js_source_maps.js" ],
+  "license": "SEE LICENSE IN ../../../LICENSE",
+  "type" : "module"
+}

tools/code_coverage/create_js_source_maps/test/create_js_source_maps_test.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env vpython3
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from pathlib import Path
+
+_HERE_DIR = Path(__file__).parent.resolve()
+_SOURCE_MAP_PROCESSOR = (_HERE_DIR.parent /
+                         'create_js_source_maps.js').resolve()
+_SOURCE_MAP_TRANSLATOR = (_HERE_DIR / 'translate_source_map.js').resolve()
+
+_NODE_PATH = (_HERE_DIR.parent.parent.parent.parent / 'third_party' /
+              'node').resolve()
+sys.path.append(str(_NODE_PATH))
+import node
+
+
+class CreateSourceMapsTest(unittest.TestCase):
+  def setUp(self):
+    self._out_folder = None
+
+  def tearDown(self):
+    if self._out_folder:
+      shutil.rmtree(self._out_folder)
+
+  def _translate(self, source_map, line, column):
+    """ Translates from post-transform to pre-transform using a source map.
+
+    Translates a line and column in some hypothetical processed JavaScript
+    back into the hypothetical original line and column using the indicated
+    source map. Returns the pre-processed line and column.
+    """
+    stdout = node.RunNode([
+        str(_SOURCE_MAP_TRANSLATOR), "--source_map", source_map, "--line",
+        str(line), "--column",
+        str(column)
+    ])
+    result = json.loads(stdout)
+    assert isinstance(result['line'], int)
+    assert isinstance(result['column'], int)
+    return result['line'], result['column']
+
+  def testPostProcessedFile(self):
+    ''' Test that a known starting file translates back correctly
+
+    Assume we start with the following file:
+    Line 1
+    // <if expr="foo"> Line 2
+    Line 3 deleted
+    // Line 4 </if>
+    Line 5
+    // <if expr="bar"> Line 6
+    Line 7 deleted
+    Line 8 deleted
+    // Line 9 </if>
+    Line 10
+    Line 11
+
+    Make sure we can map the various non-deleted lines back to their correct
+    locations.
+    '''
+    assert not self._out_folder
+    self._out_folder = tempfile.mkdtemp(dir=_HERE_DIR)
+
+    file_after_preprocess = b'''Line 1
+      // /*grit-removed-lines:2*/
+      Line 5
+      // /*grit-removed-lines:3*/
+      Line 10
+      Line 11
+      '''
+    input_fd, input_file_name = tempfile.mkstemp(dir=self._out_folder,
+                                                 text=True,
+                                                 suffix=".js")
+    os.write(input_fd, file_after_preprocess)
+    os.close(input_fd)
+    node.RunNode([str(_SOURCE_MAP_PROCESSOR), input_file_name])
+    map_path = input_file_name + ".map"
+
+    # Check mappings:
+    # Line 1 is before any removed lines, so it still maps to line 1
+    line, column = self._translate(map_path, 1, 2)
+    self.assertEqual(line, 1)
+    # Column number always snaps back to the column number of the most recent
+    # mapping point, so it's zero not the correct column number. This seems to
+    # be a limitation of the sourcemap format.
+    self.assertEqual(column, 0)
+
+    # Original line 5 ends up on translated line 3
+    line, column = self._translate(map_path, 3, 2)
+    self.assertEqual(line, 5)
+    self.assertEqual(column, 0)
+
+    # Original line 10 ends up on line 5
+    line, column = self._translate(map_path, 5, 2)
+    self.assertEqual(line, 10)
+    self.assertEqual(column, 0)
+
+    # Original line 11 ends up on line 6
+    line, column = self._translate(map_path, 6, 2)
+    self.assertEqual(line, 11)
+    self.assertEqual(column, 0)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/code_coverage/create_js_source_maps/test/translate_source_map.js

@@ -0,0 +1,47 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * @fileoverview A simple wrapper around mozilla/source-map. Takes a source map
+ * and a location in the post-processed file and prints the corresponding
+ * location in the pre-processing file.
+ *
+ * Helper for create_js_source_maps_test.py.
+ */
+import fs from 'fs';
+
+import {SourceMapConsumer} from '../../../../third_party/js_code_coverage/node_modules/source-map/source-map.js';
+// TODO(crbug.com/1307980): Move argparse to the js_code_coverage library.
+import {ArgumentParser} from '../../../../third_party/node/node_modules/argparse/index.js';
+
+const parser = new ArgumentParser({
+  description: 'Applies a JavaScript sourcemap to a line and column number'
+});
+
+parser.addArgument(
+    '--source_map',
+    {help: 'Source map to use for translation', required: true});
+parser.addArgument(
+    '--line',
+    {help: 'Line number in post-processed file', type: 'int', required: true});
+parser.addArgument('--column', {
+  help: 'Column number in post-processed file',
+  type: 'int',
+  required: true
+});
+
+const argv = parser.parseArgs();
+
+
+const sourceMap = JSON.parse(fs.readFileSync(argv.source_map));
+// Async function to get around "Cannot use keyword 'await' outside an async
+// function" complaint in ESLint. Our version of node would allow us to use
+// 'await' at the top level, but our version of ESLint fails.
+(async function() {
+  const consumer = await new SourceMapConsumer(sourceMap);
+  const result =
+      consumer.originalPositionFor({line: argv.line, column: argv.column});
+  console.log(JSON.stringify(result));
+  consumer.destroy();
+}());