Skip to content

Commit

Permalink
Encode dart2js source-map extensions in strings.
Browse files Browse the repository at this point in the history 
This reduce the memory footprint and cost of parsing these extensions.  Another
advantage is a reduction on the size of the .map files.  On a large customer
app, this was a 11% reduction, and the frames section was about 1Mb (excluding
the extra names and uris added on the existing tables).

The encoding works as follows:
 - minified names are written as a list of names and indices, in pairs.

    {'n1': 1, 'n2': 2} => 'n1,1,n2,2'

 - frames are encoded using a sequence of values with markers for the different
   kind of frames. Numbers are encoded using VLQ deltas. We use VLQ because it's
   already available to any parser that deals with the mappings).

This change also uses the dart2js_tools parser implementation for all unit tests.

Change-Id: Iacc2833c6517eb473955cc618adec501c610870f
Reviewed-on: https://dart-review.googlesource.com/c/82780
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Sigmund Cherem <sigmund@google.com>
  • Loading branch information
@sigmundch
sigmundch authored and commit-bot@chromium.org committed Nov 5, 2018
1 parent 0e35e41 commit 61df5fd
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 132 deletions.
46 changes: 21 additions & 25 deletions pkg/compiler/lib/src/io/source_map_builder.dart
View file
Expand Up @@ -211,48 +211,44 @@ class SourceMapBuilder {
void writeMinifiedNames(Map<String, String> minifiedNames,
IndexMap<String> nameMap, StringBuffer buffer) {
bool first = true;
buffer.write('{');
buffer.write('"');
minifiedNames.forEach((String minifiedName, String name) {
if (!first) buffer.write(',');
buffer.write('"');
writeJsonEscapedCharsOn(minifiedName, buffer);
buffer.write('"');
buffer.write(':');
// minifiedNames are valid JS identifiers so they don't need to be escaped
buffer.write(minifiedName);
buffer.write(',');
buffer.write(nameMap[name]);
first = false;
});
buffer.write('}');
buffer.write('"');
}

void writeFrames(
IndexMap<Uri> uriMap, IndexMap<String> nameMap, StringBuffer buffer) {
bool first = true;
buffer.write('[');
var offsetEncoder = DeltaEncoder();
var uriEncoder = DeltaEncoder();
var lineEncoder = DeltaEncoder();
var columnEncoder = DeltaEncoder();
var nameEncoder = DeltaEncoder();
buffer.write('"');
frames.forEach((int offset, List<FrameEntry> entries) {
if (!first) buffer.write(',');
buffer.write('[');
buffer.write(offset);
for (var entry in entries) {
buffer.write(',');
offsetEncoder.encode(buffer, offset);
if (entry.isPush) {
SourceLocation location = entry.pushLocation;
buffer.write('[');
buffer.write(uriMap[location.sourceUri]);
buffer.write(',');
buffer.write(location.line - 1);
buffer.write(',');
buffer.write(location.column - 1);
buffer.write(',');
buffer.write(nameMap[entry.inlinedMethodName]);
buffer.write(']');
uriEncoder.encode(buffer, uriMap[location.sourceUri]);
lineEncoder.encode(buffer, location.line - 1);
columnEncoder.encode(buffer, location.column - 1);
nameEncoder.encode(buffer, nameMap[entry.inlinedMethodName]);
} else {
buffer.write(entry.isEmptyPop ? 0 : -1);
// ; and , are not used by VLQ so we can distinguish them in the
// encoding, this is the same reason they are used in the mappings
// field.
buffer.write(entry.isEmptyPop ? ";" : ",");
}
}
buffer.write(']');
first = false;
});
buffer.write(']');
buffer.write('"');
}

/// Returns the source map tag to put at the end a .js file in [fileUri] to
Expand Down
105 changes: 67 additions & 38 deletions pkg/dart2js_tools/lib/src/dart2js_mapping.dart
View file
Expand Up @@ -9,6 +9,7 @@ import 'dart:convert';
import 'dart:io';

import 'package:source_maps/source_maps.dart';
import 'package:source_maps/src/vlq.dart';

import 'util.dart';

Expand All @@ -35,48 +36,16 @@ class Dart2jsMapping {
if (extensions == null) return;
var minifiedNames = extensions['minified_names'];
if (minifiedNames != null) {
minifiedNames['global'].forEach((minifiedName, id) {
globalNames[minifiedName] = sourceMap.names[id];
});
minifiedNames['instance'].forEach((minifiedName, id) {
instanceNames[minifiedName] = sourceMap.names[id];
});
_extractMinifedNames(minifiedNames['global'], sourceMap, globalNames);
_extractMinifedNames(minifiedNames['instance'], sourceMap, instanceNames);
}
List jsonFrames = extensions['frames'];
String jsonFrames = extensions['frames'];
if (jsonFrames != null) {
for (List values in jsonFrames) {
if (values.length < 2) {
warn("warning: incomplete frame data: $values");
continue;
}
int offset = values[0];
List<FrameEntry> entries = frames[offset] ??= [];
if (entries.length > 0) {
warn("warning: duplicate entries for $offset");
continue;
}
for (int i = 1; i < values.length; i++) {
var current = values[i];
if (current == -1) {
entries.add(new FrameEntry.pop(false));
} else if (current == 0) {
entries.add(new FrameEntry.pop(true));
} else {
if (current is List) {
if (current.length == 4) {
entries.add(new FrameEntry.push(sourceMap.urls[current[0]],
current[1], current[2], sourceMap.names[current[3]]));
} else {
warn("warning: unexpected entry $current");
}
} else {
warn("warning: unexpected entry $current");
}
}
}
}
new _FrameDecoder(jsonFrames).parseFrames(frames, sourceMap);
}
}

Dart2jsMapping.json(Map json) : this(parseJson(json), json);
}

class FrameEntry {
Expand Down Expand Up @@ -131,3 +100,63 @@ Dart2jsMapping parseMappingFor(Uri uri) {
var json = jsonDecode(sourcemapFile.readAsStringSync());
return new Dart2jsMapping(parseJson(json), json);
}

class _FrameDecoder implements Iterator<String> {
final String _internal;
final int _length;
int index = -1;
_FrameDecoder(this._internal) : _length = _internal.length;

// Iterator API is used by decodeVlq to consume VLQ entries.
bool moveNext() => ++index < _length;

String get current =>
(index >= 0 && index < _length) ? _internal[index] : null;

bool get hasTokens => index < _length - 1 && _length > 0;

int _readDelta() => decodeVlq(this);

void parseFrames(Map<int, List<FrameEntry>> frames, SingleMapping sourceMap) {
var offset = 0;
var uriId = 0;
var nameId = 0;
var line = 0;
var column = 0;
while (hasTokens) {
offset += _readDelta();
List<FrameEntry> entries = frames[offset] ??= [];
var marker = _internal[index + 1];
if (marker == ';') {
entries.add(new FrameEntry.pop(true));
index++;
continue;
} else if (marker == ',') {
entries.add(new FrameEntry.pop(false));
index++;
continue;
} else {
uriId += _readDelta();
var uri = sourceMap.urls[uriId];
line += _readDelta();
column += _readDelta();
nameId += _readDelta();
var name = sourceMap.names[nameId];
entries.add(new FrameEntry.push(uri, line, column, name));
}
}
}
}

_extractMinifedNames(String encodedInput, SingleMapping sourceMap,
Map<String, String> minifiedNames) {
List<String> input = encodedInput.split(',');
if (input.length % 2 != 0) {
warn("expected an even number of entries");
}
for (int i = 0; i < input.length; i += 2) {
String minifiedName = input[i];
int id = int.tryParse(input[i + 1]);
minifiedNames[minifiedName] = sourceMap.names[id];
}
}
61 changes: 2 additions & 59 deletions pkg/sourcemap_testing/lib/src/stacktrace_helper.dart
View file
Expand Up @@ -10,6 +10,7 @@ import 'package:expect/expect.dart';
import 'package:source_maps/source_maps.dart';
import 'package:source_maps/src/utils.dart';
import 'package:source_span/source_span.dart';
import 'package:dart2js_tools/src/dart2js_mapping.dart';

import 'annotated_code_helper.dart';

Expand Down Expand Up @@ -431,25 +432,6 @@ class LineException {
const LineException(this.methodName, this.fileName);
}

class FrameEntry {
final String callUri;
final int callLine;
final int callColumn;
final String inlinedMethodName;
final bool isEmpty;
FrameEntry.push(
this.callUri, this.callLine, this.callColumn, this.inlinedMethodName)
: isEmpty = false;
FrameEntry.pop(this.isEmpty)
: callUri = null,
callLine = null,
callColumn = null,
inlinedMethodName = null;

bool get isPush => callUri != null;
bool get isPop => callUri == null;
}

/// Search backwards in [sources] for a function declaration that includes the
/// [start] offset.
TargetEntry findEnclosingFunction(
Expand All @@ -466,44 +448,5 @@ TargetEntry findEnclosingFunction(
Map<int, List<FrameEntry>> _loadInlinedFrameData(
SingleMapping mapping, String sourceMapText) {
var json = jsonDecode(sourceMapText);
var frames = <int, List<FrameEntry>>{};
var extensions = json['x_org_dartlang_dart2js'];
if (extensions == null) return null;
List jsonFrames = extensions['frames'];
if (jsonFrames == null) return null;

for (List values in jsonFrames) {
if (values.length < 2) {
print("warning: incomplete frame data: $values");
continue;
}

int offset = values[0];
List<FrameEntry> entries = frames[offset] ??= [];
if (entries.length > 0) {
print("warning: duplicate entries for $offset");
continue;
}

for (int i = 1; i < values.length; i++) {
var current = values[i];
if (current == -1) {
entries.add(new FrameEntry.pop(false));
} else if (current == 0) {
entries.add(new FrameEntry.pop(true));
} else {
if (current is List) {
if (current.length == 4) {
entries.add(new FrameEntry.push(mapping.urls[current[0]],
current[1], current[2], mapping.names[current[3]]));
} else {
print("warning: unexpected entry $current");
}
} else {
print("warning: unexpected entry $current");
}
}
}
}
return frames;
return Dart2jsMapping(mapping, json).frames;
}
19 changes: 9 additions & 10 deletions tests/compiler/dart2js/sourcemaps/minified_names_test.dart
View file
Expand Up @@ -9,6 +9,7 @@ import 'dart:convert';
import 'package:args/args.dart';
import 'package:async_helper/async_helper.dart';
import 'package:compiler/src/commandline_options.dart';
import 'package:dart2js_tools/src/dart2js_mapping.dart';

import '../helpers/d8_helper.dart';
import 'package:expect/expect.dart';
Expand Down Expand Up @@ -115,25 +116,23 @@ checkExpectation(MinifiedNameTest test, bool minified) async {
var sourceMap = '${result.outputPath}.map';
var json = jsonDecode(await new File(sourceMap).readAsString());

var extensions = json['x_org_dartlang_dart2js'];
Expect.isNotNull(extensions, "Source-map doesn't contain dart2js extensions");
var minifiedNames = extensions['minified_names'];
Expect.isNotNull(minifiedNames, "Source-map doesn't contain minified-names");
var mapping = Dart2jsMapping.json(json);
Expect.isTrue(mapping.globalNames.isNotEmpty,
"Source-map doesn't contain minified-names");

var actualName;
if (test.isGlobal) {
var index = minifiedNames['global'][name];
Expect.isNotNull(index, "'$name' not in global name map");
actualName = json['names'][index];
actualName = mapping.globalNames[name];
Expect.isNotNull(actualName, "'$name' not in global name map");
} else if (test.isInstance) {
var index = minifiedNames['instance'][name];
// In non-minified mode some errors show the original name
// rather than the selector name (e.g. m1 instead of m1$0 in a
// NoSuchMethodError), and because of that `index` may be null.
// NoSuchMethodError), and because of that the name might not be on the
// table.
//
// TODO(sigmund): consider making all errors show the internal name, or
// include a marker to make it easier to distinguish.
actualName = index == null ? name : json['names'][index];
actualName = mapping.instanceNames[name] ?? name;
} else {
Expect.fail('unexpected');
}
Expand Down

0 comments on commit 61df5fd

Please sign in to comment.