Skip to content

Commit

Permalink
Updates frontend_server javascript bundler to be based on import uri …
Browse files Browse the repository at this point in the history 
…instead of file uri.

Converts `package:` import uris into `/packages/` modules.

Also renames the output modules to append `.lib.js` instead of just `.js`. This allows us to distinguish between modules and applications based on extension.

Updates DDC source map code to be able to convert absolute file uris in sources so that they are relative to the source map.

Bug: dart-lang/webdev#865
Change-Id: I55d70aa3761f10cc8bd7e92f5b567478040660de
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/132300
Reviewed-by: Nicholas Shahan <nshahan@google.com>
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Reviewed-by: Jonah Williams <jonahwilliams@google.com>
Commit-Queue: Jake Macdonald <jakemac@google.com>
  • Loading branch information
@jakemac53
jakemac53 authored and commit-bot@chromium.org committed Jan 21, 2020
1 parent da1b47c commit 4460f95
Show file tree
Hide file tree
Showing 8 changed files with 203 additions and 111 deletions.
39 changes: 31 additions & 8 deletions pkg/dev_compiler/lib/src/compiler/shared_command.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,18 +305,32 @@ Uri sourcePathToRelativeUri(String source, {bool windows}) {
return uri;
}

/// Adjusts the source paths in [sourceMap] to be relative to [sourceMapPath],
/// and returns the new map. Relative paths are in terms of URIs ('/'), not
/// local OS paths (e.g., windows '\'). Sources with a multi-root scheme
/// matching [multiRootScheme] are adjusted to be relative to
/// [multiRootOutputPath].
/// Adjusts the source paths in [sourceMap] to be relative paths,and returns
/// the new map.
///
/// Relative paths are in terms of URIs ('/'), not local OS paths (e.g.,
/// windows '\').
///
/// Source uris show up in two forms, absolute `file:` uris and custom
/// [multiRootScheme] uris (also "absolute" uris, but always relative to some
/// multi-root).
///
/// - `file:` uris are converted to be relative to [sourceMapBase], which
/// defaults to the dirname of [sourceMapPath] if not provided.
///
/// - [multiRootScheme] uris are prefixed by [multiRootOutputPath]. If the
/// path starts with `/lib`, then we strip that before making it relative
/// to the [multiRootOutputPath], and assert that [multiRootOutputPath]
/// starts with `/packages` (more explanation inline).
///
// TODO(jmesserly): find a new home for this.
Map placeSourceMap(Map sourceMap, String sourceMapPath, String multiRootScheme,
{String multiRootOutputPath}) {
{String multiRootOutputPath, String sourceMapBase}) {
var map = Map.from(sourceMap);
// Convert to a local file path if it's not.
sourceMapPath = sourcePathToUri(p.absolute(p.fromUri(sourceMapPath))).path;
var sourceMapDir = p.url.dirname(sourceMapPath);
sourceMapBase ??= sourceMapDir;
var list = (map['sources'] as List).toList();

String makeRelative(String sourcePath) {
Expand All @@ -330,8 +344,17 @@ Map placeSourceMap(Map sourceMap, String sourceMapPath, String multiRootScheme,
var shortPath = uri.path
.replaceAll('/sdk/', '/dart-sdk/')
.replaceAll('/sdk_nnbd/', '/dart-sdk/');
// A multi-root uri starting with a path under `/lib` indicates that
// the multi-root is at the root of a package (typically, the
// application root package). These should be converted into a
// `/packages` path, we do that by stripping the `/lib` prefix and
// relying on the `multiRootOutputPath` to be set to the proper
// packages dir (so /packages/<package>).
if (shortPath.startsWith('/lib')) {
assert(multiRootOutputPath.startsWith('/packages'));
shortPath = shortPath.substring(4);
}
var multiRootPath = "${multiRootOutputPath ?? ''}$shortPath";
multiRootPath = multiRootPath;
multiRootPath = p.url.relative(multiRootPath, from: sourceMapDir);
return multiRootPath;
}
Expand All @@ -342,7 +365,7 @@ Map placeSourceMap(Map sourceMap, String sourceMapPath, String multiRootScheme,
sourcePath = sourcePathToUri(p.absolute(p.fromUri(uri))).path;

// Fall back to a relative path against the source map itself.
sourcePath = p.url.relative(sourcePath, from: sourceMapDir);
sourcePath = p.url.relative(sourcePath, from: sourceMapBase);

// Convert from relative local path to relative URI.
return p.toUri(sourcePath).path;
Expand Down
7 changes: 6 additions & 1 deletion pkg/dev_compiler/lib/src/kernel/command.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,11 +536,16 @@ class JSCode {
JSCode(this.code, this.sourceMap);
}

/// Converts [moduleTree] to [JSCode], using [format].
///
/// See [placeSourceMap] for a description of [sourceMapBase], [customScheme],
/// and [multiRootOutputPath] arguments.
JSCode jsProgramToCode(js_ast.Program moduleTree, ModuleFormat format,
{bool buildSourceMap = false,
bool inlineSourceMap = false,
String jsUrl,
String mapUrl,
String sourceMapBase,
String customScheme,
String multiRootOutputPath}) {
var opts = js_ast.JavaScriptPrintingOptions(
Expand All @@ -562,7 +567,7 @@ JSCode jsProgramToCode(js_ast.Program moduleTree, ModuleFormat format,
Map builtMap;
if (buildSourceMap && sourceMap != null) {
builtMap = placeSourceMap(sourceMap.build(jsUrl), mapUrl, customScheme,
multiRootOutputPath: multiRootOutputPath);
multiRootOutputPath: multiRootOutputPath, sourceMapBase: sourceMapBase);
var jsDir = p.dirname(p.fromUri(jsUrl));
var relative = p.relative(p.fromUri(mapUrl), from: jsDir);
var relativeMapUrl = p.toUri(relative).toString();
Expand Down
26 changes: 14 additions & 12 deletions pkg/frontend_server/lib/frontend_server.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import 'package:kernel/binary/limited_ast_to_binary.dart';
import 'package:kernel/kernel.dart'
show Component, loadComponentSourceFromBytes;
import 'package:kernel/target/targets.dart' show targets, TargetFlags;
import 'package:package_resolver/package_resolver.dart';
import 'package:path/path.dart' as path;
import 'package:usage/uuid/uuid.dart';

Expand Down Expand Up @@ -456,7 +457,8 @@ class FrontendCompiler implements CompilerInterface {
transformer?.transform(results.component);

if (_compilerOptions.target.name == 'dartdevc') {
await writeJavascriptBundle(results, _kernelBinaryFilename);
await writeJavascriptBundle(
results, _kernelBinaryFilename, options['filesystem-scheme']);
} else {
await writeDillFile(results, _kernelBinaryFilename,
filterExternal: importDill != null,
Expand Down Expand Up @@ -526,16 +528,14 @@ class FrontendCompiler implements CompilerInterface {
}

/// Write a JavaScript bundle containg the provided component.
Future<void> writeJavascriptBundle(
KernelCompilationResults results, String filename) async {
Future<void> writeJavascriptBundle(KernelCompilationResults results,
String filename, String fileSystemScheme) async {
var packageResolver = await PackageResolver.loadConfig(
_compilerOptions.packagesFileUri ?? '.packages');
final Component component = results.component;
// Compute strongly connected components.
final strongComponents = StrongComponents(
component,
results.loadedLibraries,
_mainSource,
_compilerOptions.packagesFileUri,
_compilerOptions.fileSystem);
final strongComponents = StrongComponents(component,
results.loadedLibraries, _mainSource, _compilerOptions.fileSystem);
await strongComponents.computeModules();

// Create JavaScript bundler.
Expand All @@ -545,11 +545,12 @@ class FrontendCompiler implements CompilerInterface {
if (!sourceFile.parent.existsSync()) {
sourceFile.parent.createSync(recursive: true);
}
final bundler = JavaScriptBundler(component, strongComponents);
final bundler = JavaScriptBundler(
component, strongComponents, fileSystemScheme, packageResolver);
final sourceFileSink = sourceFile.openWrite();
final manifestFileSink = manifestFile.openWrite();
final sourceMapsFileSink = sourceMapsFile.openWrite();
bundler.compile(
await bundler.compile(
results.classHierarchy,
results.coreTypes,
results.loadedLibraries,
Expand Down Expand Up @@ -790,7 +791,8 @@ class FrontendCompiler implements CompilerInterface {
deltaProgram.uriToSource.keys);

if (_compilerOptions.target.name == 'dartdevc') {
await writeJavascriptBundle(results, _kernelBinaryFilename);
await writeJavascriptBundle(
results, _kernelBinaryFilename, _options['filesystem-scheme']);
} else {
await writeDillFile(results, _kernelBinaryFilename,
incrementalSerializer: _generator.incrementalSerializer);
Expand Down
49 changes: 38 additions & 11 deletions pkg/frontend_server/lib/src/javascript_bundle.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import 'package:dev_compiler/dev_compiler.dart';
import 'package:kernel/ast.dart';
import 'package:kernel/class_hierarchy.dart';
import 'package:kernel/core_types.dart';
import 'package:path/path.dart' as p;
import 'package:package_resolver/package_resolver.dart';

import 'strong_components.dart';

Expand All @@ -22,7 +24,8 @@ import 'strong_components.dart';
/// an incremental build, a different file is written for each which contains
/// only the updated libraries.
class JavaScriptBundler {
JavaScriptBundler(this._originalComponent, this._strongComponents) {
JavaScriptBundler(this._originalComponent, this._strongComponents,
this._fileSystemScheme, this._packageResolver) {
_summaries = <Component>[];
_summaryUris = <Uri>[];
_moduleImportForSummary = <Uri, String>{};
Expand All @@ -36,27 +39,29 @@ class JavaScriptBundler {
);
_summaries.add(summaryComponent);
_summaryUris.add(uri);
_moduleImportForSummary[uri] = '${uri.path}.js';
_moduleImportForSummary[uri] = '${urlForComponentUri(uri)}.lib.js';
_uriToComponent[uri] = summaryComponent;
}
}

final StrongComponents _strongComponents;
final Component _originalComponent;
final String _fileSystemScheme;
final PackageResolver _packageResolver;

List<Component> _summaries;
List<Uri> _summaryUris;
Map<Uri, String> _moduleImportForSummary;
Map<Uri, Component> _uriToComponent;

/// Compile each component into a single JavaScript module.
void compile(
Future<void> compile(
ClassHierarchy classHierarchy,
CoreTypes coreTypes,
Set<Library> loadedLibraries,
IOSink codeSink,
IOSink manifestSink,
IOSink sourceMapsSink) {
IOSink sourceMapsSink) async {
var codeOffset = 0;
var sourceMapOffset = 0;
final manifest = <String, Map<String, List<int>>>{};
Expand All @@ -66,7 +71,8 @@ class JavaScriptBundler {
library.importUri.scheme == 'dart') {
continue;
}
final Uri moduleUri = _strongComponents.moduleAssignment[library.fileUri];
final Uri moduleUri =
_strongComponents.moduleAssignment[library.importUri];
if (visited.contains(moduleUri)) {
continue;
}
Expand All @@ -81,12 +87,29 @@ class JavaScriptBundler {
);
final jsModule = compiler.emitModule(
summaryComponent, _summaries, _summaryUris, _moduleImportForSummary);
final moduleUrl = moduleUri.toString();
final code = jsProgramToCode(jsModule, ModuleFormat.amd,
inlineSourceMap: true,
buildSourceMap: true,
jsUrl: '$moduleUrl',
mapUrl: '$moduleUrl.js.map');

final moduleUrl = urlForComponentUri(moduleUri);
String sourceMapBase;
if (moduleUri.scheme == 'package') {
// Source locations come through as absolute file uris. In order to
// make relative paths in the source map we get the absolute uri for
// the module and make them relative to that.
sourceMapBase =
p.dirname((await _packageResolver.resolveUri(moduleUri)).path);
}
final code = jsProgramToCode(
jsModule,
ModuleFormat.amd,
inlineSourceMap: true,
buildSourceMap: true,
jsUrl: '$moduleUrl.lib.js',
mapUrl: '$moduleUrl.lib.js.map',
sourceMapBase: sourceMapBase,
customScheme: _fileSystemScheme,
multiRootOutputPath: moduleUri.scheme == 'package'
? '/packages/${moduleUri.pathSegments.first}'
: null,
);
final codeBytes = utf8.encode(code.code);
final sourceMapBytes = utf8.encode(json.encode(code.sourceMap));
codeSink.add(codeBytes);
Expand All @@ -104,3 +127,7 @@ class JavaScriptBundler {
manifestSink.add(utf8.encode(json.encode(manifest)));
}
}

String urlForComponentUri(Uri componentUri) => componentUri.scheme == 'package'
? '/packages/${componentUri.path}'
: componentUri.path;
73 changes: 5 additions & 68 deletions pkg/frontend_server/lib/src/strong_components.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io';

import 'package:kernel/ast.dart';
import 'package:kernel/util/graph.dart';

import 'package:vm/kernel_front_end.dart';
import 'package:front_end/src/api_unstable/vm.dart' show FileSystem;

/// Compute the strongly connected components for JavaScript compilation.
Expand All @@ -31,7 +28,6 @@ class StrongComponents {
this.component,
this.loadedLibraries,
this.mainUri, [
this.packagesUri,
this.fileSystem,
]);

Expand All @@ -47,9 +43,6 @@ class StrongComponents {
/// The main URI for thiis application.
final Uri mainUri;

/// The URI of the .packages file.
final Uri packagesUri;

/// The filesystem instance for resolving files.
final FileSystem fileSystem;

Expand All @@ -72,83 +65,27 @@ class StrongComponents {
if (component.libraries.isEmpty) {
return;
}
Uri entrypointFileUri = mainUri;
if (!entrypointFileUri.isScheme('file')) {
entrypointFileUri = await asFileUri(fileSystem,
await _convertToFileUri(fileSystem, entrypointFileUri, packagesUri));
}
if (entrypointFileUri == null || !entrypointFileUri.isScheme('file')) {
throw Exception(
'Unable to map ${entrypointFileUri} back to file scheme.');
}

// If we don't have a file uri, just use the first library in the
// component.
Library entrypoint = component.libraries.firstWhere(
(Library library) => library.fileUri == entrypointFileUri,
(Library library) =>
library.fileUri == mainUri || library.importUri == mainUri,
orElse: () => null);

if (entrypoint == null) {
throw Exception(
'Could not find entrypoint ${entrypointFileUri} in Component.');
throw Exception('Could not find entrypoint ${mainUri} in Component.');
}

final List<List<Library>> results =
computeStrongComponents(_LibraryGraph(entrypoint, loadedLibraries));
for (List<Library> component in results) {
assert(component.length > 0);
final Uri moduleUri = component.first.fileUri;
final Uri moduleUri = component.first.importUri;
modules[moduleUri] = component;
for (Library componentLibrary in component) {
moduleAssignment[componentLibrary.fileUri] = moduleUri;
}
}
}

// Convert package URI to file URI if it is inside one of the packages.
Future<Uri> _convertToFileUri(
FileSystem fileSystem, Uri uri, Uri packagesUri) async {
if (uri == null || uri.scheme != 'package') {
return uri;
}
// Convert virtual URI to a real file URI.
// String uriString = (await asFileUri(fileSystem, uri)).toString();
List<String> packages;
try {
packages =
await File((await asFileUri(fileSystem, packagesUri)).toFilePath())
.readAsLines();
} on IOException {
// Can't read packages file - silently give up.
return uri;
}
// package:x.y/main.dart -> file:///a/b/x/y/main.dart
for (var line in packages) {
if (line.isEmpty || line.startsWith("#")) {
continue;
}

final colon = line.indexOf(':');
if (colon == -1) {
continue;
}
final packageName = line.substring(0, colon);
if (!uri.path.startsWith('$packageName/')) {
continue;
}
String packagePath;
try {
packagePath = (await asFileUri(
fileSystem, packagesUri.resolve(line.substring(colon + 1))))
.toString();
} on FileSystemException {
// Can't resolve package path.
continue;
moduleAssignment[componentLibrary.importUri] = moduleUri;
}
return Uri.parse(
'$packagePath${uri.path.substring(packageName.length + 1)}');
}
return uri;
}
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/frontend_server/pubspec.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ dependencies:
front_end: ^0.1.6
kernel: ^0.3.6
args: ^1.4.4
package_resolver: ^1.0.0

dev_dependencies:
test: any
test: any
Loading

0 comments on commit 4460f95

Please sign in to comment.