Skip to content

Commit

Permalink
Support browser caching using hashes in serverMode (fixes #93, fixes #92
Browse files Browse the repository at this point in the history 
).

Now the HTML can include a hash in the URL for cachable resources, and the serve knows how to include cache-control headers.

In the process of doing so, I had to change a couple things that made it possible to fix #92 as well (producing different output in the command-line than in server mode).

R=vsm@google.com

Review URL: https://codereview.chromium.org/993213003
  • Loading branch information
@sigmundch
sigmundch committed Mar 11, 2015
1 parent 135c1e5 commit 6d9564b
Show file tree
Hide file tree
Showing 22 changed files with 7,114 additions and 126 deletions.
2 changes: 1 addition & 1 deletion pkg/dev_compiler/bin/devc.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ void main(List<String> args) {
_showUsageAndExit();
}

if (!options.dumpInfo) setupLogger(options.logLevel, print);
setupLogger(options.logLevel, print);

if (options.serverMode) {
new CompilerServer(options).start();
Expand Down
64 changes: 47 additions & 17 deletions pkg/dev_compiler/lib/devc.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import 'src/dependency_graph.dart';
import 'src/info.dart' show LibraryInfo, CheckerResults;
import 'src/options.dart';
import 'src/report.dart';
import 'src/utils.dart';

/// Sets up the type checker logger to print a span that highlights error
/// messages.
Expand All @@ -50,6 +51,7 @@ class Compiler {
final SourceNode _entryNode;
List<LibraryInfo> _libraries = <LibraryInfo>[];
final List<CodeGenerator> _generators;
final bool _hashing;
bool _failure = false;

factory Compiler(CompilerOptions options,
Expand All @@ -65,7 +67,7 @@ class Compiler {
? new SummaryReporter()
: new LogReporter(options.useColors);
}
var graph = new SourceGraph(resolver.context, reporter);
var graph = new SourceGraph(resolver.context, reporter, options);
var rules = new RestrictedRules(resolver.context.typeProvider, reporter,
options: options);
var checker = new CodeChecker(rules, reporter, options);
Expand All @@ -87,11 +89,14 @@ class Compiler {
: new JSGenerator(outputDir, entryNode.uri, rules, options));
}
return new Compiler._(options, resolver, reporter, rules, checker, graph,
entryNode, generators);
entryNode, generators,
// TODO(sigmund): refactor to support hashing of the dart output?
options.serverMode && generators.length == 1 && !options.outputDart);
}

Compiler._(this._options, this._resolver, this._reporter, this._rules,
this._checker, this._graph, this._entryNode, this._generators);
this._checker, this._graph, this._entryNode, this._generators,
this._hashing);

bool _buildSource(SourceNode node) {
if (node is HtmlSourceNode) {
Expand Down Expand Up @@ -132,10 +137,12 @@ class Compiler {
// dev_compiler runtime.
if (_options.outputDir == null || _options.outputDart) return;
assert(node.uri.scheme == 'package');
var filepath = path.join(_options.outputDir, node.uri.path);
var filepath = path.join(_options.outputDir, resourceOutputPath(node.uri));
var dir = path.dirname(filepath);
new Directory(dir).createSync(recursive: true);
new File(filepath).writeAsStringSync(node.source.contents.data);
var text = node.source.contents.data;
new File(filepath).writeAsStringSync(text);
if (_hashing) node.cachingHash = computeHash(text);
}

bool _isEntry(DartSourceNode node) {
Expand Down Expand Up @@ -177,8 +184,10 @@ class Compiler {
_failure = true;
if (!_options.forceCompile) return;
}

for (var cg in _generators) {
cg.generateLibrary(units, current, _reporter);
var hash = cg.generateLibrary(units, current, _reporter);
if (_hashing) node.cachingHash = hash;
}
_reporter.leaveLibrary();
}
Expand All @@ -194,10 +203,8 @@ class Compiler {
rebuild(_entryNode, _graph, _buildSource);
_dumpInfoIfRequested();
clock.stop();
if (_options.serverMode) {
var time = (clock.elapsedMilliseconds / 1000).toStringAsFixed(2);
print('Compiled ${_libraries.length} libraries in ${time} s\n');
}
var time = (clock.elapsedMilliseconds / 1000).toStringAsFixed(2);
_log.fine('Compiled ${_libraries.length} libraries in ${time} s\n');
return new CheckerResults(
_libraries, _rules, _failure || _options.forceCompile);
}
Expand All @@ -215,7 +222,7 @@ class Compiler {
clock.stop();
if (changed > 0) _dumpInfoIfRequested();
var time = (clock.elapsedMilliseconds / 1000).toStringAsFixed(2);
print("Compiled ${changed} libraries in ${time} s\n");
_log.fine("Compiled ${changed} libraries in ${time} s\n");
}

_dumpInfoIfRequested() {
Expand Down Expand Up @@ -251,7 +258,7 @@ class CompilerServer {
exit(1);
}
var port = options.port;
print('[dev_compiler]: Serving $entryPath at http://0.0.0.0:$port/');
_log.fine('Serving $entryPath at http://0.0.0.0:$port/');
var compiler = new Compiler(options);
return new CompilerServer._(compiler, outDir, port, entryPath);
}
Expand All @@ -260,17 +267,40 @@ class CompilerServer {

Future start() async {
var handler = const shelf.Pipeline()
.addMiddleware(shelf.createMiddleware(requestHandler: rebuildIfNeeded))
.addMiddleware(rebuildAndCache)
.addHandler(shelf_static.createStaticHandler(outDir,
defaultDocument: _entryPath));
await shelf.serve(handler, '0.0.0.0', port);
compiler.run();
}

rebuildIfNeeded(shelf.Request request) {
var filepath = request.url.path;
if (filepath == '/$_entryPath' || filepath == '/') compiler._runAgain();
}
shelf.Handler rebuildAndCache(shelf.Handler handler) => (request) {
_log.fine('requested $GREEN_COLOR${request.url}$NO_COLOR');
// Trigger recompile only when requesting the HTML page.
var segments = request.url.pathSegments;
bool isEntryPage = segments.length == 0 || segments[0] == _entryPath;
if (isEntryPage) compiler._runAgain();

// To help browsers cache resources that don't change, we serve these
// resources under a path containing their hash:
// /cached/{hash}/{path-to-file.js}
bool isCached = segments.length > 1 && segments[0] == 'cached';
if (isCached) {
// Changing the request lets us record that the hash prefix is handled
// here, and that the underlying handler should use the rest of the url to
// determine where to find the resource in the file system.
request = request.change(path: path.join('cached', segments[1]));
}
var response = handler(request);
var policy = isCached ? 'max-age=${24 * 60 * 60}' : 'no-cache';
var headers = {'cache-control': policy};
if (isCached) {
// Note: the cache-control header should be enough, but this doesn't hurt
// and can help renew the policy after it expires.
headers['ETag'] = segments[1];
}
return response.change(headers: headers);
};
}

final _log = new Logger('dev_compiler');
Expand Down
5 changes: 4 additions & 1 deletion pkg/dev_compiler/lib/src/codegen/code_generator.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ abstract class CodeGenerator {
CodeGenerator(String outDir, this.root, this.rules)
: outDir = path.absolute(outDir);

void generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
/// Return a hash, if any, that can be used for caching purposes. When two
/// invocations to this function return the same hash, the underlying
/// code-generator generated the same code.
String generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
CheckerReporter reporter);
}
6 changes: 4 additions & 2 deletions pkg/dev_compiler/lib/src/codegen/dart_codegen.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,13 +452,14 @@ class DartGenerator extends codegenerator.CodeGenerator {
libs.forEach(doOne);
}

void generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
String generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
CheckerReporter reporter) {
_vm = new reifier.VariableManager();
_extraImports = new Set<LibraryElement>();
_generateLibrary(units, info, reporter);
_extraImports = null;
_vm = null;
return null;
}
}

Expand All @@ -485,14 +486,15 @@ class EmptyDartGenerator extends codegenerator.CodeGenerator {
EmptyDartGenerator(String outDir, Uri root, TypeRules rules, this.options)
: super(outDir, root, rules);

void generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
String generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
CheckerReporter reporter) {
for (var unit in units) {
var outputDir = makeOutputDirectory(info, unit);
reporter.enterSource(unit.element.source);
generateUnit(unit, info, outputDir);
reporter.leaveSource();
}
return null;
}

void generateUnit(CompilationUnit unit, LibraryInfo info, String libraryDir) {
Expand Down
49 changes: 32 additions & 17 deletions pkg/dev_compiler/lib/src/codegen/html_codegen.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ library dev_compiler.src.codegen.html_codegen;
import 'package:html5lib/dom.dart';
import 'package:html5lib/parser.dart' show parseFragment;
import 'package:logging/logging.dart' show Logger;
import 'package:path/path.dart' as path;

import 'package:dev_compiler/src/dependency_graph.dart';
import 'package:dev_compiler/src/options.dart';
import 'package:dev_compiler/src/utils.dart' show colorOf;
import 'package:dev_compiler/src/utils.dart' show colorOf, resourceOutputPath;

import 'js_codegen.dart' show jsLibraryName, jsOutputPath;

Expand Down Expand Up @@ -42,42 +43,49 @@ String generateEntryHtml(HtmlSourceNode root, CompilerOptions options) {
if (options.outputDart) return '${document.outerHtml}\n';

var libraries = [];
var resources = [];
visitInPostOrder(root, (n) {
if (n is DartSourceNode) libraries.add(n);
if (n is ResourceSourceNode) resources.add(n);
}, includeParts: false);

String mainLibraryName;
var fragment = _loadRuntimeScripts(options);
var fragment = new DocumentFragment();
for (var resource in resources) {
var resourcePath = resourceOutputPath(resource.uri);
if (resource.cachingHash != null) {
resourcePath = _addHash(resourcePath, resource.cachingHash);
}
if (path.extension(resourcePath) == '.css') {
fragment.nodes.add(_cssInclude(resourcePath));
} else {
fragment.nodes.add(_libraryInclude(resourcePath));
}
}
if (!options.checkSdk) fragment.nodes.add(_miniMockSdk);
for (var lib in libraries) {
var info = lib.info;
if (info == null) continue;
if (info.isEntry) mainLibraryName = jsLibraryName(info.library);
fragment.nodes.add(_libraryInclude(jsOutputPath(info, root.uri)));
var jsPath = jsOutputPath(info, root.uri);
if (lib.cachingHash != null) {
jsPath = _addHash(jsPath, lib.cachingHash);
}
fragment.nodes.add(_libraryInclude(jsPath));
}
fragment.nodes.add(_invokeMain(mainLibraryName));
scripts[0].replaceWith(fragment);
return '${document.outerHtml}\n';
}

/// A document fragment with scripts that check for harmony features and that
/// inject our runtime.
Node _loadRuntimeScripts(options) {
// TODO(sigmund): use dev_compiler to generate messages_widget in the future.
var widgetCode = options.serverMode
? '<script src="dev_compiler/runtime/messages_widget.js"></script>\n'
'<link rel="stylesheet" href="dev_compiler/runtime/messages.css">'
: '';
return parseFragment(
'<script src="dev_compiler/runtime/harmony_feature_check.js"></script>\n'
'<script src="dev_compiler/runtime/dart_runtime.js"></script>\n'
'$widgetCode');
}

/// A script tag that loads the .js code for a compiled library.
Node _libraryInclude(String jsUrl) =>
parseFragment('<script src="$jsUrl"></script>\n');

/// A tag that loads the .css code.
Node _cssInclude(String cssUrl) =>
parseFragment('<link rel="stylesheet" href="$cssUrl">\n');

/// A script tag that invokes the main function on the entry point library.
Node _invokeMain(String mainLibraryName) {
var code = mainLibraryName == null
Expand All @@ -86,6 +94,13 @@ Node _invokeMain(String mainLibraryName) {
return parseFragment('<script>$code</script>\n');
}

/// Convert the outputPath to include the hash in it. This function is the
/// reverse of what the server does to determine whether a request needs to have
/// cache headers added to it.
_addHash(String outPath, String hash) {
return path.join('cached', hash, outPath);
}

/// A script tag with a tiny mock of the core SDK. This is just used for testing
/// some small samples.
// TODO(sigmund,jmesserly): remove.
Expand Down
30 changes: 16 additions & 14 deletions pkg/dev_compiler/lib/src/codegen/js_codegen.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -960,9 +960,8 @@ class JSCodegenVisitor extends GeneralizingAstVisitor with ConversionVisitor {
new JS.VariableDeclaration(last.name.name),
_visit(lastInitializer.target)));

var result = <JS.Expression>[
new JS.VariableDeclarationList('let', variables)
];
var result =
<JS.Expression>[new JS.VariableDeclarationList('let', variables)];
result.addAll(_visitList(lastInitializer.cascadeSections));
_cascadeTarget = savedCascadeTemp;
return _statement(result.map((e) => new JS.ExpressionStatement(e)));
Expand Down Expand Up @@ -1017,9 +1016,8 @@ class JSCodegenVisitor extends GeneralizingAstVisitor with ConversionVisitor {

void _flushLibraryProperties(List<JS.Statement> body) {
if (_properties.isEmpty) return;
body.add(js.statement('dart.copyProperties($_EXPORTS, { # });', [
_properties.map(_emitTopLevelProperty)
]));
body.add(js.statement('dart.copyProperties($_EXPORTS, { # });',
[_properties.map(_emitTopLevelProperty)]));
_properties.clear();
}

Expand Down Expand Up @@ -1609,9 +1607,8 @@ class JSCodegenVisitor extends GeneralizingAstVisitor with ConversionVisitor {
@override
visitListLiteral(ListLiteral node) {
// TODO(jmesserly): make this faster. We're wasting an array.
var list = js.call('new List.from(#)', [
new JS.ArrayInitializer(_visitList(node.elements))
]);
var list = js.call('new List.from(#)',
[new JS.ArrayInitializer(_visitList(node.elements))]);
if (node.constKeyword != null) {
list = js.commentExpression('Unimplemented const', list);
}
Expand Down Expand Up @@ -1926,7 +1923,7 @@ class JSGenerator extends CodeGenerator {
JSGenerator(String outDir, Uri root, TypeRules rules, this.options)
: super(outDir, root, rules);

void generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
String generateLibrary(Iterable<CompilationUnit> units, LibraryInfo info,
CheckerReporter reporter) {
JS.Program jsTree =
new JSCodegenVisitor(info, rules).generateLibrary(units, reporter);
Expand All @@ -1937,14 +1934,19 @@ class JSGenerator extends CodeGenerator {
if (options.emitSourceMaps) {
var outFilename = path.basename(outputPath);
var printer = new srcmaps.Printer(outFilename);
_writeNode(new SourceMapPrintingContext(
printer, path.dirname(outputPath)), jsTree);
_writeNode(
new SourceMapPrintingContext(printer, path.dirname(outputPath)),
jsTree);
printer.add('//# sourceMappingURL=$outFilename.map');
// Write output file and source map
new File(outputPath).writeAsStringSync(printer.text);
var text = printer.text;
new File(outputPath).writeAsStringSync(text);
new File('$outputPath.map').writeAsStringSync(printer.map);
return computeHash(text);
} else {
new File(outputPath).writeAsStringSync(jsNodeToString(jsTree));
var text = jsNodeToString(jsTree);
new File(outputPath).writeAsStringSync(text);
return computeHash(text);
}
}
}
Expand Down
Loading

0 comments on commit 6d9564b

Please sign in to comment.