Include source in kernel.

- For now include source uncompressed.
- When running from kernel, use token position 0
  (i.e. dummy, but 'real' position) as start and end on functions
  and classes to enable Observatory to run with the dill file.
- Debugging does not work, but one can browse the source in
  Observatory.

R=kmillikin@google.com

Review-Url: https://codereview.chromium.org/2587673004 .

Author: jensjoha

pkg/kernel/binary.md

@@ -59,19 +59,20 @@ type StringReference {
   UInt index; // Index into the StringTable strings.
 }
 
-type LineStarts {
+type Source {
+  String source;
   // Line starts are delta-encoded (they are encoded as line lengths).  The list
   // [0, 10, 25, 32, 42] is encoded as [0, 10, 15, 7, 10].
   List<Uint> lineStarts;
 }
 
-type UriLineStarts {
+type UriSource {
   List<String> uris;
-  LineStarts[uris.length] lineStarts;
+  Source[uris.length] source;
 }
 
 type UriReference {
-  UInt index; // Index into the UriLineStarts uris.
+  UInt index; // Index into the UriSource uris.
 }
 
 type FileOffset {
@@ -93,7 +94,7 @@ type Something<T> extends Option<T> {
 type ProgramFile {
   MagicWord magic = 0x90ABCDEF;
   StringTable strings;
-  UriLineStarts lineStartsMap;
+  UriSource sourceMap;
   List<Library> libraries;
   LibraryProcedureReference mainMethod;
 }

pkg/kernel/lib/analyzer/loader.dart

@@ -120,7 +120,7 @@ class DartLoader implements ReferenceLevelLoader {
     var uri = applicationRoot.relativeUri(element.source.uri);
     return repository.getLibraryReference(uri)
       ..name ??= getLibraryName(element)
-      ..fileUri = "file://${element.source.fullName}";
+      ..fileUri = '${element.source.uri}';
   }
 
   void _buildTopLevelMember(
@@ -268,7 +268,7 @@ class DartLoader implements ReferenceLevelLoader {
     _classes[element] = classNode = new ast.Class(
         name: element.name,
         isAbstract: element.isAbstract,
-        fileUri: "file://${element.source.fullName}");
+        fileUri: '${element.source.uri}');
     classNode.level = ast.ClassLevel.Temporary;
     var library = getLibraryReference(element.library);
     library.addClass(classNode);
@@ -332,7 +332,8 @@ class DartLoader implements ReferenceLevelLoader {
               isAbstract: true,
               typeParameters: freshParameters.freshTypeParameters,
               supertype: freshParameters.substituteSuper(supertype),
-              mixedInType: freshParameters.substituteSuper(mixinType));
+              mixedInType: freshParameters.substituteSuper(mixinType),
+              fileUri: classNode.fileUri);
           mixinClass.level = ast.ClassLevel.Type;
           supertype = new ast.Supertype(mixinClass,
               classNode.typeParameters.map(makeTypeParameterType).toList());
@@ -445,7 +446,7 @@ class DartLoader implements ReferenceLevelLoader {
               isStatic: true,
               isExternal: constructor.isExternal,
               isConst: constructor.isConst,
-              fileUri: "file://${element.source.fullName}");
+              fileUri: '${element.source.uri}');
         }
         return new ast.Constructor(scope.buildFunctionInterface(constructor),
             name: _nameOfMember(element),
@@ -460,8 +461,7 @@ class DartLoader implements ReferenceLevelLoader {
             isFinal: variable.isFinal,
             isConst: variable.isConst,
             type: scope.buildType(variable.type),
-            fileUri: "file://${element.source.fullName}")
-          ..fileOffset = element.nameOffset;
+            fileUri: '${element.source.uri}')..fileOffset = element.nameOffset;
 
       case ElementKind.METHOD:
       case ElementKind.GETTER:
@@ -480,7 +480,7 @@ class DartLoader implements ReferenceLevelLoader {
             isAbstract: executable.isAbstract,
             isStatic: executable.isStatic,
             isExternal: executable.isExternal,
-            fileUri: "file://${element.source.fullName}");
+            fileUri: '${element.source.uri}');
 
       default:
         throw 'Unexpected member kind: $element';
@@ -586,7 +586,7 @@ class DartLoader implements ReferenceLevelLoader {
           typeParameters: fresh.freshTypeParameters,
           supertype: new ast.Supertype(superclass, superArgs),
           mixedInType: new ast.Supertype(mixedInClass, mixinArgs),
-          fileUri: mixedInClass.fileUri);
+          fileUri: library.fileUri);
       result.level = ast.ClassLevel.Type;
       library.addClass(result);
       return result;
@@ -694,9 +694,15 @@ class DartLoader implements ReferenceLevelLoader {
           in libraryElement.units) {
         var source = compilationUnitElement.source;
         LineInfo lineInfo = context.computeLineInfo(source);
-        program.uriToLineStarts["file://${source.fullName}"] =
-            new List<int>.generate(lineInfo.lineCount, lineInfo.getOffsetOfLine,
-                growable: false);
+        String sourceCode;
+        try {
+          sourceCode = context.getContents(source).data;
+        } catch (e) {
+          // The source's contents could not be accessed.
+          sourceCode = '';
+        }
+        program.uriToSource['${source.uri}'] =
+            new ast.Source(lineInfo.lineStarts, sourceCode);
       }
     }
     return program;

pkg/kernel/lib/ast.dart

@@ -3504,17 +3504,17 @@ class Supertype extends Node {
 class Program extends TreeNode {
   final List<Library> libraries;
 
-  /// Map from a source file uri to a line-starts table.
+  /// Map from a source file uri to a line-starts table and source code.
   /// Given a source file uri and a offset in that file one can translate
   /// it to a line:column position in that file.
-  final Map<String, List<int>> uriToLineStarts;
+  final Map<String, Source> uriToSource;
 
   /// Reference to the main method in one of the libraries.
   Procedure mainMethod;
 
-  Program([List<Library> libraries, Map<String, List<int>> uriToLineStarts])
+  Program([List<Library> libraries, Map<String, Source> uriToSource])
       : libraries = libraries ?? <Library>[],
-        uriToLineStarts = uriToLineStarts ?? {} {
+        uriToSource = uriToSource ?? <String, Source>{} {
     setParents(libraries, this);
   }
 
@@ -3533,7 +3533,7 @@ class Program extends TreeNode {
 
   /// Translates an offset to line and column numbers in the given file.
   Location getLocation(String file, int offset) {
-    List<int> lines = uriToLineStarts[file];
+    List<int> lines = uriToSource[file].lineStarts;
     int low = 0, high = lines.length - 1;
     while (low < high) {
       int mid = high - ((high - low) >> 1); // Get middle, rounding up.
@@ -3651,3 +3651,10 @@ class _ChildReplacer extends Transformer {
     }
   }
 }
+
+class Source {
+  final List<int> lineStarts;
+  final String source;
+
+  Source(this.lineStarts, this.source);
+}

pkg/kernel/lib/binary/ast_from_binary.dart

@@ -177,7 +177,7 @@ class BinaryBuilder {
           'Magic number was: ${magic.toRadixString(16)}');
     }
     readStringTable();
-    Map<String, List<int>> uriToLineStarts = readUriToLineStarts();
+    Map<String, Source> uriToSource = readUriToSource();
     importTable.length = readUInt();
     for (int i = 0; i < importTable.length; ++i) {
       importTable[i] = new Library(null);
@@ -187,15 +187,17 @@ class BinaryBuilder {
       readLibrary();
     }
     var mainMethod = readMemberReference(allowNull: true);
-    return new Program(importTable, uriToLineStarts)..mainMethod = mainMethod;
+    return new Program(importTable, uriToSource)
+      ..mainMethod = mainMethod;
   }
 
-  Map<String, List<int>> readUriToLineStarts() {
+  Map<String, Source> readUriToSource() {
     readSourceUriTable();
     int length = _sourceUriTable.length;
-    Map<String, List<int>> uriToLineStarts = {};
+    Map<String, Source> uriToLineStarts = <String, Source>{};
     for (int i = 0; i < length; ++i) {
       String uri = _sourceUriTable[i];
+      String sourceCode = readStringEntry();
       int lineCount = readUInt();
       List<int> lineStarts = new List<int>(lineCount);
       int previousLineStart = 0;
@@ -204,7 +206,7 @@ class BinaryBuilder {
         lineStarts[j] = lineStart;
         previousLineStart = lineStart;
       }
-      uriToLineStarts[uri] = lineStarts;
+      uriToLineStarts[uri] = new Source(lineStarts, sourceCode);
     }
     return uriToLineStarts;
   }

pkg/kernel/lib/binary/ast_to_binary.dart

@@ -137,20 +137,23 @@ class BinaryPrinter extends Visitor {
     _importTable = new ProgramImportTable(program);
     _stringIndexer.build(program);
     writeStringTable(_stringIndexer);
-    writeUriToLineStarts(program);
+    writeUriToSource(program);
     writeList(program.libraries, writeNode);
     writeMemberReference(program.mainMethod, allowNull: true);
     _flush();
   }
 
-  void writeUriToLineStarts(Program program) {
-    program.uriToLineStarts.keys.forEach((uri) {
+  void writeUriToSource(Program program) {
+    program.uriToSource.keys.forEach((uri) {
       _sourceUriIndexer.put(uri);
     });
     writeStringTable(_sourceUriIndexer);
     for (int i = 0; i < _sourceUriIndexer.entries.length; i++) {
       String uri = _sourceUriIndexer.entries[i].value;
-      List<int> lineStarts = program.uriToLineStarts[uri] ?? [];
+      Source source = program.uriToSource[uri] ?? new Source([], '');
+      String sourceCode = source.source;
+      writeStringTableEntry(sourceCode);
+      List<int> lineStarts = source.lineStarts;
       writeUInt30(lineStarts.length);
       int previousLineStart = 0;
       lineStarts.forEach((lineStart) {

runtime/vm/kernel.h

@@ -311,29 +311,37 @@ class StringTable {
 };
 
 
-class LineStartingTable {
+class SourceTable {
  public:
   void ReadFrom(Reader* reader);
   void WriteTo(Writer* writer);
-  ~LineStartingTable() {
+  ~SourceTable() {
     for (intptr_t i = 0; i < size_; ++i) {
-      delete[] values_[i];
+      delete source_code_[i];
+      delete[] line_starts_[i];
     }
-    delete[] values_;
+    delete[] source_code_;
+    delete[] line_starts_;
+    delete[] line_count_;
   }
 
   intptr_t size() { return size_; }
-  intptr_t* valuesFor(int i) { return values_[i]; }
+  String* SourceFor(intptr_t i) { return source_code_[i]; }
+  intptr_t* LineStartsFor(intptr_t i) { return line_starts_[i]; }
+  intptr_t LineCountFor(intptr_t i) { return line_count_[i]; }
 
  private:
-  LineStartingTable() : values_(NULL), size_(0) {}
+  SourceTable()
+      : source_code_(NULL), line_starts_(NULL), line_count_(NULL), size_(0) {}
 
   friend class Program;
 
-  intptr_t** values_;
+  String** source_code_;
+  intptr_t** line_starts_;
+  intptr_t* line_count_;
   intptr_t size_;
 
-  DISALLOW_COPY_AND_ASSIGN(LineStartingTable);
+  DISALLOW_COPY_AND_ASSIGN(SourceTable);
 };
 
 // Forward declare all classes.
@@ -2798,7 +2806,7 @@ class Program : public TreeNode {
 
   StringTable& string_table() { return string_table_; }
   StringTable& source_uri_table() { return source_uri_table_; }
-  LineStartingTable& line_starting_table() { return line_starting_table_; }
+  SourceTable& source_table() { return source_table_; }
   List<Library>& libraries() { return libraries_; }
   Procedure* main_method() { return main_method_; }
 
@@ -2809,7 +2817,7 @@ class Program : public TreeNode {
   Ref<Procedure> main_method_;
   StringTable string_table_;
   StringTable source_uri_table_;
-  LineStartingTable line_starting_table_;
+  SourceTable source_table_;
 
   DISALLOW_COPY_AND_ASSIGN(Program);
 };

runtime/vm/kernel_binary.cc

@@ -828,33 +828,37 @@ void StringTable::WriteTo(Writer* writer) {
 }
 
 
-void LineStartingTable::ReadFrom(Reader* reader) {
+void SourceTable::ReadFrom(Reader* reader) {
   size_ = reader->helper()->program()->source_uri_table().strings().length();
-  values_ = new intptr_t*[size_];
+  source_code_ = new String*[size_];
+  line_starts_ = new intptr_t*[size_];
+  line_count_ = new intptr_t[size_];
   for (intptr_t i = 0; i < size_; ++i) {
+    source_code_[i] = StringImpl::ReadFrom(reader);
     intptr_t line_count = reader->ReadUInt();
-    intptr_t* line_starts = new intptr_t[line_count + 1];
-    line_starts[0] = line_count;
+    intptr_t* line_starts = new intptr_t[line_count];
+    line_count_[i] = line_count;
     intptr_t previous_line_start = 0;
     for (intptr_t j = 0; j < line_count; ++j) {
       intptr_t line_start = reader->ReadUInt() + previous_line_start;
-      line_starts[j + 1] = line_start;
+      line_starts[j] = line_start;
       previous_line_start = line_start;
     }
-    values_[i] = line_starts;
+    line_starts_[i] = line_starts;
   }
 }
 
 
-void LineStartingTable::WriteTo(Writer* writer) {
+void SourceTable::WriteTo(Writer* writer) {
   for (intptr_t i = 0; i < size_; ++i) {
-    intptr_t* line_starts = values_[i];
-    intptr_t line_count = line_starts[0];
+    StringImpl::WriteTo(writer, source_code_[i]);
+    intptr_t* line_starts = line_starts_[i];
+    intptr_t line_count = line_count_[i];
     writer->WriteUInt(line_count);
 
     intptr_t previous_line_start = 0;
     for (intptr_t j = 0; j < line_count; ++j) {
-      intptr_t line_start = line_starts[j + 1];
+      intptr_t line_start = line_starts[j];
       writer->WriteUInt(line_start - previous_line_start);
       previous_line_start = line_start;
     }
@@ -2828,7 +2832,7 @@ Program* Program::ReadFrom(Reader* reader) {
 
   program->string_table_.ReadFrom(reader);
   program->source_uri_table_.ReadFrom(reader);
-  program->line_starting_table_.ReadFrom(reader);
+  program->source_table_.ReadFrom(reader);
 
   int libraries = reader->ReadUInt();
   program->libraries().EnsureInitialized(libraries);
@@ -2853,7 +2857,7 @@ void Program::WriteTo(Writer* writer) {
   // strings in nodes are present in [string_table_].
   string_table_.WriteTo(writer);
   source_uri_table_.WriteTo(writer);
-  line_starting_table_.WriteTo(writer);
+  source_table_.WriteTo(writer);
 
   libraries_.WriteTo(writer);
   Reference::WriteMemberTo(writer, main_method_);