Changed handling for token indexes in visual parse trees (fixes #52)

- Rules which don't match anything don't have an stop token and hence that cannot reliably be used to display the token range for a non-terminal. Instead the explicit range comes with the data.
- Fixed also a problem where token range strings in graphical parse trees were misplaced.

misc/light.css

@@ -335,7 +335,7 @@ svg.railroad-diagram g.diagram-text:hover path.diagram-text {
 g .token-value {
     fill: #404040;
     stroke: none;
-    font: 12pt "Source Code Pro", "Hack", "Consolas", "Andale Mono", monospace;;
+    font: 12pt "Source Code Pro", "Hack", "Consolas", "Andale Mono", monospace;
 }
 
 g .token-range {
@@ -347,7 +347,6 @@ g .token-range {
 /* Internal elements, only used in vscode */
 
 .header {
-    position: fixed;
     font-size: 12pt;
     z-index: 9999;
     top: 0;
@@ -362,9 +361,8 @@ g .token-range {
 
 .graph-initial {
     font-size: 30pt;
-    font-weight: bold;
+    font-weight: 800;
     vertical-align: middle;
-    padding-left: 10px;
 }
 
 .rule-index {

misc/parse-tree.js

@@ -12,7 +12,7 @@ const rectH = 25;
 var nodeSizeFactor = 1;
 
 var zoom = d3.zoom()
-    .scaleExtent([0.1, 3])
+    .scaleExtent([0.01, 3])
     .on("zoom", zoomed);
 
 var svg = d3.select("svg")
@@ -92,18 +92,21 @@ function update(parent) {
         .style("opacity", 0)
         .on("click", click);
 
-    nodeEnter.append("rect")
+    var rects = nodeEnter.append("rect")
         .attr("width", rectW)
         .attr("height", rectH)
         .attr("rx", function (d) {
-            return (d.children || d._children) ? 0 : 10;
+            return (d.data.type == 0) ? 0 : 10;
         })
         .attr("ry", function (d) {
-            return (d.children || d._children) ? 0 : 10;
+            return (d.data.type == 0) ? 0 : 10;
         });
 
     createText(nodeEnter);
 
+    // This would resize the rect to tightly wrap the text, but unfortunately the tree/cluster layout relies on a fixed node size.
+    //rects.attr("width", d => this.parentNode.childNodes[1].getComputedTextLength() + 20);
+
     var t = d3.transition().duration(duration);
 
     updateExistingNodes(t);
@@ -235,14 +238,14 @@ function createText(nodeEnter) {
             return d.data.name;
         });
     nodeText // The bounding box is valid not before the node addition happened actually.
-        .attr("x", function (d) { return (rectW - this.getBBox().width) / 2; });
+        .attr("x", function (d) { return (rectW - this.getComputedTextLength()) / 2; });
 
-    // The node's token index/rang info.
+    // The node's token index/range info.
     nodeEnter.append("text")
         .attr("class", "token-range")
-        .attr("x", d => horizontal ? 0 : rectW)
+        .attr("x", 0)
         .attr("y", rectH / 2)
-        .attr("dx", d => horizontal ? 0 : "-1em")
+        .attr("dx", 0)
         .attr("dy", "-1.8em")
         .text(function (d) {
             if (d.data.type != 0) {
@@ -251,18 +254,22 @@ function createText(nodeEnter) {
                 }
                 return "\u2A33 " + d.data.symbol.tokenIndex;
             }
-            if (d.data.start.tokenIndex == d.data.stop.tokenIndex) {
-                return "\u2A33 " + d.data.start.tokenIndex;
+
+            if (d.data.range.length == 0) {
+                return "\u2A33 --";
+            }
+            if (d.data.range.length == 1) {
+                return "\u2A33 " + d.data.range.startIndex;
             }
-            return "\u2A33 " + d.data.start.tokenIndex + "-" + d.data.stop.tokenIndex;
+            return "\u2A33 " + d.data.range.startIndex + "-" + d.data.range.stopIndex;
         });
 
     // The node's content if this it is a terminal.
     nodeText = nodeEnter.append("text")
         .attr("class", "token-value")
         .attr("x", 0)
         .attr("y", rectH / 2)
-        .attr("dy", d => horizontal ? "0.25em" : "2.5em")
+        .attr("dy", horizontal ? "0.25em" : "2.5em")
         .text(function (d) {
             if (d.data.type == 0) {
                 return "";
@@ -278,20 +285,19 @@ function createText(nodeEnter) {
             return d.data.symbol.text;
         });
     nodeText
-        .attr("dx", function (d) { return horizontal ? rectW + 20: (rectW - this.getBBox().width) / 2; });
+        .attr("dx", function (d) {
+            return horizontal ? rectW + 20 : (rectW - this.getComputedTextLength()) / 2;
+        });
 }
 
 function updateExistingNodes(t) {
     // Update existing nodes (and their text elements) if there was a size change.
     nodeSelection.selectAll("rect").transition(t)
         .attr("width", rectW);
     nodeSelection.selectAll(".node-text").transition(t)
-        .attr("x", function (d) { return (rectW - this.getBBox().width) / 2; });
-    nodeSelection.selectAll(".token-range").transition(t)
-        .attr("x", 0)
-        .attr("dx", function (d) { return horizontal ? 0 : rectW - this.getBBox().width; })
+        .attr("x", function (d) { return (rectW - this.getComputedTextLength()) / 2; });
     nodeSelection.selectAll(".token-value").transition(t)
-        .attr("dx", function (d) { return horizontal ? rectW + 20: (rectW - this.getBBox().width) / 2; })
+        .attr("dx", function (d) { return horizontal ? rectW + 20: (rectW - this.getComputedTextLength()) / 2; })
         .attr("dy", d => horizontal ? "0.25em" : "2.5em")
 }
 

src/backend/GrapsDebugger.ts

@@ -25,7 +25,8 @@ import { Symbol, ScopedSymbol, BlockSymbol, VariableSymbol } from "antlr4-c3";
 import { InterpreterData } from "./InterpreterDataReader";
 import {
     LexerToken, ParseTreeNode, ParseTreeNodeType, SymbolInfo, LexicalRange,
-    AntlrFacade
+    AntlrFacade,
+    IndexRange
 } from "../backend/facade";
 import {
     AlternativeSymbol, ContextSymbolTable, RuleReferenceSymbol, EbnfSuffixSymbol, RuleSymbol, ActionSymbol
@@ -554,6 +555,12 @@ export class GrapsDebugger extends EventEmitter {
             result.name = this.parser!.ruleNames[tree.ruleIndex];
             result.start = this.convertToken(tree.start as CommonToken);
             result.stop = this.convertToken(tree.stop as CommonToken);
+
+            result.range = new IndexRange();
+            result.range.startIndex = tree.sourceInterval.a;
+            result.range.stopIndex = tree.sourceInterval.b;
+            result.range.length = tree.sourceInterval.length;
+
             if (tree.children) {
                 for (let child of tree.children) {
                     if ((child instanceof TerminalNode) && (child.symbol.type == Token.EOF)) {

src/backend/facade.ts

@@ -102,6 +102,16 @@ export enum ParseTreeNodeType {
     Error
 };
 
+/**
+ * Describes the a range in an input stream (character indexes in a char stream or token indexes in a token stream).
+ * Indexes can be < 0 if there's no input representation for a tree node (e.g. when it did not match anything).
+ */
+export class IndexRange {
+    startIndex: number;
+    stopIndex: number;
+    length: number;
+};
+
 /**
  * This node class is what exported parse trees are made of, which are created by the debugger interface.
  * Each node stands either for an invoked rule, a terminal node or an error node.
@@ -113,6 +123,7 @@ export class ParseTreeNode {
     name: string;
     start?: LexerToken;  // ditto
     stop?: LexerToken;   // ditto
+    range: IndexRange;   // ditto
 
     symbol?: LexerToken; // Only valid for non-rule nodes.
 

src/frontend/ATNGraphProvider.ts

@@ -153,7 +153,7 @@ export class AntlrATNGraphProvider extends WebviewProvider {
 
     protected handleMessage(message: any): boolean {
         if (message.command == "saveATNState") {
-            // This is the bounce back from the script code for our call to `cacheATNState` triggered from
+            // This is the bounce back from the script code for our call to `cacheATNLayout` triggered from
             // the `update()` function.
             let hash = Utils.hashFromPath(message.file);
             let basePath = path.dirname(message.file);