Bugfixes: past token, past end of, subtoken out of range, sort tokens

src/NavigationMap.ts

@@ -65,8 +65,36 @@ export default class NavigationMap {
   }
 
   public getTokenForRange(range: Range) {
-    return Object.values(this.map).find(
+    const matches = Object.values(this.map).filter(
       (token) => token.range.intersection(range) != null
     );
+
+    // If multiple matches sort and take the first
+    matches.sort((a, b) => {
+      // First sort on alphanumeric
+      const aIsAlphaNum = isAlphaNum(a.text);
+      const bIsAlphaNum = isAlphaNum(b.text);
+      if (aIsAlphaNum && !bIsAlphaNum) {
+        return -1;
+      }
+      if (bIsAlphaNum && !aIsAlphaNum) {
+        return 1;
+      }
+
+      // Second sort on length
+      const lengthDiff = b.text.length - a.text.length;
+      if (lengthDiff !== 0) {
+        return lengthDiff;
+      }
+
+      // Lastly sort on start position. ie leftmost
+      return a.startOffset - b.startOffset;
+    });
+
+    return matches[0] ?? null;
   }
 }
+
+function isAlphaNum(text: string) {
+  return /^\w+$/.test(text);
+}

src/inferFullTargets.ts

@@ -332,7 +332,10 @@ function inferRangeStartTarget(
   prototypeTargets: Target[],
   actionPreferences: ActionPreferences
 ): PrimitiveTarget {
-  const mark = target.mark ?? CURSOR_MARK;
+  const mark =
+    target.mark ??
+    (target.selectionType === "token" ? CURSOR_MARK_TOKEN : CURSOR_MARK);
+
   prototypeTargets = hasContent(target) ? [] : prototypeTargets;
 
   const selectionType =
@@ -377,6 +380,7 @@ export function inferRangeEndTarget(
 
   const mark =
     target.mark ??
+    (target.selectionType === "token" ? CURSOR_MARK_TOKEN : null) ??
     extractAttributeFromList(
       possiblePrototypeTargetsIncludingStartTarget,
       "mark"

src/processTargets.ts

@@ -24,6 +24,8 @@ import {
   SelectionWithEditor,
   Target,
   TypedSelection,
+  Position as TargetPosition,
+  InsideOutsideType,
 } from "./Types";
 
 export default function processTargets(
@@ -334,6 +336,15 @@ function transformSelection(
       const activeIndex =
         modifier.active < 0 ? modifier.active + pieces.length : modifier.active;
 
+      if (
+        anchorIndex < 0 ||
+        activeIndex < 0 ||
+        anchorIndex >= pieces.length ||
+        activeIndex >= pieces.length
+      ) {
+        throw new Error("Subtoken index out of range");
+      }
+
       const isReversed = activeIndex < anchorIndex;
 
       const anchor = selection.selection.start.translate(
@@ -461,6 +472,8 @@ function createTypedSelection(
         selectionContext: getTokenSelectionContext(
           selection,
           modifier,
+          position,
+          insideOutsideType,
           selectionContext
         ),
       };
@@ -600,6 +613,8 @@ function performPositionAdjustment(
 function getTokenSelectionContext(
   selection: SelectionWithEditor,
   modifier: Modifier,
+  position: TargetPosition,
+  insideOutsideType: InsideOutsideType,
   selectionContext: SelectionContext
 ): SelectionContext {
   if (!isSelectionContextEmpty(selectionContext)) {
@@ -611,32 +626,38 @@ function getTokenSelectionContext(
 
   const document = selection.editor.document;
   const { start, end } = selection.selection;
-
-  const startLine = document.lineAt(start);
-  const leadingText = startLine.text.slice(0, start.character);
-  const leadingDelimiters = leadingText.match(/\s+$/);
-  const leadingDelimiterRange =
-    leadingDelimiters != null
-      ? new Range(
-          start.line,
-          start.character - leadingDelimiters[0].length,
-          start.line,
-          start.character
-        )
-      : null;
-
   const endLine = document.lineAt(end);
-  const trailingText = endLine.text.slice(end.character);
-  const trailingDelimiters = trailingText.match(/^\s+/);
-  const trailingDelimiterRange =
-    trailingDelimiters != null
-      ? new Range(
-          end.line,
-          end.character,
-          end.line,
-          end.character + trailingDelimiters[0].length
-        )
-      : null;
+  let leadingDelimiterRange, trailingDelimiterRange;
+
+  // Position start/end of has no delimiter
+  if (position !== "before" || insideOutsideType !== "inside") {
+    const startLine = document.lineAt(start);
+    const leadingText = startLine.text.slice(0, start.character);
+    const leadingDelimiters = leadingText.match(/\s+$/);
+    leadingDelimiterRange =
+      leadingDelimiters != null
+        ? new Range(
+            start.line,
+            start.character - leadingDelimiters[0].length,
+            start.line,
+            start.character
+          )
+        : null;
+  }
+
+  if (position !== "after" || insideOutsideType !== "inside") {
+    const trailingText = endLine.text.slice(end.character);
+    const trailingDelimiters = trailingText.match(/^\s+/);
+    trailingDelimiterRange =
+      trailingDelimiters != null
+        ? new Range(
+            end.line,
+            end.character,
+            end.line,
+            end.character + trailingDelimiters[0].length
+          )
+        : null;
+  }
 
   const isInDelimitedList =
     (leadingDelimiterRange != null || trailingDelimiterRange != null) &&

src/test/suite/fixtures/recorded/compoundTargets/takePastEndOfToken.yml

@@ -0,0 +1,26 @@
+spokenForm: take past end of token
+languageId: typescript
+command:
+  actionName: setSelection
+  partialTargets:
+    - type: range
+      start: {type: primitive}
+      end: {type: primitive, position: after, insideOutsideType: inside, selectionType: token}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks: {}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 8}
+finalState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 11}
+  thatMark:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 11}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: cursor}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: inside}, end: {type: primitive, mark: {type: cursorToken}, selectionType: token, position: after, modifier: {type: identity}, insideOutsideType: inside}}]

src/test/suite/fixtures/recorded/compoundTargets/takePastStartOfToken.yml

@@ -0,0 +1,26 @@
+spokenForm: take past start of token
+languageId: typescript
+command:
+  actionName: setSelection
+  partialTargets:
+    - type: range
+      start: {type: primitive}
+      end: {type: primitive, position: before, insideOutsideType: inside, selectionType: token}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks: {}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 8}
+finalState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 6}
+  thatMark:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 6}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: cursor}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: inside}, end: {type: primitive, mark: {type: cursorToken}, selectionType: token, position: before, modifier: {type: identity}, insideOutsideType: inside}}]

src/test/suite/fixtures/recorded/compoundTargets/takeTokenPastTrap.yml

@@ -0,0 +1,31 @@
+spokenForm: take token past trap
+languageId: typescript
+command:
+  actionName: setSelection
+  partialTargets:
+    - type: range
+      start: {type: primitive, selectionType: token}
+      end:
+        type: primitive
+        mark: {type: decoratedSymbol, symbolColor: default, character: t}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks:
+  default.t:
+    start: {line: 0, character: 6}
+    end: {line: 0, character: 11}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 3}
+      active: {line: 0, character: 3}
+finalState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 0}
+      active: {line: 0, character: 11}
+  thatMark:
+    - anchor: {line: 0, character: 0}
+      active: {line: 0, character: 11}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: cursorToken}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: inside}, end: {type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: t}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: inside}}]

src/test/suite/fixtures/recorded/compoundTargets/takeTrapPastToken.yml

@@ -0,0 +1,31 @@
+spokenForm: take trap past token
+languageId: typescript
+command:
+  actionName: setSelection
+  partialTargets:
+    - type: range
+      start:
+        type: primitive
+        mark: {type: decoratedSymbol, symbolColor: default, character: t}
+      end: {type: primitive, selectionType: token}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks:
+  default.t:
+    start: {line: 0, character: 6}
+    end: {line: 0, character: 11}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+finalState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 11}
+      active: {line: 0, character: 0}
+  thatMark:
+    - anchor: {line: 0, character: 11}
+      active: {line: 0, character: 0}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: t}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: inside}, end: {type: primitive, mark: {type: cursorToken}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: inside}}]

src/test/suite/fixtures/recorded/positions/chuckPastAfterLook.yml

@@ -0,0 +1,32 @@
+spokenForm: chuck past after look
+languageId: typescript
+command:
+  actionName: delete
+  partialTargets:
+    - type: range
+      start: {type: primitive}
+      end:
+        type: primitive
+        position: after
+        mark: {type: decoratedSymbol, symbolColor: default, character: l}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks:
+  default.l:
+    start: {line: 0, character: 0}
+    end: {line: 0, character: 5}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+finalState:
+  documentContents: hethere
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+  thatMark:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: cursor}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: outside}, end: {type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: l}, selectionType: token, position: after, modifier: {type: identity}, insideOutsideType: outside}}]

src/test/suite/fixtures/recorded/positions/chuckPastBeforeTrap.yml

@@ -0,0 +1,32 @@
+spokenForm: chuck past before trap
+languageId: typescript
+command:
+  actionName: delete
+  partialTargets:
+    - type: range
+      start: {type: primitive}
+      end:
+        type: primitive
+        position: before
+        mark: {type: decoratedSymbol, symbolColor: default, character: t}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks:
+  default.t:
+    start: {line: 0, character: 6}
+    end: {line: 0, character: 11}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 8}
+finalState:
+  documentContents: helloere
+  selections:
+    - anchor: {line: 0, character: 5}
+      active: {line: 0, character: 5}
+  thatMark:
+    - anchor: {line: 0, character: 5}
+      active: {line: 0, character: 5}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: cursor}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: outside}, end: {type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: t}, selectionType: token, position: before, modifier: {type: identity}, insideOutsideType: outside}}]

src/test/suite/fixtures/recorded/positions/chuckPastEndOfLine.yml

@@ -0,0 +1,26 @@
+spokenForm: chuck past end of line
+languageId: typescript
+command:
+  actionName: delete
+  partialTargets:
+    - type: range
+      start: {type: primitive}
+      end: {type: primitive, position: after, insideOutsideType: inside, selectionType: line}
+      excludeStart: false
+      excludeEnd: false
+  extraArgs: []
+marks: {}
+initialState:
+  documentContents: hello there
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+finalState:
+  documentContents: he
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+  thatMark:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+fullTargets: [{type: range, excludeStart: false, excludeEnd: false, start: {type: primitive, mark: {type: cursor}, selectionType: token, position: contents, modifier: {type: identity}, insideOutsideType: outside}, end: {type: primitive, mark: {type: cursor}, selectionType: line, position: after, modifier: {type: identity}, insideOutsideType: inside}}]