Significant refactor of search system and :s[ubstitute]

Central to this refactor is a rationalization of pattern parsing - we now have an immutable`Pattern` class with its own Parsimmon parser; it's now used everywhere.

A few enhancements, such as supporting `/{pattern}/` as line specifier in line range

And some bugs fixed:
- `\n` now expands to `\r?\n` (to work with CRLF files)
- `\<` and `\>` (word boundaries) are now mapped to `\b`

Refs #3996

src/actions/commands/actions.ts

@@ -20,7 +20,6 @@ import {
   visualBlockGetBottomRightPosition,
 } from './../../mode/mode';
 import { Register, RegisterMode } from './../../register/register';
-import { SearchDirection } from './../../state/searchState';
 import { EditorScrollByUnit, EditorScrollDirection, TextEditor } from './../../textEditor';
 import { isTextTransformation, Transformation } from './../../transformations/transformations';
 import { RegisterAction, BaseCommand } from './../base';
@@ -35,6 +34,7 @@ import { Position } from 'vscode';
 import { WriteQuitCommand } from '../../cmd_line/commands/writequit';
 import { shouldWrapKey } from '../wrapping';
 import { ErrorCode, VimError } from '../../error';
+import { SearchDirection } from '../../vimscript/pattern';
 
 /**
  * A very special snowflake.

src/actions/commands/commandLine.ts

@@ -14,11 +14,11 @@ import { StatusBar } from '../../statusBar';
 import { getPathDetails, readDirectory } from '../../util/path';
 import { Clipboard } from '../../util/clipboard';
 import { VimError, ErrorCode } from '../../error';
-import { SearchDirection } from '../../state/searchState';
 import { scrollView } from '../../util/util';
 import { getWordLeftInText, getWordRightInText, WordType } from '../../textobject/word';
 import { Position } from 'vscode';
 import { builtinExCommands } from '../../vimscript/exCommandParser';
+import { SearchDirection } from '../../vimscript/pattern';
 
 /**
  * Commands that are only relevant when entering a command or search
@@ -421,7 +421,7 @@ class CommandInsertInSearchMode extends BaseCommand {
         searchState.searchString.slice(vimState.statusBarCursorCharacterPos);
       vimState.statusBarCursorCharacterPos = Math.max(vimState.statusBarCursorCharacterPos - 1, 0);
     } else if (key === '<C-f>') {
-      await new CommandShowSearchHistory(searchState.searchDirection).exec(position, vimState);
+      await new CommandShowSearchHistory(searchState.direction).exec(position, vimState);
     } else if (key === '<C-u>') {
       searchState.searchString = searchState.searchString.slice(
         vimState.statusBarCursorCharacterPos
@@ -473,7 +473,7 @@ class CommandInsertInSearchMode extends BaseCommand {
         StatusBar.displayError(
           vimState,
           VimError.fromCode(
-            searchState.searchDirection === SearchDirection.Backward
+            searchState.direction === SearchDirection.Backward
               ? ErrorCode.SearchHitTop
               : ErrorCode.SearchHitBottom,
             searchState.searchString
@@ -790,9 +790,9 @@ class CommandCtrlLInSearchMode extends BaseCommand {
 
     const nextMatch = globalState.searchState.getNextSearchMatchRange(vimState.editor, position);
     if (nextMatch) {
-      const line = vimState.document.lineAt(nextMatch.end).text;
-      if (nextMatch.end.character < line.length) {
-        globalState.searchState.searchString += line[nextMatch.end.character];
+      const line = vimState.document.lineAt(nextMatch.range.end).text;
+      if (nextMatch.range.end.character < line.length) {
+        globalState.searchState.searchString += line[nextMatch.range.end.character];
         vimState.statusBarCursorCharacterPos++;
       }
     }

src/actions/commands/search.ts

@@ -1,17 +1,20 @@
 import * as _ from 'lodash';
-import { Position, Selection } from 'vscode';
+import { escapeRegExp } from 'lodash';
+import {} from 'vscode';
+import { Position, Range, Selection } from 'vscode';
 import { sorted } from '../../common/motion/position';
 import { configuration } from '../../configuration/configuration';
 import { VimError, ErrorCode } from '../../error';
 import { Mode } from '../../mode/mode';
 import { Register } from '../../register/register';
 import { globalState } from '../../state/globalState';
-import { SearchDirection, SearchState } from '../../state/searchState';
+import { SearchState } from '../../state/searchState';
 import { VimState } from '../../state/vimState';
 import { StatusBar } from '../../statusBar';
 import { TextEditor } from '../../textEditor';
 import { TextObject } from '../../textobject/textobject';
 import { reportSearch } from '../../util/statusBarTextUtils';
+import { SearchDirection } from '../../vimscript/pattern';
 import { RegisterAction, BaseCommand } from '../base';
 import { failedMovement, IMovement } from '../baseMotion';
 
@@ -84,26 +87,26 @@ async function searchCurrentSelection(vimState: VimState, direction: SearchDirec
  * Used by [g]* and [g]#
  */
 async function createSearchStateAndMoveToMatch(args: {
-  needle?: string | undefined;
+  needle: string;
   vimState: VimState;
   direction: SearchDirection;
   isExact: boolean;
   searchStartCursorPosition: Position;
 }): Promise<void> {
   const { needle, vimState, isExact } = args;
 
-  if (needle === undefined || needle.length === 0) {
+  if (needle.length === 0) {
     return;
   }
 
-  const searchString = isExact ? `\\b${needle}\\b` : needle;
+  const searchString = isExact ? `\\<${escapeRegExp(needle)}\\>` : escapeRegExp(needle);
 
   // Start a search for the given term.
   globalState.searchState = new SearchState(
     args.direction,
     vimState.cursorStopPosition,
     searchString,
-    { isRegex: isExact, ignoreSmartcase: true },
+    { ignoreSmartcase: true },
     vimState.currentMode
   );
   Register.setReadonlyRegister('/', globalState.searchState.searchString);
@@ -238,7 +241,7 @@ class CommandSearchForwards extends BaseCommand {
       SearchDirection.Forward,
       vimState.cursorStopPosition,
       '',
-      { isRegex: true },
+      {},
       vimState.currentMode
     );
     await vimState.setCurrentMode(Mode.SearchInProgressMode);
@@ -263,7 +266,7 @@ class CommandSearchBackwards extends BaseCommand {
       SearchDirection.Backward,
       vimState.cursorStopPosition,
       '',
-      { isRegex: true },
+      {},
       vimState.currentMode
     );
     await vimState.setCurrentMode(Mode.SearchInProgressMode);
@@ -287,26 +290,25 @@ abstract class SearchObject extends TextObject {
       this.direction,
       vimState.cursorStopPosition,
       searchState.searchString,
-      { isRegex: true },
+      {},
       vimState.currentMode
     );
 
     let result:
       | {
-          start: Position;
-          end: Position;
+          range: Range;
           index: number;
         }
       | undefined;
 
     // At first, try to search for current word, and stop searching if matched.
     // Try to search for the next word if not matched or
     // if the cursor is at the end of a match string in visual-mode.
-    result = newSearchState.getSearchMatchRangeOf(vimState.editor, vimState.cursorStopPosition);
+    result = newSearchState.findContainingMatchRange(vimState.editor, vimState.cursorStopPosition);
     if (
       result &&
       vimState.currentMode === Mode.Visual &&
-      vimState.cursorStopPosition.isEqual(result.end.getLeftThroughLineBreaks())
+      vimState.cursorStopPosition.isEqual(result.range.end.getLeftThroughLineBreaks())
     ) {
       result = undefined;
     }
@@ -322,8 +324,8 @@ abstract class SearchObject extends TextObject {
     reportSearch(result.index, searchState.getMatchRanges(vimState.editor).length, vimState);
 
     let [start, stop] = [
-      vimState.currentMode === Mode.Normal ? result.start : vimState.cursorStopPosition,
-      result.end.getLeftThroughLineBreaks(),
+      vimState.currentMode === Mode.Normal ? result.range.start : vimState.cursorStopPosition,
+      result.range.end.getLeftThroughLineBreaks(),
     ];
 
     if (vimState.recordedState.operator) {

src/actions/motion.ts

@@ -18,7 +18,6 @@ import { globalState } from '../state/globalState';
 import { reportSearch } from '../util/statusBarTextUtils';
 import { SneakForward, SneakBackward } from './plugins/sneak';
 import { Notation } from '../configuration/notation';
-import { SearchDirection } from '../state/searchState';
 import { StatusBar } from '../statusBar';
 import { clamp } from '../util/util';
 import { getCurrentParagraphBeginning, getCurrentParagraphEnd } from '../textobject/paragraph';
@@ -27,6 +26,7 @@ import { Position } from 'vscode';
 import { sorted } from '../common/motion/position';
 import { WordType } from '../textobject/word';
 import { CommandInsertAtCursor } from './commands/actions';
+import { SearchDirection } from '../vimscript/pattern';
 
 /**
  * A movement is something like 'h', 'k', 'w', 'b', 'gg', etc.
@@ -505,7 +505,7 @@ class CommandNextSearchMatch extends BaseMovement {
     // we have to handle a special case here: searching for $ or \n,
     // which we approximate by positionIsEOL. In that case (but only when searching forward)
     // we need to "offset" by getRight for searching the next match, otherwise we get stuck.
-    const searchForward = searchState.searchDirection === SearchDirection.Forward;
+    const searchForward = searchState.direction === SearchDirection.Forward;
     const positionIsEOL = position.getRight().isEqual(position.getLineEnd());
     const nextMatch =
       positionIsEOL && searchForward
@@ -516,7 +516,7 @@ class CommandNextSearchMatch extends BaseMovement {
       StatusBar.displayError(
         vimState,
         VimError.fromCode(
-          searchState.searchDirection === SearchDirection.Forward
+          searchState.direction === SearchDirection.Forward
             ? ErrorCode.SearchHitBottom
             : ErrorCode.SearchHitTop,
           searchState.searchString
@@ -557,7 +557,7 @@ class CommandPreviousSearchMatch extends BaseMovement {
       return failedMovement(vimState);
     }
 
-    const searchForward = searchState.searchDirection === SearchDirection.Forward;
+    const searchForward = searchState.direction === SearchDirection.Forward;
     const positionIsEOL = position.getRight().isEqual(position.getLineEnd());
 
     // see implementation of n, above.
@@ -578,7 +578,7 @@ class CommandPreviousSearchMatch extends BaseMovement {
       StatusBar.displayError(
         vimState,
         VimError.fromCode(
-          searchState.searchDirection === SearchDirection.Forward
+          searchState.direction === SearchDirection.Forward
             ? ErrorCode.SearchHitTop
             : ErrorCode.SearchHitBottom,
           searchState.searchString

src/cmd_line/commands/history.ts

@@ -3,8 +3,8 @@ import {
   CommandShowSearchHistory,
   CommandShowCommandHistory,
 } from '../../actions/commands/actions';
-import { SearchDirection } from '../../state/searchState';
 import { ExCommand } from '../../vimscript/exCommand';
+import { SearchDirection } from '../../vimscript/pattern';
 
 export enum HistoryCommandType {
   Cmd,