Skip to content

make Internal.Utilities.Text.Lexing.Position a struct #664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

make Internal.Utilities.Text.Lexing.Position a struct #664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 33 additions & 21 deletions src/fsharp/LexFilter.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,12 +403,18 @@ type LexbufState(startPos: Position,
member x.EndPos = endPos
member x.PastEOF = pastEOF

[<Struct>]
type PositionTuple =
val X: Position
val Y: Position
new (x: Position, y: Position) = { X = x; Y = y }

/// Used to save the state related to a token
[<Class>]
type TokenTup =
val Token : token
val LexbufState : LexbufState
val LastTokenPos: Position * Position
val LastTokenPos: PositionTuple
new (token,state,lastTokenPos) = { Token=token; LexbufState=state;LastTokenPos=lastTokenPos }

/// Returns starting position of the token
Expand Down Expand Up @@ -485,6 +491,12 @@ let (|TyparsCloseOp|_|) (txt:string) =
| _ -> None
Some([| for _c in angles do yield GREATER |],afterOp)

[<Struct>]
type PositionWithColumn =
val Position: Position
val Column: int
new (position: Position, column: int) = { Position = position; Column = column }

//----------------------------------------------------------------------------
// build a LexFilter
//--------------------------------------------------------------------------*)
Expand Down Expand Up @@ -553,7 +565,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
let tokenLexbufState = getLexbufState()
savedLexbufState <- tokenLexbufState
haveLexbufState <- true
TokenTup(token,tokenLexbufState,(lastTokenStart,lastTokenEnd))
TokenTup(token,tokenLexbufState,PositionTuple(lastTokenStart,lastTokenEnd))

//----------------------------------------------------------------------------
// Fetch a raw token, either from the old lexer or from our delayedStack
Expand Down Expand Up @@ -623,7 +635,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
let pushCtxt tokenTup (newCtxt:Context) =
let rec unindentationLimit strict stack =
match newCtxt,stack with
| _, [] -> (newCtxt.StartPos, -1)
| _, [] -> PositionWithColumn(newCtxt.StartPos, -1)

// ignore Vanilla because a SeqBlock is always coming
| _, (CtxtVanilla _ :: rest) -> unindentationLimit strict rest
Expand All @@ -635,8 +647,8 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// '(match' limited by minimum of two
| _,(((CtxtMatch _) as ctxt1) :: CtxtSeqBlock _ :: (CtxtParen ((BEGIN | LPAREN),_) as ctxt2) :: _rest)
-> if ctxt1.StartCol <= ctxt2.StartCol
then (ctxt1.StartPos,ctxt1.StartCol)
else (ctxt2.StartPos,ctxt2.StartCol)
then PositionWithColumn(ctxt1.StartPos,ctxt1.StartCol)
else PositionWithColumn(ctxt2.StartPos,ctxt2.StartCol)

// 'let ... = function' limited by 'let', precisely
// This covers the common form
Expand All @@ -645,15 +657,15 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// | Case1 -> ...
// | Case2 -> ...
| (CtxtMatchClauses _), (CtxtFunction _ :: CtxtSeqBlock _ :: (CtxtLetDecl _ as limitCtxt) :: _rest)
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)

// Otherwise 'function ...' places no limit until we hit a CtxtLetDecl etc... (Recursive)
| (CtxtMatchClauses _), (CtxtFunction _ :: rest)
-> unindentationLimit false rest

// 'try ... with' limited by 'try'
| _,(CtxtMatchClauses _ :: (CtxtTry _ as limitCtxt) :: _rest)
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)

// 'fun ->' places no limit until we hit a CtxtLetDecl etc... (Recursive)
| _,(CtxtFun _ :: rest)
Expand All @@ -672,7 +684,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// This is a serious thing to allow, but is required since there is no "return" in this language.
// Without it there is no way of escaping special cases in large bits of code without indenting the main case.
| CtxtSeqBlock _, (CtxtElse _ :: (CtxtIf _ as limitCtxt) :: _rest)
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)

// Permitted inner-construct precise block alighnment:
// interface ...
Expand All @@ -683,7 +695,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// with ...
// end
| CtxtWithAsAugment _,((CtxtInterfaceHead _ | CtxtMemberHead _ | CtxtException _ | CtxtTypeDefns _) as limitCtxt :: _rest)
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)

// Permit unindentation via parentheses (or begin/end) following a 'then', 'else' or 'do':
// if nr > 0 then (
Expand Down Expand Up @@ -754,12 +766,12 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// 'type C = interface ... ' limited by 'type'
// 'type C = struct ... ' limited by 'type'
| _,(CtxtParen ((CLASS | STRUCT | INTERFACE),_) :: CtxtSeqBlock _ :: (CtxtTypeDefns _ as limitCtxt) :: _)
-> (limitCtxt.StartPos,limitCtxt.StartCol + 1)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol + 1)

// REVIEW: document these
| _,(CtxtSeqBlock _ :: CtxtParen((BEGIN | LPAREN | LBRACK | LBRACK_BAR),_) :: CtxtVanilla _ :: (CtxtSeqBlock _ as limitCtxt) :: _)
| (CtxtSeqBlock _),(CtxtParen ((BEGIN | LPAREN | LBRACE | LBRACK | LBRACK_BAR) ,_) :: CtxtSeqBlock _ :: ((CtxtTypeDefns _ | CtxtLetDecl _ | CtxtMemberBody _ | CtxtWithAsLet _) as limitCtxt) :: _)
-> (limitCtxt.StartPos,limitCtxt.StartCol + 1)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol + 1)

// Permitted inner-construct (e.g. "then" block and "else" block in overall
// "if-then-else" block ) block alighnment:
Expand All @@ -768,34 +780,34 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// elif expr
// else expr
| (CtxtIf _ | CtxtElse _ | CtxtThen _), (CtxtIf _ as limitCtxt) :: _rest
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)
// Permitted inner-construct precise block alighnment:
// while ...
// do expr
// done
| (CtxtDo _), ((CtxtFor _ | CtxtWhile _) as limitCtxt) :: _rest
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)


// These contexts all require indentation by at least one space
| _,((CtxtInterfaceHead _ | CtxtNamespaceHead _ | CtxtModuleHead _ | CtxtException _ | CtxtModuleBody (_,false) | CtxtIf _ | CtxtWithAsLet _ | CtxtLetDecl _ | CtxtMemberHead _ | CtxtMemberBody _) as limitCtxt :: _)
-> (limitCtxt.StartPos,limitCtxt.StartCol + 1)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol + 1)

// These contexts can have their contents exactly aligning
| _,((CtxtParen _ | CtxtFor _ | CtxtWhen _ | CtxtWhile _ | CtxtTypeDefns _ | CtxtMatch _ | CtxtModuleBody (_,true) | CtxtNamespaceBody _ | CtxtTry _ | CtxtMatchClauses _ | CtxtSeqBlock _) as limitCtxt :: _)
-> (limitCtxt.StartPos,limitCtxt.StartCol)
-> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol)

match newCtxt with
// Don't bother to check pushes of Vanilla blocks since we've
// always already pushed a SeqBlock at this position.
| CtxtVanilla _ -> ()
| _ ->
let p1,c1 = unindentationLimit true offsideStack
let p1 = unindentationLimit true offsideStack
let c2 = newCtxt.StartCol
if c2 < c1 then
if c2 < p1.Column then
warn tokenTup
(if debug then (sprintf "possible incorrect indentation: this token is offside of context at position %s, newCtxt = %A, stack = %A, newCtxtPos = %s, c1 = %d, c2 = %d" (warningStringOfPos p1) newCtxt offsideStack (stringOfPos (newCtxt.StartPos)) c1 c2)
else (FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPos p1)) )
(if debug then (sprintf "possible incorrect indentation: this token is offside of context at position %s, newCtxt = %A, stack = %A, newCtxtPos = %s, c1 = %d, c2 = %d" (warningStringOfPos p1.Position) newCtxt offsideStack (stringOfPos (newCtxt.StartPos)) p1.Column c2)
else (FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPos p1.Position)) )
let newOffsideStack = newCtxt :: offsideStack
if debug then dprintf "--> pushing, stack = %A\n" newOffsideStack
offsideStack <- newOffsideStack
Expand Down Expand Up @@ -1042,7 +1054,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
// span of inserted token lasts from the col + 1 of the prev token
// to the beginning of current token
let lastTokenPos =
let pos = snd tokenTup.LastTokenPos
let pos = tokenTup.LastTokenPos.Y
pos.ShiftColumnBy 1
returnToken (lexbufStateForInsertedDummyTokens (lastTokenPos, tokenTup.LexbufState.StartPos)) tok

Expand Down Expand Up @@ -2151,7 +2163,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer,
| PERCENT_OP s -> (s = "%") || (s = "%%")
| _ -> true) &&
nextTokenIsAdjacent tokenTup &&
not (prevWasAtomicEnd && (snd(tokenTup.LastTokenPos) = startPosOfTokenTup tokenTup))) ->
not (prevWasAtomicEnd && (tokenTup.LastTokenPos.Y = startPosOfTokenTup tokenTup))) ->

let plus =
match tokenTup.Token with
Expand Down
102 changes: 57 additions & 45 deletions src/utils/prim-lexing.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,53 +9,65 @@ namespace Internal.Utilities.Text.Lexing
open Microsoft.FSharp.Collections
open System.Collections.Generic

// REVIEW: This type showed up on a parsing-intensive performance measurement.
// REVIEW: Consider whether it can be smaller or can be a struct.
type internal Position =
{ /// The file name index for the position, use fileOfFileIndex in range.fs to decode
posFileIndex: int;
/// The line number for the position
posLineNum: int;
/// The line number for the position in the original source file
posOriginalLineNum : int;
/// The absolute offset of the beginning of the line
posStartOfLineOffset: int;
/// The absolute offset of the column for the position
posColumnOffset: int; }
member x.FileIndex = x.posFileIndex
member x.Line = x.posLineNum
member x.OriginalLine = x.posOriginalLineNum
member x.AbsoluteOffset = x.posColumnOffset
member x.StartOfLine = x.posStartOfLineOffset
member x.StartOfLineAbsoluteOffset = x.posStartOfLineOffset
member x.Column = x.posColumnOffset - x.posStartOfLineOffset
member pos.NextLine =
{ pos with
posOriginalLineNum = pos.OriginalLine + 1;
posLineNum = pos.Line+1;
posStartOfLineOffset = pos.AbsoluteOffset }
member pos.EndOfToken n = {pos with posColumnOffset=pos.posColumnOffset + n }
member pos.ShiftColumnBy by = {pos with posColumnOffset = pos.posColumnOffset + by}
member pos.ColumnMinusOne = { pos with posColumnOffset = pos.posStartOfLineOffset-1 }

member pos.ApplyLineDirective (fileIdx, line) =
{pos with posFileIndex = fileIdx;
posStartOfLineOffset= pos.posColumnOffset;
posLineNum=line };

static member Empty =
{ posFileIndex=0;
posLineNum= 0;
posOriginalLineNum = 0;
posStartOfLineOffset= 0;
posColumnOffset=0 }
[<Struct>]
type internal Position =
val FileIndex: int
val Line: int
val OriginalLine: int
val AbsoluteOffset: int
val StartOfLineAbsoluteOffset: int
member x.Column = x.AbsoluteOffset - x.StartOfLineAbsoluteOffset

new (fileIndex: int, line: int, originalLine: int, startOfLineAbsoluteOffset: int, absoluteOffset: int) =
{ FileIndex = fileIndex
Line = line
OriginalLine = originalLine
AbsoluteOffset = absoluteOffset
StartOfLineAbsoluteOffset = startOfLineAbsoluteOffset }

member x.NextLine =
Position (x.FileIndex,
x.Line + 1,
x.OriginalLine + 1,
x.AbsoluteOffset,
x.AbsoluteOffset)

member x.EndOfToken n =
Position (x.FileIndex,
x.Line,
x.OriginalLine,
x.StartOfLineAbsoluteOffset,
x.AbsoluteOffset + n)

member x.ShiftColumnBy by =
Position (x.FileIndex,
x.Line,
x.OriginalLine,
x.StartOfLineAbsoluteOffset,
x.AbsoluteOffset + by)

member x.ColumnMinusOne =
Position (x.FileIndex,
x.Line,
x.OriginalLine,
x.StartOfLineAbsoluteOffset,
x.StartOfLineAbsoluteOffset - 1)

member x.ApplyLineDirective (fileIdx, line) =
Position (fileIdx,
line,
x.OriginalLine,
x.AbsoluteOffset,
x.AbsoluteOffset)

static member Empty = Position ()

static member FirstLine fileIdx =
{ posFileIndex= fileIdx;
posStartOfLineOffset=0;
posColumnOffset=0;
posOriginalLineNum = 0;
posLineNum=1 }
Position (fileIdx,
1,
0,
0,
0)

type internal LexBufferFiller<'Char> = (LexBuffer<'Char> -> unit)

Expand Down
13 changes: 6 additions & 7 deletions src/utils/prim-lexing.fsi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,20 @@ open Microsoft.FSharp.Core
open Microsoft.FSharp.Control

/// Position information stored for lexing tokens
[<Sealed>]
[<Struct>]
type internal Position =
interface System.IComparable
/// The file index for the file associated with the input stream, use fileOfFileIndex in range.fs to decode
member FileIndex : int
val FileIndex : int
/// The line number in the input stream, assuming fresh positions have been updated
/// for the new line by modifying the EndPos property of the LexBuffer.
member Line : int
val Line : int
/// The line number for the position in the input stream, assuming fresh positions have been updated
/// using for the new line
member OriginalLine : int
val OriginalLine : int
/// The character number in the input stream
member AbsoluteOffset : int
val AbsoluteOffset : int
/// Return absolute offset of the start of the line marked by the position
member StartOfLineAbsoluteOffset : int
val StartOfLineAbsoluteOffset : int
/// Return the column number marked by the position, i.e. the difference between the AbsoluteOffset and the StartOfLineAbsoluteOffset
member Column : int
// Given a position just beyond the end of a line, return a position at the start of the next line
Expand Down