From 0e3e471253ffde38960a14adf1c498dc18870940 Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Thu, 1 Oct 2015 23:20:14 +0300 Subject: [PATCH] make Internal.Utilities.Text.Lexing.Position a struct use a structs instead of couple Tuples in LexFilter --- src/fsharp/LexFilter.fs | 54 ++++++++++++-------- src/utils/prim-lexing.fs | 102 +++++++++++++++++++++----------------- src/utils/prim-lexing.fsi | 13 +++-- 3 files changed, 96 insertions(+), 73 deletions(-) diff --git a/src/fsharp/LexFilter.fs b/src/fsharp/LexFilter.fs index 06a8374a2db..0b8fcade1be 100644 --- a/src/fsharp/LexFilter.fs +++ b/src/fsharp/LexFilter.fs @@ -403,12 +403,18 @@ type LexbufState(startPos: Position, member x.EndPos = endPos member x.PastEOF = pastEOF +[] +type PositionTuple = + val X: Position + val Y: Position + new (x: Position, y: Position) = { X = x; Y = y } + /// Used to save the state related to a token [] type TokenTup = val Token : token val LexbufState : LexbufState - val LastTokenPos: Position * Position + val LastTokenPos: PositionTuple new (token,state,lastTokenPos) = { Token=token; LexbufState=state;LastTokenPos=lastTokenPos } /// Returns starting position of the token @@ -485,6 +491,12 @@ let (|TyparsCloseOp|_|) (txt:string) = | _ -> None Some([| for _c in angles do yield GREATER |],afterOp) +[] +type PositionWithColumn = + val Position: Position + val Column: int + new (position: Position, column: int) = { Position = position; Column = column } + //---------------------------------------------------------------------------- // build a LexFilter //--------------------------------------------------------------------------*) @@ -553,7 +565,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, let tokenLexbufState = getLexbufState() savedLexbufState <- tokenLexbufState haveLexbufState <- true - TokenTup(token,tokenLexbufState,(lastTokenStart,lastTokenEnd)) + TokenTup(token,tokenLexbufState,PositionTuple(lastTokenStart,lastTokenEnd)) //---------------------------------------------------------------------------- // Fetch a raw token, either from the old lexer or from our delayedStack @@ -623,7 +635,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, let pushCtxt tokenTup (newCtxt:Context) = let rec unindentationLimit strict stack = match newCtxt,stack with - | _, [] -> (newCtxt.StartPos, -1) + | _, [] -> PositionWithColumn(newCtxt.StartPos, -1) // ignore Vanilla because a SeqBlock is always coming | _, (CtxtVanilla _ :: rest) -> unindentationLimit strict rest @@ -635,8 +647,8 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // '(match' limited by minimum of two | _,(((CtxtMatch _) as ctxt1) :: CtxtSeqBlock _ :: (CtxtParen ((BEGIN | LPAREN),_) as ctxt2) :: _rest) -> if ctxt1.StartCol <= ctxt2.StartCol - then (ctxt1.StartPos,ctxt1.StartCol) - else (ctxt2.StartPos,ctxt2.StartCol) + then PositionWithColumn(ctxt1.StartPos,ctxt1.StartCol) + else PositionWithColumn(ctxt2.StartPos,ctxt2.StartCol) // 'let ... = function' limited by 'let', precisely // This covers the common form @@ -645,7 +657,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // | Case1 -> ... // | Case2 -> ... | (CtxtMatchClauses _), (CtxtFunction _ :: CtxtSeqBlock _ :: (CtxtLetDecl _ as limitCtxt) :: _rest) - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) // Otherwise 'function ...' places no limit until we hit a CtxtLetDecl etc... (Recursive) | (CtxtMatchClauses _), (CtxtFunction _ :: rest) @@ -653,7 +665,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // 'try ... with' limited by 'try' | _,(CtxtMatchClauses _ :: (CtxtTry _ as limitCtxt) :: _rest) - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) // 'fun ->' places no limit until we hit a CtxtLetDecl etc... (Recursive) | _,(CtxtFun _ :: rest) @@ -672,7 +684,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // This is a serious thing to allow, but is required since there is no "return" in this language. // Without it there is no way of escaping special cases in large bits of code without indenting the main case. | CtxtSeqBlock _, (CtxtElse _ :: (CtxtIf _ as limitCtxt) :: _rest) - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) // Permitted inner-construct precise block alighnment: // interface ... @@ -683,7 +695,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // with ... // end | CtxtWithAsAugment _,((CtxtInterfaceHead _ | CtxtMemberHead _ | CtxtException _ | CtxtTypeDefns _) as limitCtxt :: _rest) - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) // Permit unindentation via parentheses (or begin/end) following a 'then', 'else' or 'do': // if nr > 0 then ( @@ -754,12 +766,12 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // 'type C = interface ... ' limited by 'type' // 'type C = struct ... ' limited by 'type' | _,(CtxtParen ((CLASS | STRUCT | INTERFACE),_) :: CtxtSeqBlock _ :: (CtxtTypeDefns _ as limitCtxt) :: _) - -> (limitCtxt.StartPos,limitCtxt.StartCol + 1) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol + 1) // REVIEW: document these | _,(CtxtSeqBlock _ :: CtxtParen((BEGIN | LPAREN | LBRACK | LBRACK_BAR),_) :: CtxtVanilla _ :: (CtxtSeqBlock _ as limitCtxt) :: _) | (CtxtSeqBlock _),(CtxtParen ((BEGIN | LPAREN | LBRACE | LBRACK | LBRACK_BAR) ,_) :: CtxtSeqBlock _ :: ((CtxtTypeDefns _ | CtxtLetDecl _ | CtxtMemberBody _ | CtxtWithAsLet _) as limitCtxt) :: _) - -> (limitCtxt.StartPos,limitCtxt.StartCol + 1) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol + 1) // Permitted inner-construct (e.g. "then" block and "else" block in overall // "if-then-else" block ) block alighnment: @@ -768,34 +780,34 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // elif expr // else expr | (CtxtIf _ | CtxtElse _ | CtxtThen _), (CtxtIf _ as limitCtxt) :: _rest - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) // Permitted inner-construct precise block alighnment: // while ... // do expr // done | (CtxtDo _), ((CtxtFor _ | CtxtWhile _) as limitCtxt) :: _rest - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) // These contexts all require indentation by at least one space | _,((CtxtInterfaceHead _ | CtxtNamespaceHead _ | CtxtModuleHead _ | CtxtException _ | CtxtModuleBody (_,false) | CtxtIf _ | CtxtWithAsLet _ | CtxtLetDecl _ | CtxtMemberHead _ | CtxtMemberBody _) as limitCtxt :: _) - -> (limitCtxt.StartPos,limitCtxt.StartCol + 1) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol + 1) // These contexts can have their contents exactly aligning | _,((CtxtParen _ | CtxtFor _ | CtxtWhen _ | CtxtWhile _ | CtxtTypeDefns _ | CtxtMatch _ | CtxtModuleBody (_,true) | CtxtNamespaceBody _ | CtxtTry _ | CtxtMatchClauses _ | CtxtSeqBlock _) as limitCtxt :: _) - -> (limitCtxt.StartPos,limitCtxt.StartCol) + -> PositionWithColumn(limitCtxt.StartPos,limitCtxt.StartCol) match newCtxt with // Don't bother to check pushes of Vanilla blocks since we've // always already pushed a SeqBlock at this position. | CtxtVanilla _ -> () | _ -> - let p1,c1 = unindentationLimit true offsideStack + let p1 = unindentationLimit true offsideStack let c2 = newCtxt.StartCol - if c2 < c1 then + if c2 < p1.Column then warn tokenTup - (if debug then (sprintf "possible incorrect indentation: this token is offside of context at position %s, newCtxt = %A, stack = %A, newCtxtPos = %s, c1 = %d, c2 = %d" (warningStringOfPos p1) newCtxt offsideStack (stringOfPos (newCtxt.StartPos)) c1 c2) - else (FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPos p1)) ) + (if debug then (sprintf "possible incorrect indentation: this token is offside of context at position %s, newCtxt = %A, stack = %A, newCtxtPos = %s, c1 = %d, c2 = %d" (warningStringOfPos p1.Position) newCtxt offsideStack (stringOfPos (newCtxt.StartPos)) p1.Column c2) + else (FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPos p1.Position)) ) let newOffsideStack = newCtxt :: offsideStack if debug then dprintf "--> pushing, stack = %A\n" newOffsideStack offsideStack <- newOffsideStack @@ -1042,7 +1054,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, // span of inserted token lasts from the col + 1 of the prev token // to the beginning of current token let lastTokenPos = - let pos = snd tokenTup.LastTokenPos + let pos = tokenTup.LastTokenPos.Y pos.ShiftColumnBy 1 returnToken (lexbufStateForInsertedDummyTokens (lastTokenPos, tokenTup.LexbufState.StartPos)) tok @@ -2151,7 +2163,7 @@ type LexFilterImpl (lightSyntaxStatus:LightSyntaxStatus, compilingFsLib, lexer, | PERCENT_OP s -> (s = "%") || (s = "%%") | _ -> true) && nextTokenIsAdjacent tokenTup && - not (prevWasAtomicEnd && (snd(tokenTup.LastTokenPos) = startPosOfTokenTup tokenTup))) -> + not (prevWasAtomicEnd && (tokenTup.LastTokenPos.Y = startPosOfTokenTup tokenTup))) -> let plus = match tokenTup.Token with diff --git a/src/utils/prim-lexing.fs b/src/utils/prim-lexing.fs index 33e449405f1..9de2ed41863 100644 --- a/src/utils/prim-lexing.fs +++ b/src/utils/prim-lexing.fs @@ -9,53 +9,65 @@ namespace Internal.Utilities.Text.Lexing open Microsoft.FSharp.Collections open System.Collections.Generic - // REVIEW: This type showed up on a parsing-intensive performance measurement. - // REVIEW: Consider whether it can be smaller or can be a struct. - type internal Position = - { /// The file name index for the position, use fileOfFileIndex in range.fs to decode - posFileIndex: int; - /// The line number for the position - posLineNum: int; - /// The line number for the position in the original source file - posOriginalLineNum : int; - /// The absolute offset of the beginning of the line - posStartOfLineOffset: int; - /// The absolute offset of the column for the position - posColumnOffset: int; } - member x.FileIndex = x.posFileIndex - member x.Line = x.posLineNum - member x.OriginalLine = x.posOriginalLineNum - member x.AbsoluteOffset = x.posColumnOffset - member x.StartOfLine = x.posStartOfLineOffset - member x.StartOfLineAbsoluteOffset = x.posStartOfLineOffset - member x.Column = x.posColumnOffset - x.posStartOfLineOffset - member pos.NextLine = - { pos with - posOriginalLineNum = pos.OriginalLine + 1; - posLineNum = pos.Line+1; - posStartOfLineOffset = pos.AbsoluteOffset } - member pos.EndOfToken n = {pos with posColumnOffset=pos.posColumnOffset + n } - member pos.ShiftColumnBy by = {pos with posColumnOffset = pos.posColumnOffset + by} - member pos.ColumnMinusOne = { pos with posColumnOffset = pos.posStartOfLineOffset-1 } - - member pos.ApplyLineDirective (fileIdx, line) = - {pos with posFileIndex = fileIdx; - posStartOfLineOffset= pos.posColumnOffset; - posLineNum=line }; - - static member Empty = - { posFileIndex=0; - posLineNum= 0; - posOriginalLineNum = 0; - posStartOfLineOffset= 0; - posColumnOffset=0 } + [] + type internal Position = + val FileIndex: int + val Line: int + val OriginalLine: int + val AbsoluteOffset: int + val StartOfLineAbsoluteOffset: int + member x.Column = x.AbsoluteOffset - x.StartOfLineAbsoluteOffset + + new (fileIndex: int, line: int, originalLine: int, startOfLineAbsoluteOffset: int, absoluteOffset: int) = + { FileIndex = fileIndex + Line = line + OriginalLine = originalLine + AbsoluteOffset = absoluteOffset + StartOfLineAbsoluteOffset = startOfLineAbsoluteOffset } + + member x.NextLine = + Position (x.FileIndex, + x.Line + 1, + x.OriginalLine + 1, + x.AbsoluteOffset, + x.AbsoluteOffset) + + member x.EndOfToken n = + Position (x.FileIndex, + x.Line, + x.OriginalLine, + x.StartOfLineAbsoluteOffset, + x.AbsoluteOffset + n) + + member x.ShiftColumnBy by = + Position (x.FileIndex, + x.Line, + x.OriginalLine, + x.StartOfLineAbsoluteOffset, + x.AbsoluteOffset + by) + + member x.ColumnMinusOne = + Position (x.FileIndex, + x.Line, + x.OriginalLine, + x.StartOfLineAbsoluteOffset, + x.StartOfLineAbsoluteOffset - 1) + + member x.ApplyLineDirective (fileIdx, line) = + Position (fileIdx, + line, + x.OriginalLine, + x.AbsoluteOffset, + x.AbsoluteOffset) + + static member Empty = Position () static member FirstLine fileIdx = - { posFileIndex= fileIdx; - posStartOfLineOffset=0; - posColumnOffset=0; - posOriginalLineNum = 0; - posLineNum=1 } + Position (fileIdx, + 1, + 0, + 0, + 0) type internal LexBufferFiller<'Char> = (LexBuffer<'Char> -> unit) diff --git a/src/utils/prim-lexing.fsi b/src/utils/prim-lexing.fsi index 5ed7da699f7..9fb98d21662 100644 --- a/src/utils/prim-lexing.fsi +++ b/src/utils/prim-lexing.fsi @@ -10,21 +10,20 @@ open Microsoft.FSharp.Core open Microsoft.FSharp.Control /// Position information stored for lexing tokens -[] +[] type internal Position = - interface System.IComparable /// The file index for the file associated with the input stream, use fileOfFileIndex in range.fs to decode - member FileIndex : int + val FileIndex : int /// The line number in the input stream, assuming fresh positions have been updated /// for the new line by modifying the EndPos property of the LexBuffer. - member Line : int + val Line : int /// The line number for the position in the input stream, assuming fresh positions have been updated /// using for the new line - member OriginalLine : int + val OriginalLine : int /// The character number in the input stream - member AbsoluteOffset : int + val AbsoluteOffset : int /// Return absolute offset of the start of the line marked by the position - member StartOfLineAbsoluteOffset : int + val StartOfLineAbsoluteOffset : int /// Return the column number marked by the position, i.e. the difference between the AbsoluteOffset and the StartOfLineAbsoluteOffset member Column : int // Given a position just beyond the end of a line, return a position at the start of the next line