Changed LexBuffer from char to uint16

src/absil/illex.fsl

@@ -16,7 +16,7 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.AsciiParser
 open Microsoft.FSharp.Compiler.AbstractIL.Internal.AsciiConstants 
 
 
-let lexeme (lexbuf : LexBuffer<char>) = new System.String(lexbuf.Lexeme)
+let lexeme (lexbuf : LexBuffer<uint16>) = new System.String(lexbuf.Lexeme |> Array.map char)
 
 let unexpectedChar _lexbuf =
   raise Parsing.RecoverableParseError ;;

src/fsharp/UnicodeLexing.fs

@@ -12,12 +12,12 @@ open System.IO
 
 open Internal.Utilities.Text.Lexing
 
-type Lexbuf =  LexBuffer<char>
+type Lexbuf =  LexBuffer<uint16>
 
 let StringAsLexbuf (s:string) : Lexbuf =
     LexBuffer<_>.FromChars (s.ToCharArray())
 
-let FunctionAsLexbuf (bufferFiller: char[] * int * int -> int) : Lexbuf =
+let FunctionAsLexbuf (bufferFiller: uint16[] * int * int -> int) : Lexbuf =
     LexBuffer<_>.FromFunction bufferFiller 
 
 #if !FABLE_COMPILER

src/fsharp/UnicodeLexing.fsi

@@ -5,9 +5,9 @@ module internal Microsoft.FSharp.Compiler.UnicodeLexing
 open Microsoft.FSharp.Text
 open Internal.Utilities.Text.Lexing
 
-type Lexbuf =  LexBuffer<char>
+type Lexbuf =  LexBuffer<uint16>
 val internal StringAsLexbuf : string -> Lexbuf
-val public FunctionAsLexbuf : (char [] * int * int -> int) -> Lexbuf
+val public FunctionAsLexbuf : (uint16[] * int * int -> int) -> Lexbuf
 
 #if !FABLE_COMPILER
 val public UnicodeFileAsLexbuf :string * int option * (*retryLocked*) bool -> Lexbuf

src/fsharp/fsi/fsi.fs

@@ -1735,7 +1735,7 @@ type internal FsiStdinLexerProvider
 
     let LexbufFromLineReader (fsiStdinSyphon: FsiStdinSyphon) readf = 
         UnicodeLexing.FunctionAsLexbuf 
-          (fun (buf: char[], start, len) -> 
+          (fun (buf: uint16[], start, len) -> 
             //fprintf fsiConsoleOutput.Out "Calling ReadLine\n"
             let inputOption = try Some(readf()) with :? EndOfStreamException -> None
             inputOption |> Option.iter (fun t -> fsiStdinSyphon.Add (t + "\n"))
@@ -1749,7 +1749,7 @@ type internal FsiStdinLexerProvider
                 if ninput > len then fprintf fsiConsoleOutput.Error  "%s" (FSIstrings.SR.fsiLineTooLong())
                 let ntrimmed = min len ninput 
                 for i = 0 to ntrimmed-1 do
-                    buf.[i+start] <- input.[i]
+                    buf.[i+start] <- uint16 input.[i]
                 ntrimmed
         )
 

src/fsharp/lex.fsl

@@ -161,7 +161,7 @@ let shouldStartFile args lexbuf (m:range) err tok =
     else tok
 
 let evalIfDefExpression startPos args (lookup:string->bool) (lexed:string) =
-    let lexbuf          = LexBuffer<char>.FromChars (lexed.ToCharArray ())
+    let lexbuf          = LexBuffer<uint16>.FromChars (lexed.ToCharArray())
     lexbuf.StartPos     <- startPos
     lexbuf.EndPos       <- startPos
     let tokenStream     = Microsoft.FSharp.Compiler.PPLexer.tokenstream args

src/fsharp/service/ServiceLexing.fsi

@@ -227,7 +227,7 @@ type FSharpLineTokenizer =
 type FSharpSourceTokenizer =
     new : conditionalDefines:string list * fileName:string option -> FSharpSourceTokenizer
     member CreateLineTokenizer : lineText:string -> FSharpLineTokenizer
-    member CreateBufferTokenizer : bufferFiller:(char[] * int * int -> int) -> FSharpLineTokenizer
+    member CreateBufferTokenizer : bufferFiller:(uint16[] * int * int -> int) -> FSharpLineTokenizer
 
 
 module internal TestExpose =     

src/utils/prim-lexing.fs

@@ -130,8 +130,7 @@ namespace Internal.Utilities.Text.Lexing
         member lexbuf.BufferScanStart     with get() : int = bufferScanStart     and set v = bufferScanStart <- v
         member lexbuf.BufferAcceptAction  with get() = bufferAcceptAction  and set v = bufferAcceptAction <- v
         member lexbuf.RefillBuffer () = filler lexbuf
-        static member LexemeString(lexbuf:LexBuffer<char>) = 
-            new System.String(lexbuf.Buffer,lexbuf.BufferScanStart,lexbuf.LexemeLength)
+        static member LexemeString (lexbuf: LexBuffer<uint16>) = System.String(lexbuf.Lexeme |> Array.map char)
 
         member lexbuf.IsPastEndOfStream 
            with get() = eof
@@ -170,17 +169,17 @@ namespace Internal.Utilities.Text.Lexing
             LexBuffer<'Char>.FromArrayNoCopy buffer
 
         // Important: This method takes ownership of the array
-        static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy arr 
+        static member FromChars (arr:char[]) = LexBuffer.FromArrayNoCopy (arr |> Array.map uint16)
 
     module internal GenericImplFragments = 
-        let startInterpret(lexBuffer:LexBuffer<char>)= 
+        let startInterpret(lexBuffer:LexBuffer<uint16>)= 
             lexBuffer.BufferScanStart <- lexBuffer.BufferScanStart + lexBuffer.LexemeLength;
             lexBuffer.BufferMaxScanLength <- lexBuffer.BufferMaxScanLength - lexBuffer.LexemeLength;
             lexBuffer.BufferScanLength <- 0;
             lexBuffer.LexemeLength <- 0;
             lexBuffer.BufferAcceptAction <- -1;
 
-        let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer<char>,scanUntilSentinel,endOfScan,state,eofPos) = 
+        let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer<uint16>,scanUntilSentinel,endOfScan,state,eofPos) = 
             // end of file occurs if we couldn't extend the buffer 
             if lexBuffer.BufferScanLength = lexBuffer.BufferMaxScanLength then  
                 let snew = int trans.[state].[eofPos] // == EOF 
@@ -194,9 +193,9 @@ namespace Internal.Utilities.Text.Lexing
             else 
                 scanUntilSentinel lexBuffer state
 
-        let onAccept (lexBuffer:LexBuffer<char>,a) = 
-            lexBuffer.LexemeLength <- lexBuffer.BufferScanLength;
-            lexBuffer.BufferAcceptAction <- a;
+        let onAccept (lexBuffer:LexBuffer<uint16>, a) = 
+            lexBuffer.LexemeLength <- lexBuffer.BufferScanLength
+            lexBuffer.BufferAcceptAction <- a
 
     open GenericImplFragments
 
@@ -223,15 +222,15 @@ namespace Internal.Utilities.Text.Lexing
                         let baseForUnicodeCategories = numLowUnicodeChars+numSpecificUnicodeChars*2
                         let unicodeCategory = 
 #if FX_RESHAPED_GLOBALIZATION
-                            System.Globalization.CharUnicodeInfo.GetUnicodeCategory(inp)
+                            System.Globalization.CharUnicodeInfo.GetUnicodeCategory(char inp)
 #else
-                            System.Char.GetUnicodeCategory(inp)
+                            System.Char.GetUnicodeCategory(char inp)
 #endif
                         //System.Console.WriteLine("inp = {0}, unicodeCategory = {1}", [| box inp; box unicodeCategory |]);
                         int trans.[state].[baseForUnicodeCategories + int32 unicodeCategory]
                     else 
                         // This is the specific unicode character
-                        let c = char (int trans.[state].[baseForSpecificUnicodeChars+i*2])
+                        let c = trans.[state].[baseForSpecificUnicodeChars+i*2]
                         //System.Console.WriteLine("c = {0}, inp = {1}, i = {2}", [| box c; box inp; box i |]);
                         // OK, have we found the entry for a specific unicode character?
                         if c = inp
@@ -272,7 +271,7 @@ namespace Internal.Utilities.Text.Lexing
         //      30 entries, one for each UnicodeCategory
         //      1 entry for EOF
 
-        member tables.Interpret(initialState,lexBuffer : LexBuffer<char>) = 
+        member tables.Interpret(initialState, lexBuffer: LexBuffer<uint16>) = 
             startInterpret(lexBuffer)
             scanUntilSentinel lexBuffer initialState
 

src/utils/prim-lexing.fsi

@@ -56,7 +56,7 @@ type internal LexBuffer<'Char> =
     member Lexeme: 'Char []
 
     /// Fast helper to turn the matched characters into a string, avoiding an intermediate array.
-    static member LexemeString : LexBuffer<char> -> string
+    static member LexemeString : LexBuffer<uint16> -> string
 
     /// Dynamically typed, non-lexically scoped parameter table.
     member BufferLocalStore : IDictionary<string,obj>
@@ -66,7 +66,7 @@ type internal LexBuffer<'Char> =
 
     /// Create a lex buffer suitable for Unicode lexing that reads characters from the given array.
     /// Important: does take ownership of the array.
-    static member FromChars: char[] -> LexBuffer<char>
+    static member FromChars: char[] -> LexBuffer<uint16>
     /// Create a lex buffer that reads character or byte inputs by using the given function.
     static member FromFunction: ('Char[] * int * int -> int) -> LexBuffer<'Char>
 
@@ -75,5 +75,5 @@ type internal LexBuffer<'Char> =
 type internal UnicodeTables =
     static member Create : uint16[][] * uint16[] -> UnicodeTables
     /// Interpret tables for a unicode lexer generated by <c>fslex.exe</c>. 
-    member Interpret:  initialState:int * LexBuffer<char> -> int
+    member Interpret:  initialState:int * LexBuffer<uint16> -> int
 

src/utils/prim-parsing.fs

@@ -11,7 +11,7 @@ exception RecoverableParseError
 exception Accept of obj
 
 [<Sealed>]
-type internal IParseState(ruleStartPoss:Position[],ruleEndPoss:Position[],lhsPos:Position[],ruleValues:obj[],lexbuf:LexBuffer<char>) = 
+type internal IParseState(ruleStartPoss:Position[],ruleEndPoss:Position[],lhsPos:Position[],ruleValues:obj[],lexbuf:LexBuffer<uint16>) = 
     member p.LexBuffer = lexbuf
     member p.InputRange n = ruleStartPoss.[n-1], ruleEndPoss.[n-1]; 
     member p.InputStartPosition n = ruleStartPoss.[n-1]

src/utils/prim-parsing.fsi

@@ -36,7 +36,7 @@ type internal IParseState =
     /// Raise an error in this parse context.
     member RaiseError<'b> : unit -> 'b 
     /// Return the LexBuffer for this parser instance.
-    member LexBuffer : LexBuffer<char>
+    member LexBuffer : LexBuffer<uint16>
 
 
 [<Sealed>]
@@ -96,7 +96,7 @@ type internal Tables<'tok> =
 
     /// Interpret the parser table taking input from the given lexer, using the given lex buffer, and the given start state.
     /// Returns an object indicating the final synthesized value for the parse.
-    member Interpret :  lexer:(LexBuffer<char> -> 'tok) * lexbuf:LexBuffer<char> * startState:int -> obj 
+    member Interpret :  lexer:(LexBuffer<uint16> -> 'tok) * lexbuf:LexBuffer<uint16> * startState:int -> obj 
 
 /// Indicates an accept action has occurred.
 exception internal Accept of obj