add --skip-line-directives optional flag to fslex and fsyacc.

This is to work around colourization issue in VS in fsharp solution (dotnet/fsharp#6400)

FsLexYacc.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.28307.438
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.28803.352
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{1F1B4F0F-2998-4D74-865B-9122611C2B14}"
 	ProjectSection(SolutionItems) = preProject
@@ -12,13 +12,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{1F1B4F
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{A6A6AF7D-D6E3-442D-9B1E-58CC91879BE1}"
 EndProject
-Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FsLex", "src\FsLex\fslex.fsproj", "{BD2284A5-AA4D-442D-B4FB-E43B2FE9DD2A}"
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fslex", "src\FsLex\fslex.fsproj", "{BD2284A5-AA4D-442D-B4FB-E43B2FE9DD2A}"
 EndProject
-Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FsYacc", "src\FsYacc\fsyacc.fsproj", "{DDD90630-1CDA-4CB3-9A0A-6A1253478C2D}"
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsyacc", "src\FsYacc\fsyacc.fsproj", "{DDD90630-1CDA-4CB3-9A0A-6A1253478C2D}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "project", "project", "{BF60BC93-E09B-4E5F-9D85-95A519479D54}"
 	ProjectSection(SolutionItems) = preProject
+		build.cmd = build.cmd
 		build.fsx = build.fsx
+		build.sh = build.sh
 		nuget\FsLexYacc.nuspec = nuget\FsLexYacc.nuspec
 		nuget\FsLexYacc.Runtime.nuspec = nuget\FsLexYacc.Runtime.nuspec
 		README.md = README.md

src/FsLex/fslex.fs

@@ -44,7 +44,7 @@ let inputCodePage = ref None
 let light = ref None
 let modname = ref None
 let internal_module = ref false
-
+let skipLineDirectives = ref false
 let mutable lexlib = "FSharp.Text.Lexing"
 
 let usage =
@@ -56,6 +56,7 @@ let usage =
     ArgInfo ("--light-off", ArgType.Unit (fun () ->  light := Some false), "Add #light \"off\" to the top of the generated file")
     ArgInfo ("--lexlib", ArgType.String (fun s ->  lexlib <- s), "Specify the namespace for the implementation of the lexer table interpreter (default FSharp.Text.Lexing)")
     ArgInfo ("--unicode", ArgType.Set unicode, "Produce a lexer for use with 16-bit unicode characters.")  
+    ArgInfo ("--skip-line-directives", ArgType.Unit (fun () -> skipLineDirectives := true), "Skip line directives from output (workaround colorization bug)")
   ]
 
 let _ = ArgParser.Parse(usage, (fun x -> match !input with Some _ -> failwith "more than one input given" | None -> input := Some x), "fslex <filename>")
@@ -71,6 +72,16 @@ let sentinel = 255 * 256 + 255
 let lineCount = ref 0
 let cfprintfn (os: #TextWriter) fmt = Printf.kfprintf (fun () -> incr lineCount; os.WriteLine()) os fmt
 
+let printLineDirectiveIfNeededForPosition (os: #TextWriter) pos=
+  if !skipLineDirectives then ()
+  else 
+    cfprintfn os "%s" (Position.MakeLineDirective pos)
+
+let printLineDirectiveIfNeededForLineCountAndFile (os: #TextWriter) lineCount fileName =
+  if !skipLineDirectives then ()
+  else 
+    cfprintfn os "%s" (Position.MakeSimpleLineDirective lineCount fileName)
+
 let main() = 
   try 
     let filename = (match !input with Some x -> x | None -> failwith "no input given") 
@@ -112,14 +123,15 @@ let main() =
 
     let printLinesIfCodeDefined (code,pos:Position) =
         if pos <> Position.Empty  // If bottom code is unspecified, then position is empty.        
-        then 
-            cfprintfn os "%s" (Position.MakeLineDirective pos)
+        then
+            printLineDirectiveIfNeededForPosition os pos
             cfprintfn os "%s" code
 
     printLinesIfCodeDefined spec.TopCode
     let code = fst spec.TopCode
     lineCount := !lineCount + code.Replace("\r","").Split([| '\n' |]).Length
-    cfprintfn os "%s" (Position.MakeSimpleLineDirective !lineCount output)
+
+    printLineDirectiveIfNeededForLineCountAndFile os !lineCount output
 
     cfprintfn os "let trans : uint16[] array = "
     cfprintfn os "    [| "
@@ -210,19 +222,20 @@ let main() =
         cfprintfn os "  match _fslex_tables.Interpret(%d,lexbuf) with" startNode.Id
         actions |> Seq.iteri (fun i (code:string, pos) -> 
             cfprintfn os "  | %d -> ( " i
-            cfprintfn os "%s" (Position.MakeLineDirective pos)
+            printLineDirectiveIfNeededForPosition os pos
+
             let lines = code.Split([| '\r'; '\n' |], StringSplitOptions.RemoveEmptyEntries)
             for line in lines do
                 cfprintfn os "               %s" line
-            cfprintfn os "%s" (Position.MakeSimpleLineDirective !lineCount output)
+            printLineDirectiveIfNeededForLineCountAndFile os !lineCount output
             cfprintfn os "          )")
         cfprintfn os "  | _ -> failwith \"%s\"" ident
 
 
     cfprintfn os ""
 
     printLinesIfCodeDefined spec.BottomCode
-    cfprintfn os "%s" (Position.MakeSimpleLineDirective 3000000 output)
+    printLineDirectiveIfNeededForLineCountAndFile os 3000000 output
 
   with e -> 
     eprintf "FSLEX: error FSL000: %s" (match e with Failure s -> s | e -> e.ToString())

src/FsYacc/fsyacc.fs

@@ -49,7 +49,7 @@ let light = ref None
 let inputCodePage = ref None
 let mutable lexlib = "FSharp.Text.Lexing"
 let mutable parslib = "FSharp.Text.Parsing"
-
+let mutable skipLineDirectives = false
 let usage =
   [ ArgInfo("-o", ArgType.String (fun s -> out := Some s), "Name the output file.");
     ArgInfo("-v", ArgType.Unit (fun () -> log := true), "Produce a listing file."); 
@@ -62,7 +62,9 @@ let usage =
     ArgInfo("--tokens", ArgType.Set tokenize, "Simply tokenize the specification file itself."); 
     ArgInfo("--lexlib", ArgType.String (fun s ->  lexlib <- s), "Specify the namespace for the implementation of the lexer (default: FSharp.Text.Lexing)");
     ArgInfo("--parslib", ArgType.String (fun s ->  parslib <- s), "Specify the namespace for the implementation of the parser table interpreter (default: FSharp.Text.Parsing)");
-    ArgInfo("--codepage", ArgType.Int (fun i -> inputCodePage := Some i), "Assume input lexer specification file is encoded with the given codepage.");  ]
+    ArgInfo("--codepage", ArgType.Int (fun i -> inputCodePage := Some i), "Assume input lexer specification file is encoded with the given codepage.");  
+    ArgInfo ("--skip-line-directives", ArgType.Unit (fun () -> skipLineDirectives <- true), "Skip line directives from output (workaround colorization bug)")
+  ]
 
 let _ = ArgParser.Parse(usage,(fun x -> match !input with Some _ -> failwith "more than one input given" | None -> input := Some x),"fsyacc <filename>")
 
@@ -135,6 +137,18 @@ let main() =
   let cprintf (os:TextWriter,lineCount) fmt = Printf.fprintf os fmt
   let cprintfn (os:TextWriter,lineCount) fmt = Printf.kfprintf (fun () -> incr lineCount; os.WriteLine()) os fmt
 
+
+  let printLineDirectiveIfNeededForPosition cos pos =
+    if skipLineDirectives then ()
+    else 
+      cprintfn cos "%s" (Position.MakeLineDirective pos)
+
+  let printLineDirectiveIfNeededForLineCountAndFile cos lineCount fileName =
+    if skipLineDirectives then ()
+    else 
+      cprintfn cos "%s" (Position.MakeSimpleLineDirective lineCount fileName)
+
+
   let logf = 
       match outputo with 
       | None -> (fun f -> ())
@@ -189,12 +203,11 @@ let main() =
   cprintfn cos "open %s.ParseHelpers" parslib;
   if !compat then 
       cprintfn cos "open Microsoft.FSharp.Compatibility.OCaml.Parsing";
-
-  cprintfn cos "# %d \"%s\"" pos.pos_lnum pos.pos_fname;
+  printLineDirectiveIfNeededForPosition cos pos;
   cprintfn cos "%s" code;
   lineCountOutput := !lineCountOutput + code.Replace("\r","").Split([| '\n' |]).Length;
 
-  cprintfn cos "# %d \"%s\"" !lineCountOutput output;
+  printLineDirectiveIfNeededForLineCountAndFile cos !lineCountOutput output;
   // Print the datatype for the tokens
   cprintfn cos "// This type is the type of tokens accepted by the parser";
   for out in [cos;cosi] do
@@ -440,7 +453,7 @@ let main() =
       cprintf cos "let _fsyacc_reductions ()  =" ;
       cprintfn cos "    [| " ;
       for nt,ntIdx,syms,code in prods do 
-          cprintfn cos "# %d \"%s\"" !lineCountOutput output;
+          printLineDirectiveIfNeededForLineCountAndFile cos !lineCountOutput output;
           cprintfn cos "        (fun (parseState : %s.IParseState) ->"  parslib
           if !compat then 
               cprintfn cos "            Parsing.set_parse_state parseState;"
@@ -459,7 +472,7 @@ let main() =
           cprintfn cos "                (";
           cprintfn cos "                   (";
           match code with 
-          | Some (_,pos) -> cprintfn cos "# %d \"%s\"" pos.pos_lnum pos.pos_fname
+          | Some (_,pos) -> printLineDirectiveIfNeededForPosition cos pos
           | None -> ()
           match code with 
           | Some (code,_) -> 
@@ -478,13 +491,13 @@ let main() =
           cprintfn cos "                   )";
           // Place the line count back for the type constraint
           match code with 
-          | Some (_,pos) -> cprintfn cos "# %d \"%s\"" pos.pos_lnum pos.pos_fname
+          | Some (_,pos) -> printLineDirectiveIfNeededForPosition cos pos
           | None -> ()
           cprintfn cos "                 : %s));" (if types.ContainsKey nt then  types.[nt] else "'"+nt);
       done;
       cprintfn cos "|]" ;
   end;
-  cprintfn cos "# %d \"%s\"" !lineCountOutput output;
+  printLineDirectiveIfNeededForLineCountAndFile cos !lineCountOutput output;
   cprintfn cos "let tables () : %s.Tables<_> = " parslib
   cprintfn cos "  { reductions= _fsyacc_reductions ();"
   cprintfn cos "    endOfInputTag = _fsyacc_endOfInputTag;"