Exclude defines found in multiline strings. Fixes fsprojects#761

src/Fantomas.Tests/CompilerDirectivesTests.fs

@@ -1437,12 +1437,14 @@ let b = \"\"\"
 \"\"\"
 "     config)
     |> prepend newline
-    |> should equal """
-let a = "
+    |> should equal "
+let a = \"\"\"
+\"
 #if FOO
-"
+\"
+\"\"\"
 
-let b = "
+let b = \"\"\"
 #endif
-"
-"""
+\"\"\"
+"

src/Fantomas.Tests/TokenParserBoolExprTests.fs

@@ -232,22 +232,27 @@ let ``Hash ifs source format property``() =
                       result |> should equal source)))
 
 [<Test>]
-let ``get define exprs from unit test with defines in string`` () =
+let ``get define exprs from unit test with defines in triple quote string`` () =
     let source = "
+\"\"\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"\"\"
+"
+    getDefineExprs source == List.empty
+
 [<Test>]
-let ``should keep compiler directives``() =
-    formatSourceString false \"\"\"
-#if INTERACTIVE
-#load \"../FSharpx.TypeProviders/SetupTesting.fsx\"
-SetupTesting.generateSetupScript __SOURCE_DIRECTORY__
-#load \"__setup__.fsx\"
+let ``nested quote in triple quote string should not yield defines`` () =
+    let source = "
+\"\"\"
+\"
+#if FOO
+#if BAR
 #endif
-\"\"\"  config
-    |> should equal \"\"\"#if INTERACTIVE
-#load \"../FSharpx.TypeProviders/SetupTesting.fsx\"
-SetupTesting.generateSetupScript __SOURCE_DIRECTORY__
-#load \"__setup__.fsx\"
 #endif
+\"
 \"\"\"
 "
-    getDefineExprs source == List.empty
+    getDefineExprs source == List.empty

src/Fantomas.Tests/TokenParserTests.fs

@@ -414,3 +414,35 @@ let ``escaped char content`` () =
     | [{ Item = CharContent("\'\\u0000\'") }] ->
         pass()
     | _ -> fail()
+
+[<Test>]
+let ``open close of string on same line`` () =
+    let source = "
+let a = \"\"
+#if FOO
+#if BAR
+#endif
+#endif
+"
+
+    getDefines source == ["BAR";"FOO"]
+
+[<Test>]
+let ``open close of triple quote string on same line`` () =
+    let source = "
+let a = \"\"\"foo\"\"\"
+#if FOO
+#endif
+"
+
+    getDefines source == ["FOO"]
+
+[<Test>]
+let ``open, quote, close of triple quote string on same line`` () =
+    let source = "
+let a = \"\"\"fo\"o\"\"\"
+#if FOO
+#endif
+"
+
+    getDefines source == ["FOO"]

src/Fantomas/TokenParser.fs

@@ -1,14 +1,13 @@
 module internal Fantomas.TokenParser
 
+open System.Text.RegularExpressions
 open FSharp.Compiler.AbstractIL.Internal.Library
 open System
 open System.Text
 open FSharp.Compiler.SourceCodeServices
 open Fantomas
 open Fantomas.TokenParserBoolExpr
 open Fantomas.TriviaTypes
-open System.Linq
-open System.Text.RegularExpressions
 
 // workaround for cases where tokenizer dont output "delayed" part of operator after ">."
 // See https://github.com/fsharp/FSharp.Compiler.Service/issues/874
@@ -76,6 +75,10 @@ let private createHashToken lineNumber content fullMatchedLength offset =
             Tag = 0
             FullMatchedLength = fullMatchedLength } }
 
+type private SourceCodeState =
+    | Normal
+    | InsideSingleQuoteString
+    | InsideTripleQuoteString
 
 let rec private getTokenizedHashes sourceCode =
     let processLine content (trimmed:string) lineNumber fullMatchedLength offset =
@@ -93,29 +96,7 @@ let rec private getTokenizedHashes sourceCode =
         )
         |> fun rest -> (createHashToken lineNumber content fullMatchedLength offset)::rest
 
-
-    let tripleQuoteStringPositions =
-        Regex.Matches(sourceCode, "\"\"\"").Cast<Match>()
-        |> Seq.map(fun (mc:Match) -> mc.Index)
-        |> Seq.chunkBySize 2
-        |> Seq.choose (fun group ->
-            match group with
-            | ([|start ;stop |]) -> Some [start..stop]
-            | _ -> None)
-        |> Seq.concat
-        |> Seq.toArray
-
-    let lines =
-        sourceCode
-        |> String.normalizeThenSplitNewLine
-
-    let stringLines =
-        lines
-
-
-    lines
-    |> Array.indexed
-    |> Array.map (fun (idx, line) ->
+    let findDefineInLine idx line =
         let lineNumber  = idx + 1
         let fullMatchedLength = String.length line
         let trimmed = line.TrimStart()
@@ -131,9 +112,64 @@ let rec private getTokenizedHashes sourceCode =
             processLine "#endif" trimmed lineNumber fullMatchedLength offset
         else
             []
-    )
-    |> Seq.collect id
-    |> Seq.toList
+
+    let hasUnEvenAmount regex line = (Regex.Matches(line, regex).Count) % 2 = 1
+    let singleQuoteWrappedInTriple line = Regex.Match(line, "\\\"\\\"\\\".*\\\".*\\\"\\\"\\\"").Success
+
+    sourceCode
+    |> String.normalizeThenSplitNewLine
+    |> Array.indexed
+    |> Array.fold (fun (state, defines) (idx, line) ->
+        let hasTripleQuotes = hasUnEvenAmount "\"\"\"" line
+        let hasSingleQuote =
+            (not hasTripleQuotes)
+            && (hasUnEvenAmount "\"" line)
+            && not (singleQuoteWrappedInTriple line)
+
+        match state with
+        | Normal when (hasTripleQuotes) ->
+            InsideTripleQuoteString, defines
+        | Normal when (hasSingleQuote) ->
+            InsideSingleQuoteString, defines
+        | Normal when (not hasTripleQuotes && not hasSingleQuote) ->
+            Normal, (findDefineInLine idx line)::defines
+
+        | InsideTripleQuoteString when (not hasTripleQuotes) ->
+            InsideTripleQuoteString, defines
+        | InsideTripleQuoteString when hasTripleQuotes ->
+            Normal, defines
+
+        | InsideSingleQuoteString when (not hasSingleQuote) ->
+            InsideSingleQuoteString, defines
+        | InsideSingleQuoteString when (hasSingleQuote) ->
+            Normal, defines
+
+        | _ ->
+            failwithf "unexpected %A" state
+
+    )(Normal, [])
+    |> snd
+    |> List.rev
+    |> List.concat
+//    |> Array.map (fun (idx, line) ->
+//        let lineNumber  = idx + 1
+//        let fullMatchedLength = String.length line
+//        let trimmed = line.TrimStart()
+//        let offset = String.length line - String.length trimmed
+//
+//        if trimmed.StartsWith("#if") then
+//            processLine "#if" trimmed lineNumber fullMatchedLength offset
+//        elif trimmed.StartsWith("#elseif") then
+//            processLine "#elseif" trimmed lineNumber fullMatchedLength offset
+//        elif trimmed.StartsWith("#else") then
+//            processLine "#else" trimmed lineNumber fullMatchedLength offset
+//        elif trimmed.StartsWith("#endif") then
+//            processLine "#endif" trimmed lineNumber fullMatchedLength offset
+//        else
+//            []
+//    )
+//    |> Seq.collect id
+//    |> Seq.toList
 
 and tokenize defines (content : string) : Token list * int =
     let sourceTokenizer = FSharpSourceTokenizer(defines, Some "/tmp.fsx")