Exclude defines found in multiline strings (#762)

* WIP

* Exclude defines found in multiline strings. Fixes #761

* Reversed order of defines

* tests for escaped quotes

* consider escaped quotes

* trigger ci

Co-authored-by: Jindřich Ivánek <jindra.ivanek@gmail.com>

src/Fantomas.Tests/CompilerDirectivesTests.fs

@@ -1364,4 +1364,107 @@ module Dbg =
     let teePrint x = x
     let print _ = ()
 #endif
-"""
+"""
+
+[<Test>]
+let ``defines in string should be taken into account, 761`` () =
+    (formatSourceString false "
+[<Test>]
+let ``should keep compiler directives``() =
+    formatSourceString false \"\"\"
+#if INTERACTIVE
+#load \"../FSharpx.TypeProviders/SetupTesting.fsx\"
+SetupTesting.generateSetupScript __SOURCE_DIRECTORY__
+#load \"__setup__.fsx\"
+#endif
+\"\"\"  config
+    |> should equal \"\"\"#if INTERACTIVE
+#load \"../FSharpx.TypeProviders/SetupTesting.fsx\"
+SetupTesting.generateSetupScript __SOURCE_DIRECTORY__
+#load \"__setup__.fsx\"
+#endif
+\"\"\"
+" config)
+    |> prepend newline
+    |> should equal "
+[<Test>]
+let ``should keep compiler directives`` () =
+    (formatSourceString false \"\"\"
+#if INTERACTIVE
+#load \"../FSharpx.TypeProviders/SetupTesting.fsx\"
+SetupTesting.generateSetupScript __SOURCE_DIRECTORY__
+#load \"__setup__.fsx\"
+#endif
+\"\"\"   config)
+    |> should equal \"\"\"#if INTERACTIVE
+#load \"../FSharpx.TypeProviders/SetupTesting.fsx\"
+SetupTesting.generateSetupScript __SOURCE_DIRECTORY__
+#load \"__setup__.fsx\"
+#endif
+\"\"\"
+"
+
+[<Test>]
+let ``hash directive in single quote string should not have impact`` () =
+    formatSourceString false """let a = "
+#if FOO
+"
+let b = "
+#endif
+"
+"""  config
+    |> prepend newline
+    |> should equal """
+let a = "
+#if FOO
+"
+
+let b = "
+#endif
+"
+"""
+
+[<Test>]
+let ``hash directive in triple quote string with other quotes should not have impact`` () =
+    (formatSourceString false "
+let a = \"\"\"
+\"
+#if FOO
+\"
+\"\"\"
+let b = \"\"\"
+#endif
+\"\"\"
+"     config)
+    |> prepend newline
+    |> should equal "
+let a = \"\"\"
+\"
+#if FOO
+\"
+\"\"\"
+
+let b = \"\"\"
+#endif
+\"\"\"
+"
+
+[<Test>]
+let ``hash directive in single quote string should not have impact - escaped quote`` () =
+    formatSourceString false """let a = "
+#if FOO
+\""
+let b = "
+#endif
+"
+"""  config
+    |> prepend newline
+    |> should equal """
+let a = "
+#if FOO
+\""
+
+let b = "
+#endif
+"
+"""

src/Fantomas.Tests/TokenParserBoolExprTests.fs

@@ -230,3 +230,29 @@ let ``Hash ifs source format property``() =
                      (let source = boolExprsToSource es
                       let result = formatSourceString false source config
                       result |> should equal source)))
+
+[<Test>]
+let ``get define exprs from unit test with defines in triple quote string`` () =
+    let source = "
+\"\"\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"\"\"
+"
+    getDefineExprs source == List.empty
+
+[<Test>]
+let ``nested quote in triple quote string should not yield defines`` () =
+    let source = "
+\"\"\"
+\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"
+\"\"\"
+"
+    getDefineExprs source == List.empty

src/Fantomas.Tests/TokenParserTests.fs

@@ -414,3 +414,101 @@ let ``escaped char content`` () =
     | [{ Item = CharContent("\'\\u0000\'") }] ->
         pass()
     | _ -> fail()
+
+[<Test>]
+let ``open close of string on same line`` () =
+    let source = "
+let a = \"\"
+#if FOO
+#if BAR
+#endif
+#endif
+"
+
+    getDefines source == ["FOO";"BAR"]
+
+[<Test>]
+let ``open close of triple quote string on same line`` () =
+    let source = "
+let a = \"\"\"foo\"\"\"
+#if FOO
+#endif
+"
+
+    getDefines source == ["FOO"]
+
+[<Test>]
+let ``open, quote, close of triple quote string on same line`` () =
+    let source = "
+let a = \"\"\"fo\"o\"\"\"
+#if FOO
+#endif
+"
+
+    getDefines source == ["FOO"]
+
+[<Test>]
+let ``defines inside string`` () =
+    let source = "
+let a = \"
+#if FOO
+#if BAR
+#endif
+#endif
+\"
+"
+
+    getDefines source == []
+
+[<Test>]
+let ``defines inside string, escaped quote`` () =
+    let source = "
+let a = \"\\\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"
+"
+
+    getDefines source == []
+
+
+[<Test>]
+let ``defines inside triple quote string`` () =
+    let source = "
+let a = \"\"\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"\"\"
+"
+
+    getDefines source == []
+
+[<Test>]
+let ``defines inside triple quote string, escaped quote`` () =
+    let source = "
+let a = \"\"\"\\\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"\"\"
+"
+
+    getDefines source == []
+
+[<Test>]
+let ``defines inside triple quote string, escaped triple quote`` () =
+    let source = "
+let a = \"\"\"\\\"\"\"
+#if FOO
+#if BAR
+#endif
+#endif
+\"\"\"
+"
+
+    getDefines source == []

src/Fantomas/TokenParser.fs

@@ -1,5 +1,6 @@
 module internal Fantomas.TokenParser
 
+open System.Text.RegularExpressions
 open FSharp.Compiler.AbstractIL.Internal.Library
 open System
 open System.Text
@@ -74,6 +75,10 @@ let private createHashToken lineNumber content fullMatchedLength offset =
             Tag = 0
             FullMatchedLength = fullMatchedLength } }
 
+type private SourceCodeState =
+    | Normal
+    | InsideSingleQuoteString
+    | InsideTripleQuoteString
 
 let rec private getTokenizedHashes sourceCode =
     let processLine content (trimmed:string) lineNumber fullMatchedLength offset =
@@ -91,10 +96,7 @@ let rec private getTokenizedHashes sourceCode =
         )
         |> fun rest -> (createHashToken lineNumber content fullMatchedLength offset)::rest
 
-    sourceCode
-    |> String.normalizeThenSplitNewLine
-    |> Array.indexed
-    |> Array.map (fun (idx, line) ->
+    let findDefineInLine idx line =
         let lineNumber  = idx + 1
         let fullMatchedLength = String.length line
         let trimmed = line.TrimStart()
@@ -110,9 +112,45 @@ let rec private getTokenizedHashes sourceCode =
             processLine "#endif" trimmed lineNumber fullMatchedLength offset
         else
             []
-    )
-    |> Seq.collect id
-    |> Seq.toList
+
+    let hasUnEvenAmount regex line = (Regex.Matches(line, regex).Count - Regex.Matches(line, "\\\\" + regex).Count) % 2 = 1
+    let singleQuoteWrappedInTriple line = Regex.Match(line, "\\\"\\\"\\\".*\\\".*\\\"\\\"\\\"").Success
+
+    sourceCode
+    |> String.normalizeThenSplitNewLine
+    |> Array.indexed
+    |> Array.fold (fun (state, defines) (idx, line) ->
+        let hasTripleQuotes = hasUnEvenAmount "\"\"\"" line
+        let hasSingleQuote =
+            (not hasTripleQuotes)
+            && (hasUnEvenAmount "\"" line)
+            && not (singleQuoteWrappedInTriple line)
+
+        match state with
+        | Normal when (hasTripleQuotes) ->
+            InsideTripleQuoteString, defines
+        | Normal when (hasSingleQuote) ->
+            InsideSingleQuoteString, defines
+        | Normal when (not hasTripleQuotes && not hasSingleQuote) ->
+            Normal, (findDefineInLine idx line)::defines
+
+        | InsideTripleQuoteString when (not hasTripleQuotes) ->
+            InsideTripleQuoteString, defines
+        | InsideTripleQuoteString when hasTripleQuotes ->
+            Normal, defines
+
+        | InsideSingleQuoteString when (not hasSingleQuote) ->
+            InsideSingleQuoteString, defines
+        | InsideSingleQuoteString when (hasSingleQuote) ->
+            Normal, defines
+
+        | _ ->
+            failwithf "unexpected %A" state
+
+    )(Normal, [])
+    |> snd
+    |> List.rev
+    |> List.concat
 
 and tokenize defines (content : string) : Token list * int =
     let sourceTokenizer = FSharpSourceTokenizer(defines, Some "/tmp.fsx")