Initial implementation of a preprocessor, handles #ifdef (#254)

#180

Shader Minifier Library.fsproj

@@ -58,6 +58,7 @@
     <Compile Include="src\renamer.fs" />
     <Compile Include="src\analyzer.fs" />
     <Compile Include="src\rewriter.fs" />
+    <Compile Include="src\preprocessor.fs" />
     <Compile Include="src\parse.fs" />
     <Compile Include="src\api.fs" />
   </ItemGroup>

Shader Minifier.fsproj

@@ -23,7 +23,7 @@
     <OutputPath>bin\Debug\</OutputPath>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <WarningLevel>5</WarningLevel>
-    <StartArguments>-o "" ../../tests\unit\commas.frag --smoothstep</StartArguments>
+    <StartArguments>--preprocess --no-remove-unused -o "" ../../tests\unit\preprocess.frag</StartArguments>
     <OtherFlags>--warnon:1182</OtherFlags>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">

src/options.fs

@@ -38,6 +38,7 @@ type CliArguments =
     | [<CustomCommandLine("--smoothstep")>] Smoothstep
     | [<CustomCommandLine("--no-remove-unused")>] NoRemoveUnused
     | [<CustomCommandLine("--move-declarations")>] MoveDeclarations
+    | [<CustomCommandLine("--preprocess")>] Preprocess
     | [<MainCommand>] Filenames of filename:string list
 
     interface IArgParserTemplate with
@@ -58,7 +59,8 @@ type CliArguments =
             | Smoothstep -> "Use IQ's smoothstep trick"
             | NoRemoveUnused -> "Do not remove unused code"
             | MoveDeclarations -> "Move declarations to group them"
-            | Filenames _ -> "List of files to minify" 
+            | Preprocess -> "Preprocess the file"
+            | Filenames _ -> "List of files to minify"
 
 type Options() =
     member val outputName = "shader_code.h" with get, set
@@ -76,6 +78,7 @@ type Options() =
     member val noRenamingList = ["main"; "mainImage"] with get, set
     member val noRemoveUnused = false with get, set
     member val moveDeclarations = false with get, set
+    member val preprocess = false with get, set
     member val filenames = [||]: string[] with get, set
 
 module Globals =
@@ -118,6 +121,7 @@ let private initPrivate argv needFiles =
         options.noRenaming <- args.Contains(NoRenaming)
         options.noRemoveUnused <- args.Contains(NoRemoveUnused)
         options.moveDeclarations <- args.Contains(MoveDeclarations)
+        options.preprocess <- args.Contains(Preprocess)
         options.noRenamingList <- noRenamingList
         options.filenames <- filenames
         true

src/parse.fs

@@ -369,6 +369,8 @@ type ParseImpl() =
     member _.runParser streamName content : Ast.Shader =
         forbiddenNames <- [ "if"; "in"; "do" ]
         reorderFunctions <- false
+        let content =
+            if options.preprocess then Preprocessor.preprocess streamName content else content
         let res = runParserOnString parse () streamName content
         match res with
         | Success(r,_,_) -> {

src/preprocessor.fs

@@ -0,0 +1,92 @@
+module Preprocessor
+
+open FParsec.Primitives
+open FParsec.CharParsers
+open FParsec
+
+open System.Collections.Generic
+
+type Impl() =
+    // Dict of macro name to value
+    let defines = new Dictionary<string, string>()
+
+    // The stack contains information for each nested #if/#ifdef block
+    //   Some true: the condition was true, keep the text
+    //   Some false: the condition was false, delete the text
+    //   None: the condition was not evaluated, keep the text and the #end
+    let stack = new Stack<bool option>()
+
+    let currentScope () =
+        if stack.Count = 0 then None
+        else stack.Peek()
+
+    let enterScope evaluated =
+        let current = currentScope ()
+        if current = Some false then
+            stack.Push current
+        else
+            stack.Push evaluated
+
+    let keyword s = attempt (pstring s .>> notFollowedBy letter .>> notFollowedBy digit .>> notFollowedBy (pchar '_')) .>> spaces
+
+    let parseIdent =
+        manyChars (choice [letter; digit])
+
+    let parseArgs = parse {
+        let! _ = pchar '('
+        let! _ = pchar ')'
+        return "(args)"
+    }
+
+    let parseEndLine = manyCharsTill anyChar (followedBy newline) .>> newline
+
+    let parseDefine = parse {
+        let! _ = keyword "define"
+        let! name = parseIdent
+        let! args = opt parseArgs // TODO
+        do! spaces
+        let! line = parseEndLine
+        defines.Add(name, line)
+        return sprintf "#define %s %s" name line
+    }
+
+    let parseIfdef = parse {
+        let! word = keyword "ifdef" <|> keyword "ifndef"
+        let! ident = parseIdent
+        do if defines.ContainsKey ident = (word = "ifdef") then
+             enterScope (Some true)
+           else
+             enterScope (Some false)
+        return ""
+    }
+
+    let parseEndif = parse {
+        let! _ = keyword "endif"
+        let state = stack.Pop()
+        return if state = None then
+                "#endif"
+            else
+                ""
+    }
+
+    let parseText = parse {
+        let! line = parseEndLine
+        return
+            if currentScope() = Some false then ""
+            else line
+    }
+
+    let parseOther = parseEndLine |>> (fun s -> "#" + s)
+
+    let directive = pchar '#' >>. spaces >>. choice [parseDefine; parseIfdef; parseEndif; parseOther]
+
+    member _.Parse = many (directive <|> parseText)
+
+let preprocess streamName content =
+    let impl = new Impl()
+    let res = runParserOnString impl.Parse () streamName content
+    match res with
+        | Success(s,_,_) ->
+            // printfn "%s\n-------------------------" (String.concat "\n" s)
+            String.concat "\n" s
+        | Failure(str, _, _) -> failwithf "Parse error: %s" str

tests/commands.txt

@@ -26,6 +26,7 @@
 --no-remove-unused --no-renaming -o tests/unit/loop.frag.expected tests/unit/loop.frag
 --no-remove-unused --no-renaming -o tests/unit/array.frag.expected tests/unit/array.frag
 --no-remove-unused --no-renaming --format indented -o tests/unit/numbers.frag.expected tests/unit/numbers.frag
+--preprocess --no-remove-unused --no-renaming --format indented -o tests/unit/preprocess.frag.expected tests/unit/preprocess.frag
 
 # Inlining unit tests
 

tests/unit/preprocess.frag

@@ -0,0 +1,27 @@
+#version 130
+
+#define DEF bar
+#define foo() bar
+
+#ifdef DEF
+void keep_ifdef() {}
+#endif
+
+#ifdef UNKNOWN
+void remove_ifdef() {}
+#endif
+
+#ifdef DEF
+# ifdef DEF // nested
+#  ifdef UNKNOWN
+#    ifdef DEF
+int remove_this_line;
+#    endif
+int remove_this_line_too;
+#  endif
+void keep_nested() {}
+# endif
+void keep_outernest() {}
+#endif
+
+void end() {}

tests/unit/preprocess.frag.expected

@@ -0,0 +1,14 @@
+#version 130
+
+#define DEF bar
+
+#define foo bar
+
+void keep_ifdef()
+{}
+void keep_nested()
+{}
+void keep_outernest()
+{}
+void end()
+{}