#5573: Experimental PEGTL integration. Parsing works but speed (just …

…cutting out the def blocks) is one order of magnitude than using the DefBlockTokeniser, taking almost 30 seconds here in my debug environment.
codereader · Mar 24, 2021 · 6c46edd · 6c46edd
1 parent 907fb02
commit 6c46edd
Show file tree

Hide file tree

Showing 5 changed files with 151 additions and 3 deletions.
diff --git a/radiantcore/shaders/ShaderFileLoader.h b/radiantcore/shaders/ShaderFileLoader.h
@@ -11,7 +11,13 @@
 
 #include "parser/DefBlockTokeniser.h"
 #include "string/replace.h"
+#include "string/trim.h"
 #include "string/predicate.h"
+#include "os/path.h"
+
+#include "parser/Grammar.h"
+
+#include <tao/pegtl/contrib/parse_tree.hpp>
 
 namespace shaders
 {
@@ -27,9 +33,61 @@ template<typename ShaderLibrary_T> class ShaderFileLoader
     // List of shader definition files to parse
     std::vector<vfs::FileInfo> _files;
 
+    class BlockParseState
+    {
+    public:
+        BlockParseState(ShaderLibrary_T& library_, const vfs::FileInfo& fileInfo_) :
+            fileInfo(fileInfo_),
+            library(library_)
+        {}
+
+        const vfs::FileInfo& fileInfo;
+        ShaderLibrary_T& library;
+
+        std::string name;
+        std::string contents;
+    };
+
+    template<typename Rule>
+    struct ParseAction : tao::pegtl::nothing<Rule> {};
+
+    template<>
+    struct ParseAction<parser::DeclName>
+    {
+        template<typename ParseInput>
+        static void apply(const ParseInput& in, BlockParseState& state)
+        {
+            state.name = in.string();
+            state.contents.clear();
+        }
+    };
+
+    template<>
+    struct ParseAction<parser::MaterialBlockDefinition>
+    {
+        template<typename ParseInput>
+        static void apply(const ParseInput& in, BlockParseState& state)
+        {
+            state.contents = string::trim_right_copy(string::trim_left_copy(in.string(), "{"), "}");
+
+            string::replace_all(state.name, "\\", "/"); // use forward slashes
+
+            auto shaderTemplate = std::make_shared<ShaderTemplate>(state.name, state.contents);
+
+            // Construct the ShaderDefinition wrapper class
+            ShaderDefinition def(shaderTemplate, state.fileInfo);
+
+            // Insert into the definitions map, if not already present
+            if (!state.library.addDefinition(state.name, def))
+            {
+                rError() << "[shaders] " << state.fileInfo.name << ": shader " << state.name << " already defined." << std::endl;
+            }
+        }
+    };
+
 private:
 
-    bool parseTable(const parser::BlockTokeniser::Block& block, const vfs::FileInfo& fileInfo)
+    bool parseTable(const ::parser::BlockTokeniser::Block& block, const vfs::FileInfo& fileInfo)
     {
         if (block.name.length() <= 5 || !string::starts_with(block.name, "table"))
         {
@@ -61,14 +119,31 @@ template<typename ShaderLibrary_T> class ShaderFileLoader
     // Parse a shader file with the given contents and filename
     void parseShaderFile(std::istream& inStr, const vfs::FileInfo& fileInfo)
     {
+#if 1
+        using namespace tao::pegtl;
+
+        // Wrap the input stream for the pegtl parser
+        istream_input input(inStr, 1*1024*1024, os::getFilename(fileInfo.name));
+
+        BlockParseState state(_library, fileInfo);
+
+        try
+        {
+            auto parseTree = parse_tree::parse<parser::MaterialFile, parser::MaterialFileSelector, ParseAction>(input, state);
+        }
+        catch (const parser::parse_error& ex)
+        {
+            rError() << "Error parsing " << fileInfo.name << ": " << ex.what() << std::endl;
+        }
+#else
         // Parse the file with a blocktokeniser, the actual block contents
         // will be parsed separately.
-        parser::BasicDefBlockTokeniser<std::istream> tokeniser(inStr);
+        ::parser::BasicDefBlockTokeniser<std::istream> tokeniser(inStr);
 
         while (tokeniser.hasMoreBlocks())
         {
             // Get the next block
-            parser::BlockTokeniser::Block block = tokeniser.nextBlock();
+            ::parser::BlockTokeniser::Block block = tokeniser.nextBlock();
 
             // Try to parse tables
             if (parseTable(block, fileInfo))
@@ -99,6 +174,7 @@ template<typename ShaderLibrary_T> class ShaderFileLoader
                 rError() << "[shaders] " << fileInfo.name << ": shader " << block.name << " already defined." << std::endl;
             }
         }
+#endif
     }
 
 public:

diff --git a/radiantcore/shaders/parser/Grammar.h b/radiantcore/shaders/parser/Grammar.h
@@ -0,0 +1,49 @@
+#pragma once
+
+#include <tao/pegtl.hpp>
+
+namespace shaders
+{
+
+namespace parser
+{
+
+using namespace tao;
+using namespace tao::pegtl;
+
+struct Nothing : space {};
+
+struct TableKeywordPlusSpace : seq<pegtl::string<'t', 'a', 'b', 'l', 'e'>, space> {};
+
+struct DeclName : plus<alnum, plus<sor<alnum, one<'/', '_'>>>> {};
+
+struct LineComment : until<eolf> {};
+struct BlockComment : until<pegtl::string<'*', '/'>> {};
+
+struct Comment : if_must<pegtl::string<'/', '/'>, LineComment> {};
+struct CommentBlock : if_must<pegtl::string<'/', '*'>, BlockComment> {};
+
+struct BlockStart : one<'{'> {};
+struct BlockEnd : one<'}'> {};
+
+struct BlockContents : not_one<'{', '}'> {};
+
+struct BlockDefinition : seq<BlockStart, star<sor<BlockContents, BlockDefinition>>, BlockEnd> {};
+
+struct MaterialBlockDefinition : seq<BlockStart, star<sor<BlockContents, BlockDefinition>>, BlockEnd> {};
+
+struct MaterialDeclaration : if_must<DeclName, star<sor<space, Comment, CommentBlock>>, MaterialBlockDefinition> {};
+
+struct TableDeclaration : if_must<TableKeywordPlusSpace, DeclName, star<sor<space, Comment, CommentBlock>>, BlockDefinition> {};
+
+// Any possible element of a material file (Tables, Materials, Comments and plain whitespace)
+struct Anything : sor<Nothing, TableDeclaration, MaterialDeclaration, Comment, CommentBlock> {};
+
+// A Material has no required elements, it can include anything (including nothing)
+struct MaterialFile : until<eof, Anything> {};
+
+template<typename Rule> struct MaterialFileSelector : std::true_type {};
+
+}
+
+}
diff --git a/tools/msvc/DarkRadiantCore.vcxproj b/tools/msvc/DarkRadiantCore.vcxproj
@@ -1011,6 +1011,7 @@
     <ClInclude Include="..\..\radiantcore\shaders\ExpressionSlots.h" />
     <ClInclude Include="..\..\radiantcore\shaders\MapExpression.h" />
     <ClInclude Include="..\..\radiantcore\shaders\NamedBindable.h" />
+    <ClInclude Include="..\..\radiantcore\shaders\parser\Grammar.h" />
     <ClInclude Include="..\..\radiantcore\shaders\ShaderDefinition.h" />
     <ClInclude Include="..\..\radiantcore\shaders\ShaderExpression.h" />
     <ClInclude Include="..\..\radiantcore\shaders\ShaderFileLoader.h" />
@@ -1090,6 +1091,7 @@
     <Import Project="properties\zlib.props" />
     <Import Project="properties\libjpeg.props" />
     <Import Project="properties\libpng.props" />
+    <Import Project="properties\pegtl.props" />
   </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
@@ -1101,6 +1103,7 @@
     <Import Project="properties\zlib.props" />
     <Import Project="properties\libjpeg.props" />
     <Import Project="properties\libpng.props" />
+    <Import Project="properties\pegtl.props" />
   </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
@@ -1112,6 +1115,7 @@
     <Import Project="properties\zlib.props" />
     <Import Project="properties\libjpeg.props" />
     <Import Project="properties\libpng.props" />
+    <Import Project="properties\pegtl.props" />
   </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
@@ -1123,6 +1127,7 @@
     <Import Project="properties\zlib.props" />
     <Import Project="properties\libjpeg.props" />
     <Import Project="properties\libpng.props" />
+    <Import Project="properties\pegtl.props" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

diff --git a/tools/msvc/DarkRadiantCore.vcxproj.filters b/tools/msvc/DarkRadiantCore.vcxproj.filters
@@ -197,6 +197,9 @@
     <Filter Include="src\entity\algorithm">
       <UniqueIdentifier>{b7afddd6-8c82-4975-a947-00999bd70fec}</UniqueIdentifier>
     </Filter>
+    <Filter Include="src\shaders\parser">
+      <UniqueIdentifier>{f8abb79b-fbbd-405b-86a5-2cd729c021bd}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\radiantcore\modulesystem\ModuleLoader.cpp">
@@ -2091,5 +2094,8 @@
     <ClInclude Include="..\..\radiantcore\shaders\TextureMatrix.h">
       <Filter>src\shaders</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\radiantcore\shaders\parser\Grammar.h">
+      <Filter>src\shaders\parser</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>
diff --git a/tools/msvc/properties/pegtl.props b/tools/msvc/properties/pegtl.props
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Label="PropertySheets" />
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup />
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalIncludeDirectories>$(DarkRadiantRoot)\w32deps\pegtl\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup />
+</Project>