#5589: Fix a problem when reparsing complex map expressions due to in…

…consistent tokeniser delimiters.
codereader · Apr 15, 2021 · 590886e · 590886e
1 parent eec691d
commit 590886e
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 5 deletions.
diff --git a/libs/parser/DefTokeniser.h b/libs/parser/DefTokeniser.h
@@ -347,7 +347,7 @@ class DefTokeniserFunc
     }
 };
 
-const char* const WHITESPACE = " \t\n\v\r";
+constexpr const char* const WHITESPACE = " \t\n\v\r";
 
 /**
  * DefTokeniser abstract base class. This class provides a unified interface to

diff --git a/radiantcore/shaders/MapExpression.cpp b/radiantcore/shaders/MapExpression.cpp
@@ -16,6 +16,7 @@
 #include "textures/HeightmapCreator.h"
 #include "textures/TextureManipulator.h"
 #include "string/predicate.h"
+#include "ShaderTemplate.h"
 
 /* CONSTANTS */
 namespace
@@ -96,8 +97,12 @@ MapExpressionPtr MapExpression::createForString(const std::string& str)
 {
     try
     {
-        parser::BasicDefTokeniser<std::string> token(str);
-        return createForToken(token);
+        parser::BasicDefTokeniser<std::string> tokeniser(
+            str,
+            ShaderTemplate::DiscardedDelimiters, // delimiters (whitespace)
+            ShaderTemplate::KeptDelimiters
+        );
+        return createForToken(tokeniser);
     }
     catch (const parser::ParseException&)
     {

diff --git a/radiantcore/shaders/ShaderTemplate.cpp b/radiantcore/shaders/ShaderTemplate.cpp
@@ -1187,8 +1187,8 @@ void ShaderTemplate::parseDefinition()
     // Construct a local deftokeniser to parse the unparsed block
     parser::BasicDefTokeniser<std::string> tokeniser(
         _blockContents,
-		parser::WHITESPACE, // delimiters (whitespace)
-        "{}(),"  // add the comma character to the kept delimiters
+        DiscardedDelimiters, // delimiters (whitespace)
+        KeptDelimiters
     );
 
     _parsed = true; // we're parsed from now on

diff --git a/radiantcore/shaders/ShaderTemplate.h b/radiantcore/shaders/ShaderTemplate.h
@@ -36,6 +36,10 @@ class ShaderTemplate final
 
 public:
 
+    // Shared parsing constants
+    constexpr static const char* DiscardedDelimiters = parser::WHITESPACE;
+    constexpr static const char* KeptDelimiters = "{}(),"; // add the comma character to the kept delimiters
+
   	// Vector of LayerTemplates representing each stage in the material
     std::vector<Doom3ShaderLayer::Ptr> _layers;