feat(matcher): change logic to match now disregarding the order of token

xetra11 · Dec 15, 2020 · f40740e · f40740e
1 parent a74d84a
commit f40740e
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 22 deletions.
diff --git a/...lin/com/github/xetra11/ck3workbench/module/character/importer/tokenizer/GrammarMatcher.kt b/...lin/com/github/xetra11/ck3workbench/module/character/importer/tokenizer/GrammarMatcher.kt
@@ -30,36 +30,49 @@ open class GrammarMatcher {
         return if (grammar.tokenDefinition.isEmpty()) {
             MatcherResult("", hasError = true, errorReason = "Grammar was undefined")
         } else {
-            var formattedLines = scriptLines.prepareScriptString()
+            val formattedLines = scriptLines.prepareScriptString()
+            val nestedGrammar = nestGrammar(grammar)
 
-            val grammarNester = GrammarNester()
-            val nestedGrammar = grammarNester.nest(grammar)
-
-            val matchCollector = grammar.tokenDefinition.mapTo(mutableListOf<String>()) { tokenType ->
-                val regex = tokenRegexMapping[tokenType]
-                val match = regex?.find(formattedLines[NEXT])?.value ?: ""
-                if (match.isEmpty()) {
-                    if (tokenType is TokenType) {
-                        return MatcherResult("", hasError = true, errorReason = "Token order invalid")
+            var currentLevel = 0
+            val scriptValid = formattedLines.all { line ->
+                val pieces = line.split(" ")
+                pieces.all { piece ->
+                    when (piece) {
+                        "{" -> {
+                            currentLevel++
+                            true
+                        }
+                        "}" -> {
+                            currentLevel--
+                            true
+                        }
+                        else -> {
+                            val grammarSection = nestedGrammar.level(currentLevel)
+                            val tokenRegexCandidates = grammarSection.mapTo(mutableListOf<Regex>()) { tokenRegexMapping[it] ?: Regex("") }
+                            tokenRegexCandidates.any { it.matches(piece) }
+                        }
                     }
                 }
-                // reduce origin script by matched value
-                formattedLines[NEXT] = formattedLines[NEXT].replaceFirst(match, "")
-                formattedLines = formattedLines.filter { it.isNotBlank() }.toMutableList()
-
-                match
             }
-
-            MatcherResult(matchCollector.joinToString("") { it })
+            if (scriptValid) {
+                MatcherResult("")
+            } else {
+                MatcherResult("", true, "Script could not be validated against given grammar")
+            }
         }
     }
 
+    private fun nestGrammar(grammar: Grammar): GrammarNester.NestedGrammar {
+        val grammarNester = GrammarNester()
+        return grammarNester.nest(grammar)
+    }
+
     private fun List<String>.prepareScriptString(): MutableList<String> {
         return this
             .asSequence()
-            .map { it.replace(" ", "") }
             .map { it.replace("\\uFEFF", "") } // remove BOM
-            .map { it.trim() }
+            .map { it.trimStart() }
+            .map { it.trimEnd() }
             .filter { it.isNotBlank() }
             .toMutableList()
     }

diff --git a/...tlin/com/github/xetra11/ck3workbench/module/character/importer/tokenizer/GrammarNester.kt b/...tlin/com/github/xetra11/ck3workbench/module/character/importer/tokenizer/GrammarNester.kt
@@ -70,7 +70,7 @@ class GrammarNester {
          * @return the list of [TokenDefinition] at the given level
          */
         fun level(level: Int): List<TokenDefinition> {
-            require((level > 0).and(level.minus(1) < nestedToken.size))
+            require((level >= 0).and(level.minus(1) < nestedToken.size))
             return nestedToken[level].orEmpty()
         }
 

diff --git a/...com/github/xetra11/ck3workbench/module/character/importer/tokenizer/GrammarMatcherTest.kt b/...com/github/xetra11/ck3workbench/module/character/importer/tokenizer/GrammarMatcherTest.kt
@@ -31,9 +31,8 @@ internal class GrammarMatcherTest {
         )
 
         val actual = grammarMatcher.rule(grammar, SCRIPT_FULL_EXAMPLE_WITH_SUBOBJECT.split("\n"))
-        val expected = MatcherResult(SCRIPT_FULL_EXAMPLE_WITH_SUBOBJECT.trimWhiteSpace())
 
-        assertThat(actual).isEqualTo(expected)
+        assertThat(actual.hasError).isFalse
     }
 
     @Test