Fix TokenCat in prList (BNFC#159); make lexer more flexible

gapag · Nov 20, 2015 · fe86d45 · fe86d45
1 parent e7055f8
commit fe86d45
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 28 deletions.
diff --git a/source/src/BNFC/Backend/Haskell/Utils.hs b/source/src/BNFC/Backend/Haskell/Utils.hs
@@ -91,7 +91,7 @@ catToType param cat = maybeParens $ catToType' param cat
     catToType' (Just p) (CoercCat c _) = text c <+> p
     catToType' (Just _) (TokenCat c) = text c
     catToType' (Just p) (ListCat c) = lbrack <> catToType' (Just p) c <> rbrack
-
+    catToType' (Just p) (IndentationEnterCat c) = catToType' (Just p) c
 
 -- | gives you a list of variables usable for pattern matching.
 -- Ex: if you have the rule Aba. S ::= A B A ; with the generated data type

diff --git a/source/src/BNFC/Backend/Java/CFtoAntlr4Lexer.hs b/source/src/BNFC/Backend/Java/CFtoAntlr4Lexer.hs
@@ -304,10 +304,10 @@ javaLexerPreamble =
             , "private final java.util.Deque<Boolean> ignoringIndentation = initIgnoringIndentation();"
             , "private final java.util.Deque<Token> pendingTokens  = initPendingTokens();"
             , "public void increaseIndentation(String d)"
-            , codeblock 2 [ "if(isIgnoringIndentation()) return;"
-                    , "int peeking = nestingDiffs.peekFirst();"
+            , codeblock 2 [
+                    "int peeking = nestingDiffs.peekFirst();"
                     , "if(peeking == -1) nestingDiffs.push(d.length());"
-                    , "else nestingDiffs.push(d.length() - indentationLength());"
+                    , "if(!isIgnoringIndentation()) nestingDiffs.push(d.length() - indentationLength());"
                     , "indentation = new StringBuffer(d);"
                 ]
             , "public int indentationLength()"
@@ -351,33 +351,54 @@ javaLexerPreamble =
             , "public Token nextToken()"
             , codeblock 2 [ "return getNextToken();"]
             , "private Token getNextToken()"
-            , codeblock 2 [ "Token t = null;"
+            , codeblock 2 [
+                "boolean fromPending = false;"
+                ,"Token t = null;"
                 , "if(pendingTokens.size()>0)"
-		, codeblock 2 [ "t = pendingTokens.pop();"]
+		, codeblock 2 [ "t = pendingTokens.pop();"
+		    , "fromPending = true;"]
                 , "else"
-                , codeblock 2 [ "t = super.nextToken();"
-                    , "if(t.getType() == INDENTATION_DECREASED)"
+                , codeblock 2 [ "t = super.nextToken();"]
+                , "if(t.getType() == INDENTATION_DECREASED)"
+                , codeblock 2 [
+                    "if(fromPending) decreaseIndentation();"
+                    , "int diff = indentation.length()-t.getText().length();"
+                    , "if( diff == 0)"
                     , codeblock 2 [
-                        "pendingTokens.push(this._factory.create(INDENTATION,indentation.toString()));"
+                        "// One level indentation decrease. Put an indentation for enclosing scope"
+                       , "pendingTokens.push(this._factory.create(INDENTATION,indentation.toString()));"
                     ]
-                    , "else if (t.getType() == -1)"
-                    ,codeblock 2 [
-                        "pendingTokens.push(t);"
-                        , "pendingTokens.push(t);// twice is better! (Test looks for one more EOF)"
-                        , "java.util.Iterator<Integer> it = nestingDiffs.descendingIterator();"
-                        , "ignoringBottom:"
-                        , codeblock 2 [ "pendingTokens.push(this._factory.create(INDENTATION_DECREASED,\"\"));"
-                            , "it.next();"
-                        ]
-                        , "StringBuffer indVal = new StringBuffer(indentation);"
-                        , "int acc = 0;"
-                        , "while(it.hasNext())"
-                        , codeblock 2 [
-                          "acc += it.next();"
-                          , "pendingTokens.push(this._factory.create(INDENTATION_DECREASED,indVal.substring(0,acc)));"
-                        ]
-                        , "t = pendingTokens.pop();"
+                    , "else if(diff > 0)"
+                    , codeblock 2 [
+                        "// This has been decreased more than expected."
+                        , "// Inject an INDENTATION_DECREASED to go even."
+                        , "String newIndent = indentation.substring((diff));"
+                        , "pendingTokens.push(this._factory.create(INDENTATION,newIndent));"
+                        , "pendingTokens.push(this._factory.create(INDENTATION_DECREASED,newIndent));"
+                    ]
+                    , "else"
+                    , codeblock 2 [
+                        "// has not de-indented enough to jump to a lower scope, error"
+                        , "throw new RuntimeException(\"Indentation problem at token \"+t);"
+                    ]
+                ]
+                , "else if (!fromPending && t.getType() == -1)"
+                ,codeblock 2 [
+                    "pendingTokens.push(t);"
+                    , "pendingTokens.push(t);// twice is better! (Test looks for one more EOF)"
+                    , "java.util.Iterator<Integer> it = nestingDiffs.descendingIterator();"
+                    , "ignoringBottom:"
+                    , codeblock 2 [
+                        "it.next();"
+                    ]
+                    , "StringBuffer indVal = new StringBuffer(indentation);"
+                    , "int acc = 0;"
+                    , "while(it.hasNext())"
+                    , codeblock 2 [
+                      "acc += it.next();"
+                      , "pendingTokens.push(this._factory.create(INDENTATION_DECREASED,indVal.substring(0,acc)));"
                     ]
+                    , "t = pendingTokens.pop();"
                 ]
                 , "return t;"
             ]

diff --git a/source/src/BNFC/Backend/Java/CFtoJavaPrinter15.hs b/source/src/BNFC/Backend/Java/CFtoJavaPrinter15.hs
@@ -326,7 +326,7 @@ prList tm user c rules =
 -- >>> prCat "F" (AnonymousTerminal "++")
 --        render("++");
 -- <BLANKLINE>
--- >>> prCat "F" (NonTerminal (Cat "String", "string_"))
+-- >>> prCat "F" (NonTerminal (TokenCat "String", "string_"))
 --        printQuoted(F.string_);
 -- <BLANKLINE>
 -- >>> prCat "F" (NonTerminal (InternalCat, "#_"))
@@ -337,7 +337,7 @@ prList tm user c rules =
 prCat :: Doc -> RhsRuleElement (Cat, Doc) String -> Doc
 prCat _ (AnonymousTerminal t) = nest 7 ("render(\"" <> text(escapeChars t) <> "\");\n")
 prCat _ (IndentationTerminal t) = nest 7 ("render(\"" <> text(escapeChars t) <> "\");\n")
-prCat fnm (NonTerminal (Cat "String", nt))
+prCat fnm (NonTerminal (TokenCat "String", nt))
     = nest 7 ("printQuoted(" <> fnm <> "." <> nt <> ");\n")
 prCat _ (NonTerminal (InternalCat, _)) = empty
 prCat fnm (NonTerminal (cat, nt))