New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Remove comments from Lua code when bundle project #8187
Changes from all commits
e64ebbb
9e8396e
0a5ae63
c34d8f5
133a3bb
62a32e2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -103,6 +103,10 @@ public void testScanner() throws Exception { | |
assertValidRequire("require 'foo.bar' --[[ some comment]]--", "foo.bar"); | ||
assertValidRequire("require (\"foo.bar\") --[[ some comment]]--", "foo.bar"); | ||
assertValidRequire("require ('foo.bar') --[[ some comment]]--", "foo.bar"); | ||
|
||
int linesInFile = file.split("\r\n|\r|\n").length; | ||
int linesAfterScanner = scanner.getParsedLua().split("\r\n|\r|\n").length; | ||
assertEquals(linesInFile, linesAfterScanner); | ||
} | ||
|
||
private Property findProperty(List<Property> properties, String name) { | ||
|
@@ -169,6 +173,10 @@ public void testPropsStripped() throws Exception { | |
assertEquals(Status.INVALID_ARGS, properties.get(5).status); | ||
assertPropertyStatus(properties, "three_args", Status.INVALID_VALUE, 19); | ||
assertPropertyStatus(properties, "unknown_type", Status.INVALID_VALUE, 20); | ||
|
||
int linesInSource = source.split("\r\n|\r|\n").length; | ||
int linesAfterScanner = scanner.getParsedLua().split("\r\n|\r|\n").length; | ||
assertEquals(linesInSource, linesAfterScanner); | ||
|
||
// parse the already stripped source | ||
// there should be no properties left | ||
|
@@ -261,13 +269,92 @@ public void testPropsMaterial() throws Exception { | |
assertProperty(properties, "prop4", "material", 3); | ||
} | ||
|
||
public static String trimLines(String string) { | ||
String[] lines = string.split("\\r?\\n"); | ||
StringBuilder result = new StringBuilder(); | ||
for (String line : lines) { | ||
result.append(line.trim()).append("\n"); | ||
} | ||
return result.toString(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Using of this function is a bit tricky. It's easier to take into account spaces in expected results |
||
@Test | ||
public void testCommentRemoving() throws Exception { | ||
// few comments | ||
String luaCode = | ||
"local var = 1\n" + | ||
"-- comment 1\n" + | ||
"-- comment 2\n" + | ||
"-- comment 3\n" + | ||
"local var2 = 2\n"; | ||
LuaScanner scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
String parsed = scanner.getParsedLua(); | ||
String expected = | ||
"local var = 1\n" + | ||
"\n" + | ||
"\n" + | ||
"\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, parsed); | ||
|
||
// Function with comment | ||
luaCode = | ||
"local function run(self)\n" + | ||
"self.temp = 1" + | ||
"-- Some comment\n" + | ||
"end\n"; | ||
|
||
scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
"local function run(self)\n" + | ||
"self.temp = 1" + | ||
"\n" + | ||
"end\n"; | ||
assertEquals(expected, parsed); | ||
|
||
// With comment | ||
luaCode = | ||
"local var = 2 -- Some comment\n"; | ||
|
||
scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
"local var = 2 \n"; | ||
assertEquals(expected, parsed); | ||
|
||
// With multiline comment | ||
luaCode = | ||
"local var = 2 --[[Some multiline comment\n"+ | ||
"line two]]--\n"; | ||
|
||
scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
"local var = 2 \n"+ | ||
"\n"; | ||
assertEquals(expected, parsed); | ||
|
||
// With multiline comment | ||
luaCode = | ||
"--[[Some comment\n"+ | ||
"line two]]--\n"; | ||
|
||
scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
"\n"+ | ||
"\n"; | ||
assertEquals(expected, parsed); | ||
|
||
// With multiline comment | ||
luaCode = | ||
"--[[Some comment\n"+ | ||
"line two]]local var = 2\n"; | ||
|
||
scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
"\n"+ | ||
"local var = 2\n"; | ||
assertEquals(expected, parsed); | ||
} | ||
|
||
@Test | ||
|
@@ -284,11 +371,11 @@ public void testRemoveEmptyLifecycleFunctions() throws Exception { | |
String parsed = scanner.getParsedLua(); | ||
String expected = | ||
"local var = 1\n" + | ||
"\n" + | ||
" \n" + | ||
"\n" + | ||
"\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
assertEquals(expected, parsed); | ||
|
||
// With comment | ||
luaCode = | ||
|
@@ -303,29 +390,28 @@ public void testRemoveEmptyLifecycleFunctions() throws Exception { | |
parsed = scanner.getParsedLua(); | ||
expected = | ||
"local var = 1\n" + | ||
" \n" + | ||
"\n" + | ||
"-- Some comment\n" + | ||
"\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
|
||
assertEquals(expected, parsed); | ||
|
||
// With multiline comment | ||
luaCode = | ||
"function on_message(self)\n" + | ||
"--[[ Some comment\n" + | ||
"Some line2]]--\n" + | ||
"--[[Some comment\n" + | ||
"next line]]--\n" + | ||
"end\n"; | ||
|
||
scanner = new LuaScanner(); | ||
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
" \n" + | ||
"\n" + | ||
"\n" + | ||
"--[[ Some comment\n" + | ||
"Some line2]]--\n" + | ||
"\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
assertEquals(expected, parsed); | ||
|
||
// With _G | ||
luaCode = | ||
|
@@ -337,10 +423,10 @@ public void testRemoveEmptyLifecycleFunctions() throws Exception { | |
scanner.parse(luaCode); | ||
parsed = scanner.getParsedLua(); | ||
expected = | ||
"\n" + | ||
" \n" + | ||
"\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
assertEquals(expected, parsed); | ||
|
||
// local function | ||
luaCode = | ||
|
@@ -359,7 +445,7 @@ public void testRemoveEmptyLifecycleFunctions() throws Exception { | |
"\n" + | ||
"end\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
assertEquals(expected, parsed); | ||
|
||
// local function 2 | ||
luaCode = | ||
|
@@ -378,7 +464,7 @@ public void testRemoveEmptyLifecycleFunctions() throws Exception { | |
"\n" + | ||
"end\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
assertEquals(expected, parsed); | ||
|
||
// not empty | ||
luaCode = | ||
|
@@ -397,7 +483,7 @@ public void testRemoveEmptyLifecycleFunctions() throws Exception { | |
"self.var = 1\n" + | ||
"end\n" + | ||
"local var2 = 2\n"; | ||
assertEquals(expected, trimLines(parsed)); | ||
assertEquals(expected, parsed); | ||
} | ||
|
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -48,6 +48,7 @@ | |
import org.antlr.v4.runtime.ParserRuleContext; | ||
import org.antlr.v4.runtime.Token; | ||
import org.antlr.v4.runtime.tree.TerminalNode; | ||
import org.antlr.v4.runtime.TokenStreamRewriter; | ||
|
||
public class LuaScanner extends LuaParserBaseListener { | ||
|
||
|
@@ -96,8 +97,8 @@ public class LuaScanner extends LuaParserBaseListener { | |
put(LuaParser.CHARSTRING, "'"); | ||
}}; | ||
|
||
private StringBuffer parsedBuffer = null; | ||
private CommonTokenStream tokenStream = null; | ||
private TokenStreamRewriter rewriter; | ||
|
||
private List<String> modules = new ArrayList<String>(); | ||
private List<Property> properties = new ArrayList<Property>(); | ||
|
@@ -180,28 +181,46 @@ public String parse(String str) { | |
modules.clear(); | ||
properties.clear(); | ||
|
||
parsedBuffer = new StringBuffer(str); | ||
|
||
// set up the lexer and parser | ||
// walk the generated parse tree from the | ||
// first Lua chunk | ||
|
||
LuaLexer lexer = new LuaLexer(CharStreams.fromString(str)); | ||
tokenStream = new CommonTokenStream(lexer); | ||
rewriter = new TokenStreamRewriter(tokenStream); | ||
|
||
// Remove comments in rewriter | ||
tokenStream.fill(); | ||
for (Token token : tokenStream.getTokens()) { | ||
if (token.getChannel() == LuaLexer.COMMENTS) { | ||
int type = token.getType(); | ||
if (type == LuaLexer.LINE_COMMENT) { | ||
// Single line comment | ||
rewriter.replace(token, System.lineSeparator()); | ||
} | ||
else if (type == LuaLexer.COMMENT) { | ||
// Multiline comment | ||
rewriter.replace(token, System.lineSeparator().repeat(token.getText().split("\r\n|\r|\n").length - 1)); | ||
} | ||
} | ||
} | ||
|
||
// parse code | ||
LuaParser parser = new LuaParser(tokenStream); | ||
ParseTreeWalker walker = new ParseTreeWalker(); | ||
walker.walk(this, parser.chunk()); | ||
String resultText = rewriter.getText(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's better to include the first From documentation about rewriter:
|
||
TimeProfiler.stop(); | ||
// return the parsed string | ||
return parsedBuffer.toString(); | ||
return resultText; | ||
} | ||
|
||
/** | ||
* Get the parsed Lua code | ||
* @return The parsed Lua code | ||
*/ | ||
public String getParsedLua() { | ||
return parsedBuffer.toString(); | ||
return rewriter.getText(); | ||
} | ||
|
||
/** | ||
|
@@ -238,11 +257,7 @@ private List<Token> getTokens(ParserRuleContext ctx) { | |
|
||
// replace the token with an empty string | ||
private void removeToken(Token token) { | ||
int from = token.getStartIndex(); | ||
int to = from + token.getText().length() - 1; | ||
for(int i = from; i <= to; i++) { | ||
parsedBuffer.replace(i, i + 1, " "); | ||
} | ||
rewriter.delete(token); | ||
} | ||
|
||
private void removeTokens(List<Token> tokens) { | ||
|
@@ -304,22 +319,6 @@ else if (count != resultArgs.length) { | |
return true; | ||
} | ||
|
||
/** | ||
* Callback from ANTLR when a Lua chunk is encountered. We start from the | ||
* main chunk when we call parse() above. This means that the ChunkContext | ||
* will span the entire file and encompass all ANTLR tokens. | ||
* We use this callback to remove all comments. | ||
*/ | ||
@Override | ||
public void enterChunk(LuaParser.ChunkContext ctx) { | ||
List<Token> tokens = getTokens(ctx); | ||
for(Token token : tokens) { | ||
if (token.getChannel() == LuaLexer.COMMENT) { | ||
removeToken(token); | ||
} | ||
} | ||
} | ||
|
||
/** | ||
* Callback from ANTLR when a function call is entered. We use this to grab all | ||
* require() calls and all go.property() calls. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Make sure that after processing, we keep the same count of code lines