Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix #11. Improve preprocessor parsing #32

Merged
merged 2 commits into from
Apr 23, 2015
Merged

Fix #11. Improve preprocessor parsing #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0b241eb
Add proper preprocessor token lexing
AbigailBuccaneer Apr 21, 2015
ca5372f
Add preprocessor support to the parser
AbigailBuccaneer Apr 21, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 4 additions & 14 deletions src/glslplugin/GLSLColorAndFontsPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,7 @@ public class GLSLColorAndFontsPage implements ColorSettingsPage {
new AttributesDescriptor("Brackets", GLSLHighlighter.GLSL_BRACKETS[0]),
new AttributesDescriptor("Identifiers",GLSLHighlighter.GLSL_IDENTIFIER[0]),
new AttributesDescriptor("Text",GLSLHighlighter.GLSL_TEXT[0]),
new AttributesDescriptor("Directives",GLSLHighlighter.GLSL_COMPILER_DIRECTIVE[0])

/*new AttributesDescriptor("Numbers", GLSLHighlighter.GLSL_NUMBER),
new AttributesDescriptor("Type specifiers", GLSLHighlighter.GLSL_TYPE_SPECIFIER),
new AttributesDescriptor("Comments", GLSLHighlighter.GLSL_COMMENT),
new AttributesDescriptor("Version specification", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_VERSION),
new AttributesDescriptor("Extension specification", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_EXTENSION),
new AttributesDescriptor("Compiler pragma", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_PRAGMA),
new AttributesDescriptor("Compiler directives", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_OTHER),
new AttributesDescriptor("Selection and flow", GLSLHighlighter.GLSL_FLOW_KEYWORDS),
new AttributesDescriptor("Identifiers", GLSLHighlighter.GLSL_IDENTIFIER),
new AttributesDescriptor("Parameter qualifiers", GLSLHighlighter.GLSL_PARAMETER_QUALIFIERS),
new AttributesDescriptor("Type qualifiers", GLSLHighlighter.GLSL_TYPE_QUALIFIERS)*/

new AttributesDescriptor("Directives",GLSLHighlighter.GLSL_PREPROCESSOR_DIRECTIVE[0])
};
}

Expand Down Expand Up @@ -98,6 +85,7 @@ public SyntaxHighlighter getHighlighter() {
@NotNull
public String getDemoText() {
return "#version 120\n" +
"precision lowp int;\n" +
"uniform vec3 normal; // surface normal\n" +
"const vec3 light = vec3(5.0, 0.5, 1.0);\n" +
"\n" +
Expand All @@ -110,7 +98,9 @@ public String getDemoText() {
"\n" +
"/* Fragment shader */\n" +
"void main() {\n" +
"#ifdef TEXTURED\n" +
" vec2 tex = gl_TexCoord[0].xy;\n" +
"#endif\n" +
" float diffuse = dot(normal, light);\n" +
" if(diffuse < 0) {\n" +
" diffuse = -diffuse;\n" +
Expand Down
19 changes: 4 additions & 15 deletions src/glslplugin/GLSLHighlighter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,10 @@ public class GLSLHighlighter extends SyntaxHighlighterBase {
{ TextAttributesKey.createTextAttributesKey("GLSL.BRACKETS", DefaultLanguageHighlighterColors.BRACKETS) };
static final TextAttributesKey[] GLSL_IDENTIFIER =
{ TextAttributesKey.createTextAttributesKey("GLSL.IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE",DefaultLanguageHighlighterColors.DOC_COMMENT_MARKUP) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_VERSION =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_VERSION", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_EXTENSION =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_EXTENSION", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_PRAGMA =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_PRAGMA", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_OTHER =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_OTHER", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_PREPROCESSOR_DIRECTIVE =
{ TextAttributesKey.createTextAttributesKey("GLSL.PREPROCESSOR_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA) };
static final TextAttributesKey[] GLSL_PRECISION_STATEMENT =
{ TextAttributesKey.createTextAttributesKey("GLSL.PRECISION_STATEMENT", GLSL_COMPILER_DIRECTIVE[0]) };
{ TextAttributesKey.createTextAttributesKey("GLSL.PRECISION_STATEMENT", DefaultLanguageHighlighterColors.METADATA) };
static final TextAttributesKey[] GLSL_UNKNOWN =
{ TextAttributesKey.createTextAttributesKey("GLSL.UNKNOWN", HighlighterColors.BAD_CHARACTER) };
static final TextAttributesKey[] GLSL_TEXT =
Expand All @@ -94,10 +86,7 @@ public TextAttributesKey[] getTokenHighlights(IElementType type) {
if (type == COMMA) return GLSL_COMMA;
if (type == LEFT_PAREN || type == RIGHT_PAREN) return GLSL_PARENS;
if (type == LEFT_BRACKET || type == RIGHT_BRACKET) return GLSL_BRACKETS;
if (type == COMPILER_DIRECTIVE_VERSION) return GLSL_COMPILER_DIRECTIVE_VERSION;
if (type == COMPILER_DIRECTIVE_EXTENSION) return GLSL_COMPILER_DIRECTIVE_EXTENSION;
if (type == COMPILER_DIRECTIVE_PRAGMA) return GLSL_COMPILER_DIRECTIVE_PRAGMA;
if (type == COMPILER_DIRECTIVE_OTHER) return GLSL_COMPILER_DIRECTIVE_OTHER;
if (PREPROCESSOR_DIRECTIVES.contains(type)) return GLSL_PREPROCESSOR_DIRECTIVE;
if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(type)) return GLSL_TYPE_SPECIFIER;
if (QUALIFIER_TOKENS.contains(type)) return GLSL_TYPE_QUALIFIERS;
if (FLOW_KEYWORDS.contains(type)) return GLSL_FLOW_KEYWORDS;
Expand Down
2 changes: 2 additions & 0 deletions src/glslplugin/lang/elements/GLSLElementTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public class GLSLElementTypes {
public static final IElementType TRANSLATION_UNIT = new GLSLElementType("TRANSLATION_UNIT");
public static final IElementType BLOCK = new GLSLElementType("BLOCK");

public static final IElementType PREPROCESSOR_DIRECTIVE = new GLSLElementType("PREPROCESSOR_DIRECTIVE");

public static final IElementType EMPTY_STATEMENT = new GLSLElementType("EMPTY_STATEMENT");
public static final IElementType JUMP_STATEMENT = new GLSLElementType("JUMP_STATEMENT");

Expand Down
42 changes: 37 additions & 5 deletions src/glslplugin/lang/elements/GLSLTokenTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,42 @@ public class GLSLTokenTypes {

public static final IElementType PRECISION_STATEMENT = new GLSLElementType("PRECISION_STATEMENT");

public static final IElementType COMPILER_DIRECTIVE_VERSION = new GLSLElementType("COMPILER_DIRECTIVE_VERSION");
public static final IElementType COMPILER_DIRECTIVE_EXTENSION = new GLSLElementType("COMPILER_DIRECTIVE_EXTENSION");
public static final IElementType COMPILER_DIRECTIVE_PRAGMA = new GLSLElementType("COMPILER_DIRECTIVE_PRAGMA");
public static final IElementType COMPILER_DIRECTIVE_OTHER = new GLSLElementType("COMPILER_DIRECTIVE_OTHER");
public static final IElementType PREPROCESSOR_BEGIN = new GLSLElementType("PREPROCESSOR_BEGIN");
public static final IElementType PREPROCESSOR_END = new GLSLElementType("PREPROCESSOR_END");
public static final IElementType PREPROCESSOR_DEFINE = new GLSLElementType("PREPROCESSOR_DEFINE");
public static final IElementType PREPROCESSOR_UNDEF = new GLSLElementType("PREPROCESSOR_UNDEF");
public static final IElementType PREPROCESSOR_IF = new GLSLElementType("PREPROCESSOR_IF");
public static final IElementType PREPROCESSOR_IFDEF = new GLSLElementType("PREPROCESSOR_IFDEF");
public static final IElementType PREPROCESSOR_IFNDEF = new GLSLElementType("PREPROCESSOR_IFNDEF");
public static final IElementType PREPROCESSOR_ELSE = new GLSLElementType("PREPROCESSOR_ELSE");
public static final IElementType PREPROCESSOR_ELIF = new GLSLElementType("PREPROCESSOR_ELIF");
public static final IElementType PREPROCESSOR_ENDIF = new GLSLElementType("PREPROCESSOR_ENDIF");
public static final IElementType PREPROCESSOR_ERROR = new GLSLElementType("PREPROCESSOR_ERROR");
public static final IElementType PREPROCESSOR_PRAGMA = new GLSLElementType("PREPROCESSOR_PRAGMA");
public static final IElementType PREPROCESSOR_EXTENSION = new GLSLElementType("PREPROCESSOR_EXTENSION");
public static final IElementType PREPROCESSOR_VERSION = new GLSLElementType("PREPROCESSOR_VERSION");
public static final IElementType PREPROCESSOR_LINE = new GLSLElementType("PREPROCESSOR_LINE");
public static final IElementType PREPROCESSOR_DEFINED = new GLSLElementType("PREPROCESSOR_DEFINED");
public static final IElementType PREPROCESSOR_CONCAT = new GLSLElementType("PREPROCESSOR_CONCAT");

public static final TokenSet PREPROCESSOR_DIRECTIVES = TokenSet.create(
PREPROCESSOR_BEGIN,
PREPROCESSOR_END,
PREPROCESSOR_DEFINE,
PREPROCESSOR_UNDEF,
PREPROCESSOR_IF,
PREPROCESSOR_IFDEF,
PREPROCESSOR_IFNDEF,
PREPROCESSOR_ELSE,
PREPROCESSOR_ELIF,
PREPROCESSOR_ENDIF,
PREPROCESSOR_ERROR,
PREPROCESSOR_PRAGMA,
PREPROCESSOR_EXTENSION,
PREPROCESSOR_VERSION,
PREPROCESSOR_LINE,
PREPROCESSOR_DEFINED,
PREPROCESSOR_CONCAT);

// Type specifiers
public static final TokenSet FLOAT_TYPE_SPECIFIER_NONARRAY = TokenSet.create(FLOAT_TYPE, VEC2_TYPE, VEC3_TYPE, VEC4_TYPE);
Expand Down Expand Up @@ -209,7 +241,7 @@ public class GLSLTokenTypes {
FLAT_KEYWORD,
NOPERSPECTIVE_KEYWORD);

public static final TokenSet COMMENTS = TokenSet.create(COMMENT_BLOCK, COMMENT_LINE, COMPILER_DIRECTIVE_VERSION, COMPILER_DIRECTIVE_EXTENSION, COMPILER_DIRECTIVE_PRAGMA, COMPILER_DIRECTIVE_OTHER, PRECISION_STATEMENT);
public static final TokenSet COMMENTS = TokenSet.create(COMMENT_BLOCK, COMMENT_LINE, PRECISION_STATEMENT);

public static final TokenSet ITERATION_KEYWORDS = TokenSet.create(WHILE_KEYWORD, DO_KEYWORD, FOR_KEYWORD);
public static final TokenSet JUMP_KEYWORDS = TokenSet.create(BREAK_JUMP_STATEMENT, CONTINUE_JUMP_STATEMENT, RETURN_JUMP_STATEMENT, DISCARD_JUMP_STATEMENT);
Expand Down
20 changes: 3 additions & 17 deletions src/glslplugin/lang/elements/statements/GLSLCompoundStatement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.intellij.lang.ASTNode;
import com.intellij.psi.PsiElement;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.util.PsiTreeUtil;
import glslplugin.lang.elements.GLSLTokenTypes;
import org.jetbrains.annotations.NotNull;

Expand All @@ -42,23 +43,8 @@ public GLSLCompoundStatement(@NotNull ASTNode astNode) {

@NotNull
public GLSLStatement[] getStatements() {
// convert the list of children to a list of GLSLStatement objects while performing sanity check.
PsiElement[] children = getChildren();
List<GLSLStatement> result = new ArrayList<GLSLStatement>(children.length);
for (PsiElement child : children) {
if (child instanceof GLSLStatement) {
result.add((GLSLStatement) child);
} else {
final ASTNode node = child.getNode();
if (node != null) {
final IElementType type = node.getElementType();
if (!GLSLTokenTypes.COMMENTS.contains(type)) {
throw new RuntimeException("Compound statement contains non-comment, non-statement element.");
}
}
}
}
return result.toArray(new GLSLStatement[result.size()]);
GLSLStatement[] statements = PsiTreeUtil.getChildrenOfType(this, GLSLStatement.class);
return (statements == null) ? new GLSLStatement[0] : statements;
}

public String toString() {
Expand Down
70 changes: 51 additions & 19 deletions src/glslplugin/lang/parser/GLSLParsing.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class GLSLParsing {
*/
private void skipTo(TokenSet set) {
while (b.getTokenType() != null && set.contains(b.getTokenType())) {
b.advanceLexer();
advanceLexer();
}
}

Expand All @@ -82,7 +82,7 @@ private void skipTo(TokenSet set) {
*/
private void skipTo(IElementType type) {
while (b.getTokenType() != null && b.getTokenType() != type) {
b.advanceLexer();
advanceLexer();
}
}

Expand All @@ -99,7 +99,7 @@ private boolean match(IElementType type, String error) {
}
boolean match = b.getTokenType() == type;
if (match) {
b.advanceLexer();
advanceLexer();
} else {
b.error(error);
}
Expand All @@ -121,7 +121,7 @@ private boolean tryMatch(IElementType... types) {
match |= b.getTokenType() == type;
}
if (match) {
b.advanceLexer();
advanceLexer();
}
return match;
}
Expand All @@ -135,21 +135,53 @@ private boolean tryMatch(IElementType... types) {
private boolean tryMatch(TokenSet types) {
boolean match = types.contains(b.getTokenType());
if (match) {
b.advanceLexer();
advanceLexer();
}
return match;
}

private void advanceLexer() {
b.advanceLexer();
if (b.getTokenType() == PREPROCESSOR_BEGIN) {
parsePreprocessor();
}
}

private void parsePreprocessor() {
// We can't use tryMatch etc. in here because we'll end up
// potentially parsing a preprocessor directive inside this one.
PsiBuilder.Marker preprocessor = b.mark();
while (!b.eof()) {
b.advanceLexer();
if (b.getTokenType() == PREPROCESSOR_END) {
b.advanceLexer();
break;
}
}
preprocessor.done(PREPROCESSOR_DIRECTIVE);

if (b.getTokenType() == PREPROCESSOR_BEGIN) {
parsePreprocessor();
}
}

/**
* Entry for parser. Tries to parse whole shader file.
*/
public void parseTranslationUnit() {
// translation_unit: external_declaration+

PsiBuilder.Marker unit = b.mark();

// We normally parse preprocessor directives whenever we advance the lexer - which means that if the first
// token is a preprocessor directive we won't catch it, so we just parse them all at the beginning here.
while (b.getTokenType() == PREPROCESSOR_BEGIN) {
parsePreprocessor();
}

do {
if (!parseExternalDeclaration()) {
b.advanceLexer();
advanceLexer();
b.error("Unable to parse external declaration.");
}
}
Expand Down Expand Up @@ -278,7 +310,7 @@ private boolean parseExternalDeclaration() {
mark = b.mark();
if(!parseExpression()){
//There is no expression! Consume what triggered me. (Would lead to infinite loop otherwise)
b.advanceLexer();
advanceLexer();
}
tryMatch(SEMICOLON);
mark.error("Expression not allowed here.");
Expand All @@ -287,7 +319,7 @@ private boolean parseExternalDeclaration() {
GLSLTokenTypes.CONSTANT_TOKENS.contains(b.getTokenType())) {
postType.drop();
text = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.error("Unexpected '" + text + "'");
return true;
} else if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(b.getTokenType())) {
Expand Down Expand Up @@ -404,7 +436,7 @@ private void eatInvalidOperators() {
PsiBuilder.Marker mark = b.mark();
while (OPERATORS.contains(b.getTokenType())) {
String operator = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.error("Unexpected operator '" + operator + "'.");
mark = b.mark();
}
Expand Down Expand Up @@ -557,7 +589,7 @@ private void parseForInitStatement() {
if (b.getTokenType() == IDENTIFIER) {
// needs lookahead
PsiBuilder.Marker rollback = b.mark();
b.advanceLexer();
advanceLexer();
if (b.getTokenType() == IDENTIFIER) {
// IDENTIFIER IDENTIFIER means declaration statement
// where the first is the type specifier
Expand Down Expand Up @@ -1066,7 +1098,7 @@ private void parseParameterList() {
PsiBuilder.Marker mark = b.mark();

if (b.getTokenType() == VOID_TYPE) {
b.advanceLexer();
advanceLexer();
} else if (b.getTokenType() == RIGHT_PAREN) {
// do nothing
} else if (parseAssignmentExpression()) {
Expand Down Expand Up @@ -1094,16 +1126,16 @@ private boolean parsePrimaryExpression() {
final IElementType type = b.getTokenType();
if (type == IDENTIFIER) {
final PsiBuilder.Marker mark2 = b.mark();
b.advanceLexer();
advanceLexer();
mark2.done(VARIABLE_NAME);
mark.done(VARIABLE_NAME_EXPRESSION);
return true;
} else if (type == INTEGER_CONSTANT || type == FLOAT_CONSTANT || type == BOOL_CONSTANT) {
b.advanceLexer();
advanceLexer();
mark.done(CONSTANT_EXPRESSION);
return true;
} else if (type == LEFT_PAREN) {
b.advanceLexer();
advanceLexer();
if (!parseExpression()) {
if (b.getTokenType() == RIGHT_PAREN) {
b.error("Expected expression after '('");
Expand All @@ -1126,7 +1158,7 @@ private String parseIdentifier() {
boolean success = b.getTokenType() == IDENTIFIER;
if (success) {
String name = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.done(VARIABLE_NAME);
return name;
} else {
Expand All @@ -1140,7 +1172,7 @@ private String parseFieldIdentifier() {
boolean success = b.getTokenType() == IDENTIFIER;
if (success) {
String name = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.done(FIELD_NAME);
return name;
} else {
Expand Down Expand Up @@ -1185,7 +1217,7 @@ private boolean parseTypeSpecifierNoArray() {
parseStructSpecifier();
mark.done(TYPE_SPECIFIER_STRUCT);
} else if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(b.getTokenType())) {
b.advanceLexer();
advanceLexer();
mark.done(TYPE_SPECIFIER_PRIMITIVE);
} else if (b.getTokenType() == IDENTIFIER) {
parseIdentifier();
Expand Down Expand Up @@ -1275,7 +1307,7 @@ private void parseStructOrParameterDeclarator(IElementType type) {
parseIdentifier();

if (b.getTokenType() == LEFT_BRACKET) {
b.advanceLexer();
advanceLexer();
parseConstantExpression();
match(RIGHT_BRACKET, "Expected ']' after constant expression.");
}
Expand All @@ -1297,7 +1329,7 @@ private void parseQualifierList(boolean validPlacement) {
while (QUALIFIER_TOKENS.contains(b.getTokenType())) {
final PsiBuilder.Marker mark2 = b.mark();

b.advanceLexer();
advanceLexer();

if(validPlacement)
mark2.done(QUALIFIER);
Expand Down
Loading