Skip to content

Commit

Permalink
Merge pull request #32 from AbigailBuccaneer/preprocessor
Browse files Browse the repository at this point in the history 
Fix #11. Improve preprocessor parsing
  • Loading branch information
@Darkyenus
Darkyenus committed Apr 23, 2015
2 parents 3e8645d + ca5372f commit 8700a2a
Show file tree
Hide file tree
Showing 8 changed files with 981 additions and 740 deletions.
18 changes: 4 additions & 14 deletions src/glslplugin/GLSLColorAndFontsPage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,7 @@ public class GLSLColorAndFontsPage implements ColorSettingsPage {
new AttributesDescriptor("Brackets", GLSLHighlighter.GLSL_BRACKETS[0]),
new AttributesDescriptor("Identifiers",GLSLHighlighter.GLSL_IDENTIFIER[0]),
new AttributesDescriptor("Text",GLSLHighlighter.GLSL_TEXT[0]),
new AttributesDescriptor("Directives",GLSLHighlighter.GLSL_COMPILER_DIRECTIVE[0])

/*new AttributesDescriptor("Numbers", GLSLHighlighter.GLSL_NUMBER),
new AttributesDescriptor("Type specifiers", GLSLHighlighter.GLSL_TYPE_SPECIFIER),
new AttributesDescriptor("Comments", GLSLHighlighter.GLSL_COMMENT),
new AttributesDescriptor("Version specification", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_VERSION),
new AttributesDescriptor("Extension specification", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_EXTENSION),
new AttributesDescriptor("Compiler pragma", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_PRAGMA),
new AttributesDescriptor("Compiler directives", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_OTHER),
new AttributesDescriptor("Selection and flow", GLSLHighlighter.GLSL_FLOW_KEYWORDS),
new AttributesDescriptor("Identifiers", GLSLHighlighter.GLSL_IDENTIFIER),
new AttributesDescriptor("Parameter qualifiers", GLSLHighlighter.GLSL_PARAMETER_QUALIFIERS),
new AttributesDescriptor("Type qualifiers", GLSLHighlighter.GLSL_TYPE_QUALIFIERS)*/

new AttributesDescriptor("Directives",GLSLHighlighter.GLSL_PREPROCESSOR_DIRECTIVE[0])
};
}

Expand Down Expand Up @@ -98,6 +85,7 @@ public SyntaxHighlighter getHighlighter() {
@NotNull
public String getDemoText() {
return "#version 120\n" +
"precision lowp int;\n" +
"uniform vec3 normal; // surface normal\n" +
"const vec3 light = vec3(5.0, 0.5, 1.0);\n" +
"\n" +
Expand All @@ -110,7 +98,9 @@ public String getDemoText() {
"\n" +
"/* Fragment shader */\n" +
"void main() {\n" +
"#ifdef TEXTURED\n" +
" vec2 tex = gl_TexCoord[0].xy;\n" +
"#endif\n" +
" float diffuse = dot(normal, light);\n" +
" if(diffuse < 0) {\n" +
" diffuse = -diffuse;\n" +
Expand Down
19 changes: 4 additions & 15 deletions src/glslplugin/GLSLHighlighter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,10 @@ public class GLSLHighlighter extends SyntaxHighlighterBase {
{ TextAttributesKey.createTextAttributesKey("GLSL.BRACKETS", DefaultLanguageHighlighterColors.BRACKETS) };
static final TextAttributesKey[] GLSL_IDENTIFIER =
{ TextAttributesKey.createTextAttributesKey("GLSL.IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE",DefaultLanguageHighlighterColors.DOC_COMMENT_MARKUP) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_VERSION =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_VERSION", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_EXTENSION =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_EXTENSION", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_PRAGMA =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_PRAGMA", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_OTHER =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_OTHER", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_PREPROCESSOR_DIRECTIVE =
{ TextAttributesKey.createTextAttributesKey("GLSL.PREPROCESSOR_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA) };
static final TextAttributesKey[] GLSL_PRECISION_STATEMENT =
{ TextAttributesKey.createTextAttributesKey("GLSL.PRECISION_STATEMENT", GLSL_COMPILER_DIRECTIVE[0]) };
{ TextAttributesKey.createTextAttributesKey("GLSL.PRECISION_STATEMENT", DefaultLanguageHighlighterColors.METADATA) };
static final TextAttributesKey[] GLSL_UNKNOWN =
{ TextAttributesKey.createTextAttributesKey("GLSL.UNKNOWN", HighlighterColors.BAD_CHARACTER) };
static final TextAttributesKey[] GLSL_TEXT =
Expand All @@ -94,10 +86,7 @@ public TextAttributesKey[] getTokenHighlights(IElementType type) {
if (type == COMMA) return GLSL_COMMA;
if (type == LEFT_PAREN || type == RIGHT_PAREN) return GLSL_PARENS;
if (type == LEFT_BRACKET || type == RIGHT_BRACKET) return GLSL_BRACKETS;
if (type == COMPILER_DIRECTIVE_VERSION) return GLSL_COMPILER_DIRECTIVE_VERSION;
if (type == COMPILER_DIRECTIVE_EXTENSION) return GLSL_COMPILER_DIRECTIVE_EXTENSION;
if (type == COMPILER_DIRECTIVE_PRAGMA) return GLSL_COMPILER_DIRECTIVE_PRAGMA;
if (type == COMPILER_DIRECTIVE_OTHER) return GLSL_COMPILER_DIRECTIVE_OTHER;
if (PREPROCESSOR_DIRECTIVES.contains(type)) return GLSL_PREPROCESSOR_DIRECTIVE;
if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(type)) return GLSL_TYPE_SPECIFIER;
if (QUALIFIER_TOKENS.contains(type)) return GLSL_TYPE_QUALIFIERS;
if (FLOW_KEYWORDS.contains(type)) return GLSL_FLOW_KEYWORDS;
Expand Down
2 changes: 2 additions & 0 deletions src/glslplugin/lang/elements/GLSLElementTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public class GLSLElementTypes {
public static final IElementType TRANSLATION_UNIT = new GLSLElementType("TRANSLATION_UNIT");
public static final IElementType BLOCK = new GLSLElementType("BLOCK");

public static final IElementType PREPROCESSOR_DIRECTIVE = new GLSLElementType("PREPROCESSOR_DIRECTIVE");

public static final IElementType EMPTY_STATEMENT = new GLSLElementType("EMPTY_STATEMENT");
public static final IElementType JUMP_STATEMENT = new GLSLElementType("JUMP_STATEMENT");

Expand Down
42 changes: 37 additions & 5 deletions src/glslplugin/lang/elements/GLSLTokenTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,42 @@ public class GLSLTokenTypes {

public static final IElementType PRECISION_STATEMENT = new GLSLElementType("PRECISION_STATEMENT");

public static final IElementType COMPILER_DIRECTIVE_VERSION = new GLSLElementType("COMPILER_DIRECTIVE_VERSION");
public static final IElementType COMPILER_DIRECTIVE_EXTENSION = new GLSLElementType("COMPILER_DIRECTIVE_EXTENSION");
public static final IElementType COMPILER_DIRECTIVE_PRAGMA = new GLSLElementType("COMPILER_DIRECTIVE_PRAGMA");
public static final IElementType COMPILER_DIRECTIVE_OTHER = new GLSLElementType("COMPILER_DIRECTIVE_OTHER");
public static final IElementType PREPROCESSOR_BEGIN = new GLSLElementType("PREPROCESSOR_BEGIN");
public static final IElementType PREPROCESSOR_END = new GLSLElementType("PREPROCESSOR_END");
public static final IElementType PREPROCESSOR_DEFINE = new GLSLElementType("PREPROCESSOR_DEFINE");
public static final IElementType PREPROCESSOR_UNDEF = new GLSLElementType("PREPROCESSOR_UNDEF");
public static final IElementType PREPROCESSOR_IF = new GLSLElementType("PREPROCESSOR_IF");
public static final IElementType PREPROCESSOR_IFDEF = new GLSLElementType("PREPROCESSOR_IFDEF");
public static final IElementType PREPROCESSOR_IFNDEF = new GLSLElementType("PREPROCESSOR_IFNDEF");
public static final IElementType PREPROCESSOR_ELSE = new GLSLElementType("PREPROCESSOR_ELSE");
public static final IElementType PREPROCESSOR_ELIF = new GLSLElementType("PREPROCESSOR_ELIF");
public static final IElementType PREPROCESSOR_ENDIF = new GLSLElementType("PREPROCESSOR_ENDIF");
public static final IElementType PREPROCESSOR_ERROR = new GLSLElementType("PREPROCESSOR_ERROR");
public static final IElementType PREPROCESSOR_PRAGMA = new GLSLElementType("PREPROCESSOR_PRAGMA");
public static final IElementType PREPROCESSOR_EXTENSION = new GLSLElementType("PREPROCESSOR_EXTENSION");
public static final IElementType PREPROCESSOR_VERSION = new GLSLElementType("PREPROCESSOR_VERSION");
public static final IElementType PREPROCESSOR_LINE = new GLSLElementType("PREPROCESSOR_LINE");
public static final IElementType PREPROCESSOR_DEFINED = new GLSLElementType("PREPROCESSOR_DEFINED");
public static final IElementType PREPROCESSOR_CONCAT = new GLSLElementType("PREPROCESSOR_CONCAT");

public static final TokenSet PREPROCESSOR_DIRECTIVES = TokenSet.create(
PREPROCESSOR_BEGIN,
PREPROCESSOR_END,
PREPROCESSOR_DEFINE,
PREPROCESSOR_UNDEF,
PREPROCESSOR_IF,
PREPROCESSOR_IFDEF,
PREPROCESSOR_IFNDEF,
PREPROCESSOR_ELSE,
PREPROCESSOR_ELIF,
PREPROCESSOR_ENDIF,
PREPROCESSOR_ERROR,
PREPROCESSOR_PRAGMA,
PREPROCESSOR_EXTENSION,
PREPROCESSOR_VERSION,
PREPROCESSOR_LINE,
PREPROCESSOR_DEFINED,
PREPROCESSOR_CONCAT);

// Type specifiers
public static final TokenSet FLOAT_TYPE_SPECIFIER_NONARRAY = TokenSet.create(FLOAT_TYPE, VEC2_TYPE, VEC3_TYPE, VEC4_TYPE);
Expand Down Expand Up @@ -209,7 +241,7 @@ public class GLSLTokenTypes {
FLAT_KEYWORD,
NOPERSPECTIVE_KEYWORD);

public static final TokenSet COMMENTS = TokenSet.create(COMMENT_BLOCK, COMMENT_LINE, COMPILER_DIRECTIVE_VERSION, COMPILER_DIRECTIVE_EXTENSION, COMPILER_DIRECTIVE_PRAGMA, COMPILER_DIRECTIVE_OTHER, PRECISION_STATEMENT);
public static final TokenSet COMMENTS = TokenSet.create(COMMENT_BLOCK, COMMENT_LINE, PRECISION_STATEMENT);

public static final TokenSet ITERATION_KEYWORDS = TokenSet.create(WHILE_KEYWORD, DO_KEYWORD, FOR_KEYWORD);
public static final TokenSet JUMP_KEYWORDS = TokenSet.create(BREAK_JUMP_STATEMENT, CONTINUE_JUMP_STATEMENT, RETURN_JUMP_STATEMENT, DISCARD_JUMP_STATEMENT);
Expand Down
20 changes: 3 additions & 17 deletions src/glslplugin/lang/elements/statements/GLSLCompoundStatement.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.intellij.lang.ASTNode;
import com.intellij.psi.PsiElement;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.util.PsiTreeUtil;
import glslplugin.lang.elements.GLSLTokenTypes;
import org.jetbrains.annotations.NotNull;

Expand All @@ -42,23 +43,8 @@ public GLSLCompoundStatement(@NotNull ASTNode astNode) {

@NotNull
public GLSLStatement[] getStatements() {
// convert the list of children to a list of GLSLStatement objects while performing sanity check.
PsiElement[] children = getChildren();
List<GLSLStatement> result = new ArrayList<GLSLStatement>(children.length);
for (PsiElement child : children) {
if (child instanceof GLSLStatement) {
result.add((GLSLStatement) child);
} else {
final ASTNode node = child.getNode();
if (node != null) {
final IElementType type = node.getElementType();
if (!GLSLTokenTypes.COMMENTS.contains(type)) {
throw new RuntimeException("Compound statement contains non-comment, non-statement element.");
}
}
}
}
return result.toArray(new GLSLStatement[result.size()]);
GLSLStatement[] statements = PsiTreeUtil.getChildrenOfType(this, GLSLStatement.class);
return (statements == null) ? new GLSLStatement[0] : statements;
}

public String toString() {
Expand Down
70 changes: 51 additions & 19 deletions src/glslplugin/lang/parser/GLSLParsing.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class GLSLParsing {
*/
private void skipTo(TokenSet set) {
while (b.getTokenType() != null && set.contains(b.getTokenType())) {
b.advanceLexer();
advanceLexer();
}
}

Expand All @@ -82,7 +82,7 @@ private void skipTo(TokenSet set) {
*/
private void skipTo(IElementType type) {
while (b.getTokenType() != null && b.getTokenType() != type) {
b.advanceLexer();
advanceLexer();
}
}

Expand All @@ -99,7 +99,7 @@ private boolean match(IElementType type, String error) {
}
boolean match = b.getTokenType() == type;
if (match) {
b.advanceLexer();
advanceLexer();
} else {
b.error(error);
}
Expand All @@ -121,7 +121,7 @@ private boolean tryMatch(IElementType... types) {
match |= b.getTokenType() == type;
}
if (match) {
b.advanceLexer();
advanceLexer();
}
return match;
}
Expand All @@ -135,21 +135,53 @@ private boolean tryMatch(IElementType... types) {
private boolean tryMatch(TokenSet types) {
boolean match = types.contains(b.getTokenType());
if (match) {
b.advanceLexer();
advanceLexer();
}
return match;
}

private void advanceLexer() {
b.advanceLexer();
if (b.getTokenType() == PREPROCESSOR_BEGIN) {
parsePreprocessor();
}
}

private void parsePreprocessor() {
// We can't use tryMatch etc. in here because we'll end up
// potentially parsing a preprocessor directive inside this one.
PsiBuilder.Marker preprocessor = b.mark();
while (!b.eof()) {
b.advanceLexer();
if (b.getTokenType() == PREPROCESSOR_END) {
b.advanceLexer();
break;
}
}
preprocessor.done(PREPROCESSOR_DIRECTIVE);

if (b.getTokenType() == PREPROCESSOR_BEGIN) {
parsePreprocessor();
}
}

/**
* Entry for parser. Tries to parse whole shader file.
*/
public void parseTranslationUnit() {
// translation_unit: external_declaration+

PsiBuilder.Marker unit = b.mark();

// We normally parse preprocessor directives whenever we advance the lexer - which means that if the first
// token is a preprocessor directive we won't catch it, so we just parse them all at the beginning here.
while (b.getTokenType() == PREPROCESSOR_BEGIN) {
parsePreprocessor();
}

do {
if (!parseExternalDeclaration()) {
b.advanceLexer();
advanceLexer();
b.error("Unable to parse external declaration.");
}
}
Expand Down Expand Up @@ -278,7 +310,7 @@ private boolean parseExternalDeclaration() {
mark = b.mark();
if(!parseExpression()){
//There is no expression! Consume what triggered me. (Would lead to infinite loop otherwise)
b.advanceLexer();
advanceLexer();
}
tryMatch(SEMICOLON);
mark.error("Expression not allowed here.");
Expand All @@ -287,7 +319,7 @@ private boolean parseExternalDeclaration() {
GLSLTokenTypes.CONSTANT_TOKENS.contains(b.getTokenType())) {
postType.drop();
text = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.error("Unexpected '" + text + "'");
return true;
} else if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(b.getTokenType())) {
Expand Down Expand Up @@ -404,7 +436,7 @@ private void eatInvalidOperators() {
PsiBuilder.Marker mark = b.mark();
while (OPERATORS.contains(b.getTokenType())) {
String operator = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.error("Unexpected operator '" + operator + "'.");
mark = b.mark();
}
Expand Down Expand Up @@ -557,7 +589,7 @@ private void parseForInitStatement() {
if (b.getTokenType() == IDENTIFIER) {
// needs lookahead
PsiBuilder.Marker rollback = b.mark();
b.advanceLexer();
advanceLexer();
if (b.getTokenType() == IDENTIFIER) {
// IDENTIFIER IDENTIFIER means declaration statement
// where the first is the type specifier
Expand Down Expand Up @@ -1066,7 +1098,7 @@ private void parseParameterList() {
PsiBuilder.Marker mark = b.mark();

if (b.getTokenType() == VOID_TYPE) {
b.advanceLexer();
advanceLexer();
} else if (b.getTokenType() == RIGHT_PAREN) {
// do nothing
} else if (parseAssignmentExpression()) {
Expand Down Expand Up @@ -1094,16 +1126,16 @@ private boolean parsePrimaryExpression() {
final IElementType type = b.getTokenType();
if (type == IDENTIFIER) {
final PsiBuilder.Marker mark2 = b.mark();
b.advanceLexer();
advanceLexer();
mark2.done(VARIABLE_NAME);
mark.done(VARIABLE_NAME_EXPRESSION);
return true;
} else if (type == INTEGER_CONSTANT || type == FLOAT_CONSTANT || type == BOOL_CONSTANT) {
b.advanceLexer();
advanceLexer();
mark.done(CONSTANT_EXPRESSION);
return true;
} else if (type == LEFT_PAREN) {
b.advanceLexer();
advanceLexer();
if (!parseExpression()) {
if (b.getTokenType() == RIGHT_PAREN) {
b.error("Expected expression after '('");
Expand All @@ -1126,7 +1158,7 @@ private String parseIdentifier() {
boolean success = b.getTokenType() == IDENTIFIER;
if (success) {
String name = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.done(VARIABLE_NAME);
return name;
} else {
Expand All @@ -1140,7 +1172,7 @@ private String parseFieldIdentifier() {
boolean success = b.getTokenType() == IDENTIFIER;
if (success) {
String name = b.getTokenText();
b.advanceLexer();
advanceLexer();
mark.done(FIELD_NAME);
return name;
} else {
Expand Down Expand Up @@ -1185,7 +1217,7 @@ private boolean parseTypeSpecifierNoArray() {
parseStructSpecifier();
mark.done(TYPE_SPECIFIER_STRUCT);
} else if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(b.getTokenType())) {
b.advanceLexer();
advanceLexer();
mark.done(TYPE_SPECIFIER_PRIMITIVE);
} else if (b.getTokenType() == IDENTIFIER) {
parseIdentifier();
Expand Down Expand Up @@ -1275,7 +1307,7 @@ private void parseStructOrParameterDeclarator(IElementType type) {
parseIdentifier();

if (b.getTokenType() == LEFT_BRACKET) {
b.advanceLexer();
advanceLexer();
parseConstantExpression();
match(RIGHT_BRACKET, "Expected ']' after constant expression.");
}
Expand All @@ -1297,7 +1329,7 @@ private void parseQualifierList(boolean validPlacement) {
while (QUALIFIER_TOKENS.contains(b.getTokenType())) {
final PsiBuilder.Marker mark2 = b.mark();

b.advanceLexer();
advanceLexer();

if(validPlacement)
mark2.done(QUALIFIER);
Expand Down
Loading

0 comments on commit 8700a2a

Please sign in to comment.