Skip to content

Commit

Permalink
Add proper preprocessor token lexing
Browse files Browse the repository at this point in the history 
It's not 100% in-line with the C++ preprocessor spec - which involves
odd things like using a different definition for preprocessor number
tokens and compiler number tokens - but this should be adequate for the
vast majority of preprocessor parsing.
  • Loading branch information
@AbigailBuccaneer
AbigailBuccaneer committed Apr 22, 2015
1 parent 3e8645d commit 0b241eb
Show file tree
Hide file tree
Showing 5 changed files with 925 additions and 704 deletions.
18 changes: 4 additions & 14 deletions src/glslplugin/GLSLColorAndFontsPage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,7 @@ public class GLSLColorAndFontsPage implements ColorSettingsPage {
new AttributesDescriptor("Brackets", GLSLHighlighter.GLSL_BRACKETS[0]),
new AttributesDescriptor("Identifiers",GLSLHighlighter.GLSL_IDENTIFIER[0]),
new AttributesDescriptor("Text",GLSLHighlighter.GLSL_TEXT[0]),
new AttributesDescriptor("Directives",GLSLHighlighter.GLSL_COMPILER_DIRECTIVE[0])

/*new AttributesDescriptor("Numbers", GLSLHighlighter.GLSL_NUMBER),
new AttributesDescriptor("Type specifiers", GLSLHighlighter.GLSL_TYPE_SPECIFIER),
new AttributesDescriptor("Comments", GLSLHighlighter.GLSL_COMMENT),
new AttributesDescriptor("Version specification", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_VERSION),
new AttributesDescriptor("Extension specification", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_EXTENSION),
new AttributesDescriptor("Compiler pragma", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_PRAGMA),
new AttributesDescriptor("Compiler directives", GLSLHighlighter.GLSL_COMPILER_DIRECTIVE_OTHER),
new AttributesDescriptor("Selection and flow", GLSLHighlighter.GLSL_FLOW_KEYWORDS),
new AttributesDescriptor("Identifiers", GLSLHighlighter.GLSL_IDENTIFIER),
new AttributesDescriptor("Parameter qualifiers", GLSLHighlighter.GLSL_PARAMETER_QUALIFIERS),
new AttributesDescriptor("Type qualifiers", GLSLHighlighter.GLSL_TYPE_QUALIFIERS)*/

new AttributesDescriptor("Directives",GLSLHighlighter.GLSL_PREPROCESSOR_DIRECTIVE[0])
};
}

Expand Down Expand Up @@ -98,6 +85,7 @@ public SyntaxHighlighter getHighlighter() {
@NotNull
public String getDemoText() {
return "#version 120\n" +
"precision lowp int;\n" +
"uniform vec3 normal; // surface normal\n" +
"const vec3 light = vec3(5.0, 0.5, 1.0);\n" +
"\n" +
Expand All @@ -110,7 +98,9 @@ public String getDemoText() {
"\n" +
"/* Fragment shader */\n" +
"void main() {\n" +
"#ifdef TEXTURED\n" +
" vec2 tex = gl_TexCoord[0].xy;\n" +
"#endif\n" +
" float diffuse = dot(normal, light);\n" +
" if(diffuse < 0) {\n" +
" diffuse = -diffuse;\n" +
Expand Down
19 changes: 4 additions & 15 deletions src/glslplugin/GLSLHighlighter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,10 @@ public class GLSLHighlighter extends SyntaxHighlighterBase {
{ TextAttributesKey.createTextAttributesKey("GLSL.BRACKETS", DefaultLanguageHighlighterColors.BRACKETS) };
static final TextAttributesKey[] GLSL_IDENTIFIER =
{ TextAttributesKey.createTextAttributesKey("GLSL.IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE",DefaultLanguageHighlighterColors.DOC_COMMENT_MARKUP) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_VERSION =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_VERSION", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_EXTENSION =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_EXTENSION", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_PRAGMA =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_PRAGMA", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_COMPILER_DIRECTIVE_OTHER =
{ TextAttributesKey.createTextAttributesKey("GLSL.COMPILER_DIRECTIVE_OTHER", GLSL_COMPILER_DIRECTIVE[0]) };
static final TextAttributesKey[] GLSL_PREPROCESSOR_DIRECTIVE =
{ TextAttributesKey.createTextAttributesKey("GLSL.PREPROCESSOR_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA) };
static final TextAttributesKey[] GLSL_PRECISION_STATEMENT =
{ TextAttributesKey.createTextAttributesKey("GLSL.PRECISION_STATEMENT", GLSL_COMPILER_DIRECTIVE[0]) };
{ TextAttributesKey.createTextAttributesKey("GLSL.PRECISION_STATEMENT", DefaultLanguageHighlighterColors.METADATA) };
static final TextAttributesKey[] GLSL_UNKNOWN =
{ TextAttributesKey.createTextAttributesKey("GLSL.UNKNOWN", HighlighterColors.BAD_CHARACTER) };
static final TextAttributesKey[] GLSL_TEXT =
Expand All @@ -94,10 +86,7 @@ public TextAttributesKey[] getTokenHighlights(IElementType type) {
if (type == COMMA) return GLSL_COMMA;
if (type == LEFT_PAREN || type == RIGHT_PAREN) return GLSL_PARENS;
if (type == LEFT_BRACKET || type == RIGHT_BRACKET) return GLSL_BRACKETS;
if (type == COMPILER_DIRECTIVE_VERSION) return GLSL_COMPILER_DIRECTIVE_VERSION;
if (type == COMPILER_DIRECTIVE_EXTENSION) return GLSL_COMPILER_DIRECTIVE_EXTENSION;
if (type == COMPILER_DIRECTIVE_PRAGMA) return GLSL_COMPILER_DIRECTIVE_PRAGMA;
if (type == COMPILER_DIRECTIVE_OTHER) return GLSL_COMPILER_DIRECTIVE_OTHER;
if (PREPROCESSOR_DIRECTIVES.contains(type)) return GLSL_PREPROCESSOR_DIRECTIVE;
if (TYPE_SPECIFIER_NONARRAY_TOKENS.contains(type)) return GLSL_TYPE_SPECIFIER;
if (QUALIFIER_TOKENS.contains(type)) return GLSL_TYPE_QUALIFIERS;
if (FLOW_KEYWORDS.contains(type)) return GLSL_FLOW_KEYWORDS;
Expand Down
42 changes: 37 additions & 5 deletions src/glslplugin/lang/elements/GLSLTokenTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,42 @@ public class GLSLTokenTypes {

public static final IElementType PRECISION_STATEMENT = new GLSLElementType("PRECISION_STATEMENT");

public static final IElementType COMPILER_DIRECTIVE_VERSION = new GLSLElementType("COMPILER_DIRECTIVE_VERSION");
public static final IElementType COMPILER_DIRECTIVE_EXTENSION = new GLSLElementType("COMPILER_DIRECTIVE_EXTENSION");
public static final IElementType COMPILER_DIRECTIVE_PRAGMA = new GLSLElementType("COMPILER_DIRECTIVE_PRAGMA");
public static final IElementType COMPILER_DIRECTIVE_OTHER = new GLSLElementType("COMPILER_DIRECTIVE_OTHER");
public static final IElementType PREPROCESSOR_BEGIN = new GLSLElementType("PREPROCESSOR_BEGIN");
public static final IElementType PREPROCESSOR_END = new GLSLElementType("PREPROCESSOR_END");
public static final IElementType PREPROCESSOR_DEFINE = new GLSLElementType("PREPROCESSOR_DEFINE");
public static final IElementType PREPROCESSOR_UNDEF = new GLSLElementType("PREPROCESSOR_UNDEF");
public static final IElementType PREPROCESSOR_IF = new GLSLElementType("PREPROCESSOR_IF");
public static final IElementType PREPROCESSOR_IFDEF = new GLSLElementType("PREPROCESSOR_IFDEF");
public static final IElementType PREPROCESSOR_IFNDEF = new GLSLElementType("PREPROCESSOR_IFNDEF");
public static final IElementType PREPROCESSOR_ELSE = new GLSLElementType("PREPROCESSOR_ELSE");
public static final IElementType PREPROCESSOR_ELIF = new GLSLElementType("PREPROCESSOR_ELIF");
public static final IElementType PREPROCESSOR_ENDIF = new GLSLElementType("PREPROCESSOR_ENDIF");
public static final IElementType PREPROCESSOR_ERROR = new GLSLElementType("PREPROCESSOR_ERROR");
public static final IElementType PREPROCESSOR_PRAGMA = new GLSLElementType("PREPROCESSOR_PRAGMA");
public static final IElementType PREPROCESSOR_EXTENSION = new GLSLElementType("PREPROCESSOR_EXTENSION");
public static final IElementType PREPROCESSOR_VERSION = new GLSLElementType("PREPROCESSOR_VERSION");
public static final IElementType PREPROCESSOR_LINE = new GLSLElementType("PREPROCESSOR_LINE");
public static final IElementType PREPROCESSOR_DEFINED = new GLSLElementType("PREPROCESSOR_DEFINED");
public static final IElementType PREPROCESSOR_CONCAT = new GLSLElementType("PREPROCESSOR_CONCAT");

public static final TokenSet PREPROCESSOR_DIRECTIVES = TokenSet.create(
PREPROCESSOR_BEGIN,
PREPROCESSOR_END,
PREPROCESSOR_DEFINE,
PREPROCESSOR_UNDEF,
PREPROCESSOR_IF,
PREPROCESSOR_IFDEF,
PREPROCESSOR_IFNDEF,
PREPROCESSOR_ELSE,
PREPROCESSOR_ELIF,
PREPROCESSOR_ENDIF,
PREPROCESSOR_ERROR,
PREPROCESSOR_PRAGMA,
PREPROCESSOR_EXTENSION,
PREPROCESSOR_VERSION,
PREPROCESSOR_LINE,
PREPROCESSOR_DEFINED,
PREPROCESSOR_CONCAT);

// Type specifiers
public static final TokenSet FLOAT_TYPE_SPECIFIER_NONARRAY = TokenSet.create(FLOAT_TYPE, VEC2_TYPE, VEC3_TYPE, VEC4_TYPE);
Expand Down Expand Up @@ -209,7 +241,7 @@ public class GLSLTokenTypes {
FLAT_KEYWORD,
NOPERSPECTIVE_KEYWORD);

public static final TokenSet COMMENTS = TokenSet.create(COMMENT_BLOCK, COMMENT_LINE, COMPILER_DIRECTIVE_VERSION, COMPILER_DIRECTIVE_EXTENSION, COMPILER_DIRECTIVE_PRAGMA, COMPILER_DIRECTIVE_OTHER, PRECISION_STATEMENT);
public static final TokenSet COMMENTS = TokenSet.create(COMMENT_BLOCK, COMMENT_LINE, PRECISION_STATEMENT);

public static final TokenSet ITERATION_KEYWORDS = TokenSet.create(WHILE_KEYWORD, DO_KEYWORD, FOR_KEYWORD);
public static final TokenSet JUMP_KEYWORDS = TokenSet.create(BREAK_JUMP_STATEMENT, CONTINUE_JUMP_STATEMENT, RETURN_JUMP_STATEMENT, DISCARD_JUMP_STATEMENT);
Expand Down
42 changes: 34 additions & 8 deletions src/glslplugin/lang/scanner/GLSL.flex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import java.util.ArrayList;
%function advance
%type IElementType

%state PREPROCESSOR

%{
%}

Expand All @@ -41,7 +43,8 @@ OCTAL_DIGIT = [0-7]
HEX_DIGIT = [0-9A-Fa-f]
NON_DIGIT = [_a-zA-Z]

WHITE_SPACE = [ \n\r\t\f]
LINE_TERMINATOR = \r|\n|\r\n
WHITE_SPACE = [ \t\f]

IDENTIFIER = {NON_DIGIT}({NON_DIGIT} | {DIGIT})*

Expand All @@ -64,10 +67,40 @@ GLSL_ES_TYPE = void|float|(u?int)|bool|((i|b|u)?vec([2-4]))|(mat[2-4](x[2-4])?)|
GLSL_ES_PRECISION_MODIFIER = (high|medium|low)p

%%


/**
* LEXICAL RULES:
*/

\\{LINE_TERMINATOR} { return WHITE_SPACE; }

/* Preprocessor rules */
<PREPROCESSOR> {
{WHITE_SPACE}+ { return WHITE_SPACE; }
{LINE_TERMINATOR} { yybegin(YYINITIAL); return PREPROCESSOR_END; }
define { return PREPROCESSOR_DEFINE; }
undef { return PREPROCESSOR_UNDEF; }
if { return PREPROCESSOR_IF; }
ifdef { return PREPROCESSOR_IFDEF; }
ifndef { return PREPROCESSOR_IFNDEF; }
else { return PREPROCESSOR_ELSE; }
elif { return PREPROCESSOR_ELIF; }
endif { return PREPROCESSOR_ENDIF; }
error { return PREPROCESSOR_ERROR; }
pragma { return PREPROCESSOR_PRAGMA; }
extension { return PREPROCESSOR_EXTENSION; }
version { return PREPROCESSOR_VERSION; }
line { return PREPROCESSOR_LINE; }
defined { return PREPROCESSOR_DEFINED; }
## { return PREPROCESSOR_CONCAT; }
}

<YYINITIAL> {
# { yybegin(PREPROCESSOR); return PREPROCESSOR_BEGIN; }
({WHITE_SPACE}|{LINE_TERMINATOR})+ {return WHITE_SPACE;}
}

true {return BOOL_CONSTANT; }
false {return BOOL_CONSTANT; }

Expand Down Expand Up @@ -201,13 +234,6 @@ precision{WHITE_SPACE}+{GLSL_ES_PRECISION_MODIFIER}{WHITE_SPACE}+{GLSL_ES_TYPE}"
";" {return SEMICOLON; }
"," {return COMMA; }

{WHITE_SPACE}+ {return WHITE_SPACE;}

#{WHITE_SPACE}*version[^\n\r]* {return COMPILER_DIRECTIVE_VERSION;}
#{WHITE_SPACE}*extension[^\n\r]* {return COMPILER_DIRECTIVE_EXTENSION;}
#{WHITE_SPACE}*pragma[^\n\r]* {return COMPILER_DIRECTIVE_PRAGMA;}
#{WHITE_SPACE}*[^\n\r]* {return COMPILER_DIRECTIVE_OTHER;}

{IDENTIFIER} {return IDENTIFIER;}

{INTEGER_CONSTANT} {return INTEGER_CONSTANT; }
Expand Down
Loading

0 comments on commit 0b241eb

Please sign in to comment.