Skip to content

Commit

Permalink
Merge pull request #57 from jansorg/jansorg/lexer-state-cleanup
Browse files Browse the repository at this point in the history 
Fix state handling of the lexer
  • Loading branch information
@JojOatXGME
JojOatXGME committed Apr 1, 2023
2 parents bb547b4 + cd8b85b commit a924917
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 102 deletions.
42 changes: 4 additions & 38 deletions src/main/java/org/nixos/idea/lang/NixLexer.java
View file
Original file line number Diff line number Diff line change
@@ -1,51 +1,17 @@
package org.nixos.idea.lang;

import com.intellij.lexer.FlexAdapter;
import com.intellij.lexer.FlexLexer;
import com.intellij.psi.tree.IElementType;

import java.io.IOException;

public class NixLexer extends FlexAdapter {
// todo: Implement RestartableLexer when it becomes non-experimental. See
// https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010305800/comments/360002861979

public NixLexer() {
// We need this anonymous wrapper class to map between internal states
// of Flex and external states for IntelliJ. See
// https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010305800
super(new FlexLexer() {
private final _NixLexer lexer = new _NixLexer(null);

@Override
public void yybegin(int state) {
lexer.restoreState(state);
}

@Override
public int yystate() {
return lexer.getStateIndex();
}

@Override
public int getTokenStart() {
return lexer.getTokenStart();
}

@Override
public int getTokenEnd() {
return lexer.getTokenEnd();
}

@Override
public IElementType advance() throws IOException {
return lexer.advance();
}

super(new _NixLexer(null) {
@Override
public void reset(CharSequence buf, int start, int end, int initialState) {
lexer.reset(buf, start, end, _NixLexer.YYINITIAL);
lexer.restoreState(initialState);
public void reset(CharSequence buffer, int start, int end, int initialState) {
onReset();
super.reset(buffer, start, end, initialState);
}
});
}
Expand Down
122 changes: 59 additions & 63 deletions src/main/lang/Nix.flex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,53 +2,39 @@ package org.nixos.idea.lang;

import com.intellij.lexer.FlexLexer;
import com.intellij.psi.tree.IElementType;
import it.unimi.dsi.fastutil.longs.Long2IntMap;
import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongList;
import it.unimi.dsi.fastutil.ints.AbstractIntList;
import it.unimi.dsi.fastutil.ints.IntArrayList;

import static org.nixos.idea.psi.NixTypes.*;

%%

%{
private final LongList states = new LongArrayList();
private final Long2IntMap stateIndexMap = new Long2IntOpenHashMap();

{
states.add(YYINITIAL);
stateIndexMap.put(YYINITIAL, 0);
}

private int currentStateIndex = 0;
private int parentStateIndex = 0;

public int getStateIndex() {
return currentStateIndex;
private final AbstractIntList states = new IntArrayList();

private void pushState(int newState) {
if (newState == YYINITIAL){
throw new IllegalStateException("Pusing YYINITIAL is not supported");
}
// store current state on the stack to allow restoring it in popState(...)
states.push(yystate());
yybegin(newState);
}

public void restoreState(int stateIndex) {
long state = states.getLong(stateIndex);
currentStateIndex = stateIndex;
parentStateIndex = (int) (state >> 32);
yybegin((int) state);
}

private void pushState(int yystate) {
long state = ((long) currentStateIndex << 32) | ((long) yystate & 0x0FFFFFFFFL);
int stateIndex = stateIndexMap.get(state); // Returns 0 if not found
if (stateIndex == 0 && state != YYINITIAL) {
stateIndex = states.size();
states.add(state);
stateIndexMap.put(state, stateIndex);
private void popState(int expectedState) {
if (states.isEmpty()){
throw new IllegalStateException("Popping an empty stack of states. Expected: " + expectedState);
}
// safe-guard, because we always know which state we're currently in in the rules below
if (yystate() != expectedState) {
throw new IllegalStateException(String.format("Unexpected state. Current: %d, expected: %d", yystate(), expectedState));
}
parentStateIndex = currentStateIndex;
currentStateIndex = stateIndex;
yybegin(yystate);
// start the lexer with the previous state, which was stored by pushState(...)
yybegin(states.popInt());
}

private void popState() {
restoreState(parentStateIndex);
protected void onReset() {
states.clear();
}
%}

Expand All @@ -57,7 +43,7 @@ import static org.nixos.idea.psi.NixTypes.*;
%function advance
%type IElementType
%unicode
%xstate STRING IND_STRING ANTIQUOTATION_START
%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION

ANY=[^]
ID=[a-zA-Z_][a-zA-Z0-9_'-]*
Expand All @@ -74,7 +60,38 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/

%%

<YYINITIAL> {
<STRING> {
[^\$\"\\]+ { return STR; }
"$"|"$$"|\\ { return STR; }
\\{ANY} { return STR_ESCAPE; }
"$"/"{" { pushState(ANTIQUOTATION_START); return DOLLAR; }
\" { popState(STRING); return STRING_CLOSE; }
}

<IND_STRING> {
[^\$\']+ { return IND_STR; }
"$"|"$$"|"'" { return IND_STR; }
"''$"|"'''" { return IND_STR_ESCAPE; }
"''"\\{ANY} { return IND_STR_ESCAPE; }
"$"/"{" { pushState(ANTIQUOTATION_START); return DOLLAR; }
"''" { popState(IND_STRING); return IND_STRING_CLOSE; }
}

<ANTIQUOTATION_START> {
// '$' and '{' must be two separate tokens to make NixBraceMatcher work
// correctly with Grammar-Kit.
"{" { popState(ANTIQUOTATION_START); pushState(ANTIQUOTATION); return LCURLY; }
}

<ANTIQUOTATION> {
"}" { popState(ANTIQUOTATION); return RCURLY; }
}

<BLOCK> {
"}" { popState(BLOCK); return RCURLY; }
}

<YYINITIAL, BLOCK, ANTIQUOTATION> {
"if" { return IF; }
"then" { return THEN; }
"else" { return ELSE; }
Expand All @@ -94,8 +111,8 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
"@" { return AT; }
"(" { return LPAREN; }
")" { return RPAREN; }
"{" { pushState(YYINITIAL); return LCURLY; }
"}" { popState(); return RCURLY; }
"{" { pushState(BLOCK); return LCURLY; }
"}" { return RCURLY; }
"[" { return LBRAC; }
"]" { return RBRAC; }
// '$' and '{' must be two separate tokens to make NixBraceMatcher work
Expand Down Expand Up @@ -137,28 +154,7 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
{SCOMMENT} { return SCOMMENT; }
{MCOMMENT} { return MCOMMENT; }
{WHITE_SPACE} { return com.intellij.psi.TokenType.WHITE_SPACE; }
[^] { return com.intellij.psi.TokenType.BAD_CHARACTER; }
}

<STRING> {
[^\$\"\\]+ { return STR; }
"$"|"$$"|\\ { return STR; }
\\{ANY} { return STR_ESCAPE; }
"$"/"{" { pushState(ANTIQUOTATION_START); return DOLLAR; }
\" { popState(); return STRING_CLOSE; }
}

<IND_STRING> {
[^\$\']+ { return IND_STR; }
"$"|"$$"|"'" { return IND_STR; }
"''$"|"'''" { return IND_STR_ESCAPE; }
"''"\\{ANY} { return IND_STR_ESCAPE; }
"$"/"{" { pushState(ANTIQUOTATION_START); return DOLLAR; }
"''" { popState(); return IND_STRING_CLOSE; }
}

<ANTIQUOTATION_START> {
// '$' and '{' must be two separate tokens to make NixBraceMatcher work
// correctly with Grammar-Kit.
"{" { popState(); pushState(YYINITIAL); return LCURLY; }
}
// matched by all %state states
[^] { return com.intellij.psi.TokenType.BAD_CHARACTER; }
2 changes: 1 addition & 1 deletion src/test/java/org/nixos/idea/lang/LexerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ public final class LexerTest extends LexerTestCase {
public void testRestartabilityWithAntiquotations() {
// Checks that the lexer is restartable. See
// https://intellij-support.jetbrains.com/hc/en-us/community/posts/360010305800
checkCorrectRestartOnEveryToken(
checkCorrectRestart(
"''\n" +
" ${\n" +
" [\n" +
Expand Down

0 comments on commit a924917

Please sign in to comment.