Skip to content
Permalink
Browse files

ExpressionParser: Add support for /* */ style comments.

  • Loading branch information...
jordan-woyak committed Oct 12, 2019
1 parent b4e2b3c commit 72302d9c4224271a606f2782dec7dc725658b150
@@ -44,14 +44,6 @@ QTextCharFormat GetSpecialCharFormat()
return format;
}

QTextCharFormat GetOperatorCharFormat()
{
QTextCharFormat format;
format.setFontWeight(QFont::Weight::Bold);
format.setForeground(QBrush{Qt::darkBlue});
return format;
}

QTextCharFormat GetLiteralCharFormat()
{
QTextCharFormat format;
@@ -77,14 +69,21 @@ QTextCharFormat GetControlCharFormat()
QTextCharFormat GetVariableCharFormat()
{
QTextCharFormat format;
format.setForeground(QBrush{Qt::magenta});
format.setForeground(QBrush{Qt::darkYellow});
return format;
}

QTextCharFormat GetBarewordCharFormat()
{
QTextCharFormat format;
format.setForeground(QBrush{Qt::darkCyan});
format.setForeground(QBrush{Qt::darkBlue});
return format;
}

QTextCharFormat GetCommentCharFormat()
{
QTextCharFormat format;
format.setForeground(QBrush{Qt::darkGray});
return format;
}
} // namespace
@@ -95,16 +94,35 @@ ControlExpressionSyntaxHighlighter::ControlExpressionSyntaxHighlighter(QTextDocu
{
}

void ControlExpressionSyntaxHighlighter::highlightBlock(const QString& text)
void ControlExpressionSyntaxHighlighter::highlightBlock(const QString&)
{
// TODO: This is going to result in improper highlighting with non-ascii characters:
ciface::ExpressionParser::Lexer lexer(text.toStdString());
ciface::ExpressionParser::Lexer lexer(document()->toPlainText().toStdString());

std::vector<ciface::ExpressionParser::Token> tokens;
const auto tokenize_status = lexer.Tokenize(tokens);

using ciface::ExpressionParser::TokenType;

const auto set_block_format = [this](int start, int count, const QTextCharFormat& format) {
if (start + count <= currentBlock().position() ||
start >= currentBlock().position() + currentBlock().length())
{
// This range is not within the current block.
return;
}

int block_start = start - currentBlock().position();

if (block_start < 0)
{
count += block_start;
block_start = 0;
}

setFormat(block_start, count, format);
};

for (auto& token : tokens)
{
std::optional<QTextCharFormat> char_format;
@@ -131,22 +149,27 @@ void ControlExpressionSyntaxHighlighter::highlightBlock(const QString& text)
case TokenType::TOK_VARIABLE:
char_format = GetVariableCharFormat();
break;
case TokenType::TOK_COMMENT:
char_format = GetCommentCharFormat();
break;
default:
if (token.IsBinaryOperator())
char_format = GetOperatorCharFormat();
char_format = GetSpecialCharFormat();
break;
}

if (char_format.has_value())
setFormat(int(token.string_position), int(token.string_length), *char_format);
set_block_format(int(token.string_position), int(token.string_length), *char_format);
}

// This doesn't need to be run for every "block", but it works.
if (ciface::ExpressionParser::ParseStatus::Successful != tokenize_status)
{
m_result_text->setText(tr("Invalid Token."));
}
else
{
ciface::ExpressionParser::RemoveInertTokens(&tokens);
const auto parse_status = ciface::ExpressionParser::ParseTokens(tokens);

m_result_text->setText(
@@ -155,7 +178,8 @@ void ControlExpressionSyntaxHighlighter::highlightBlock(const QString& text)
if (ciface::ExpressionParser::ParseStatus::Successful != parse_status.status)
{
const auto token = *parse_status.token;
setFormat(int(token.string_position), int(token.string_length), GetInvalidCharFormat());
set_block_format(int(token.string_position), int(token.string_length),
GetInvalidCharFormat());
}
}
}
@@ -87,6 +87,14 @@ Token Lexer::GetRealLiteral(char first_char)
return Token(TOK_INVALID);
}

Token Lexer::PeekToken()
{
const auto old_it = it;
const auto tok = NextToken();
it = old_it;
return tok;
}

Token Lexer::NextToken()
{
if (it == expr.end())
@@ -99,7 +107,7 @@ Token Lexer::NextToken()
case '\t':
case '\n':
case '\r':
return Token(TOK_DISCARD);
return Token(TOK_WHITESPACE);
case '(':
return Token(TOK_LPAREN);
case ')':
@@ -154,8 +162,19 @@ ParseStatus Lexer::Tokenize(std::vector<Token>& tokens)
tok.string_position = string_position;
tok.string_length = it - expr.begin();

if (tok.type == TOK_DISCARD)
continue;
// Handle /* */ style comments.
if (tok.type == TOK_DIV && PeekToken().type == TOK_MUL)
{
const auto end_of_comment = expr.find("*/", it - expr.begin());

if (end_of_comment == std::string::npos)
return ParseStatus::SyntaxError;

tok.type = TOK_COMMENT;
tok.string_length = end_of_comment + 4;

it = expr.begin() + end_of_comment + 2;
}

tokens.push_back(tok);

@@ -671,9 +690,19 @@ static ParseResult ParseComplexExpression(const std::string& str)
if (tokenize_status != ParseStatus::Successful)
return ParseResult::MakeErrorResult(Token(TOK_INVALID), _trans("Tokenizing failed."));

RemoveInertTokens(&tokens);
return ParseTokens(tokens);
}

void RemoveInertTokens(std::vector<Token>* tokens)
{
tokens->erase(std::remove_if(tokens->begin(), tokens->end(),
[](const Token& tok) {
return tok.type == TOK_COMMENT || tok.type == TOK_WHITESPACE;
}),
tokens->end());
}

static std::unique_ptr<Expression> ParseBarewordExpression(const std::string& str)
{
ControlQualifier qualifier;
@@ -15,7 +15,7 @@ namespace ciface::ExpressionParser
{
enum TokenType
{
TOK_DISCARD,
TOK_WHITESPACE,
TOK_INVALID,
TOK_EOF,
TOK_LPAREN,
@@ -25,6 +25,7 @@ enum TokenType
TOK_LITERAL,
TOK_VARIABLE,
TOK_BAREWORD,
TOK_COMMENT,
// Binary Ops:
TOK_BINARY_OPS_BEGIN,
TOK_AND = TOK_BINARY_OPS_BEGIN,
@@ -95,6 +96,7 @@ class Lexer
Token GetBareword(char c);
Token GetRealLiteral(char c);

Token PeekToken();
Token NextToken();
};

@@ -186,5 +188,6 @@ class ParseResult

ParseResult ParseExpression(const std::string& expr);
ParseResult ParseTokens(const std::vector<Token>& tokens);
void RemoveInertTokens(std::vector<Token>* tokens);

} // namespace ciface::ExpressionParser

0 comments on commit 72302d9

Please sign in to comment.
You can’t perform that action at this time.