From 98031f950d495754766ba866f2939a01830f2cdd Mon Sep 17 00:00:00 2001 From: Flex Ferrum Date: Tue, 28 May 2019 11:09:13 +0300 Subject: [PATCH 1/2] Implement 'do' extension statement --- include/jinja2cpp/error_info.h | 1 + include/jinja2cpp/template_env.h | 13 +++++++++++ src/error_info.cpp | 3 +++ src/expression_parser.cpp | 3 ++- src/expression_parser.h | 3 ++- src/lexer.h | 2 ++ src/statements.cpp | 4 ++++ src/statements.h | 13 +++++++++++ src/template_parser.cpp | 38 ++++++++++++++++++++++++-------- src/template_parser.h | 16 +++++++++++--- test/expressions_test.cpp | 36 ++++++++++++++++++++++++++++++ 11 files changed, 118 insertions(+), 14 deletions(-) diff --git a/include/jinja2cpp/error_info.h b/include/jinja2cpp/error_info.h index 64d82576..16bfcc2f 100644 --- a/include/jinja2cpp/error_info.h +++ b/include/jinja2cpp/error_info.h @@ -34,6 +34,7 @@ enum class ErrorCode TemplateNotFound, TemplateNotParsed, InvalidValueType, + ExtensionDisabled, }; struct SourceLocation diff --git a/include/jinja2cpp/template_env.h b/include/jinja2cpp/template_env.h index c72267e8..3b94f2b4 100644 --- a/include/jinja2cpp/template_env.h +++ b/include/jinja2cpp/template_env.h @@ -14,11 +14,24 @@ namespace jinja2 class IErrorHandler; class IFilesystemHandler; +enum class Jinja2CompatMode +{ + None, + Vesrsion_2_10, +}; + struct Settings { + struct Extensions + { + bool Do = false; + }; + bool useLineStatements = false; bool trimBlocks = false; bool lstripBlocks = false; + Extensions extensions; + Jinja2CompatMode jinja2CompatMode = Jinja2CompatMode::None; }; class TemplateEnv diff --git a/src/error_info.cpp b/src/error_info.cpp index 31f19198..bc82c438 100644 --- a/src/error_info.cpp +++ b/src/error_info.cpp @@ -200,6 +200,9 @@ void RenderErrorInfo(std::basic_ostream& os, const ErrorInfoTpl& e case ErrorCode::InvalidValueType: os << UNIVERSAL_STR("Invalid value type"); break; + case ErrorCode::ExtensionDisabled: + os << UNIVERSAL_STR("Extension disabled"); + break; } os << std::endl << errInfo.GetLocationDescr(); } diff --git a/src/expression_parser.cpp b/src/expression_parser.cpp index 6aec9651..1c198541 100644 --- a/src/expression_parser.cpp +++ b/src/expression_parser.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace jinja2 { @@ -16,7 +17,7 @@ auto ReplaceErrorIfPossible(T& result, const Token& pivotTok, ErrorCode newError return result.get_unexpected(); } -ExpressionParser::ExpressionParser() +ExpressionParser::ExpressionParser(const Settings& /* settings */) { } diff --git a/src/expression_parser.h b/src/expression_parser.h index 9ba0d909..43efd971 100644 --- a/src/expression_parser.h +++ b/src/expression_parser.h @@ -7,6 +7,7 @@ #include "renderer.h" #include +#include namespace jinja2 { @@ -16,7 +17,7 @@ class ExpressionParser template using ParseResult = nonstd::expected; - ExpressionParser(); + ExpressionParser(const Settings& settings); ParseResult Parse(LexScanner& lexer); ParseResult> ParseFullExpression(LexScanner& lexer, bool includeIfPart = true); ParseResult ParseCallParams(LexScanner& lexer); diff --git a/src/lexer.h b/src/lexer.h index 8cd6dfea..f16f4c63 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -92,6 +92,7 @@ struct Token Context, From, As, + Do, // Template control CommentBegin, @@ -169,6 +170,7 @@ enum class Keyword Context, From, As, + Do, }; struct LexerHelper diff --git a/src/statements.cpp b/src/statements.cpp index 809f2970..b5a081c2 100644 --- a/src/statements.cpp +++ b/src/statements.cpp @@ -635,4 +635,8 @@ void MacroCallStatement::SetupMacroScope(InternalValueMap&) } +void DoStatement::Render(OutStream& os, RenderContext& values) +{ + m_expr->Evaluate(values); +} } // jinja2 diff --git a/src/statements.h b/src/statements.h index b341fb83..bb7a3c85 100644 --- a/src/statements.h +++ b/src/statements.h @@ -322,6 +322,19 @@ class MacroCallStatement : public MacroStatement std::string m_macroName; CallParams m_callParams; }; + +class DoStatement : public Statement +{ +public: + VISITABLE_STATEMENT(); + + DoStatement(ExpressionEvaluatorPtr <> expr) : m_expr(expr) {} + + void Render(OutStream &os, RenderContext &values) override; + +private: + ExpressionEvaluatorPtr<> m_expr; +}; } // jinja2 diff --git a/src/template_parser.cpp b/src/template_parser.cpp index 0b473a6e..6dc95bed 100644 --- a/src/template_parser.cpp +++ b/src/template_parser.cpp @@ -62,6 +62,11 @@ StatementsParser::ParseResult StatementsParser::Parse(LexScanner& lexer, Stateme case Keyword::From: result = ParseFrom(lexer, statementsInfo, tok); break; + case Keyword::Do: + if (!m_settings.extensions.Do) + return MakeParseError(ErrorCode::ExtensionDisabled, tok); + result = ParseDo(lexer, statementsInfo, tok); + break; case Keyword::Filter: case Keyword::EndFilter: case Keyword::EndSet: @@ -142,7 +147,7 @@ StatementsParser::ParseResult StatementsParser::ParseFor(LexScanner &lexer, Stat } auto pivotToken = lexer.PeekNextToken(); - ExpressionParser exprPraser; + ExpressionParser exprPraser(m_settings); auto valueExpr = exprPraser.ParseFullExpression(lexer, false); if (!valueExpr) return valueExpr.get_unexpected(); @@ -213,7 +218,7 @@ StatementsParser::ParseResult StatementsParser::ParseIf(LexScanner &lexer, State const Token &stmtTok) { auto pivotTok = lexer.PeekNextToken(); - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto valueExpr = exprParser.ParseFullExpression(lexer); if (!valueExpr) return MakeParseError(ErrorCode::ExpectedExpression, pivotTok); @@ -239,7 +244,7 @@ StatementsParser::ParseResult StatementsParser::ParseElIf(LexScanner& lexer, Sta , const Token& stmtTok) { auto pivotTok = lexer.PeekNextToken(); - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto valueExpr = exprParser.ParseFullExpression(lexer); if (!valueExpr) return MakeParseError(ErrorCode::ExpectedExpression, pivotTok); @@ -309,7 +314,7 @@ StatementsParser::ParseResult StatementsParser::ParseSet(LexScanner& lexer, Stat ExpressionEvaluatorPtr<> valueExpr; if (operTok == '=') { - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto expr = exprParser.ParseFullExpression(lexer); if (!expr) return expr.get_unexpected(); @@ -478,7 +483,7 @@ nonstd::expected StatementsParser::ParseMacroParams(Lex if (lexer.EatIfEqual(')')) return std::move(items); - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); do { @@ -562,7 +567,7 @@ StatementsParser::ParseResult StatementsParser::ParseCall(LexScanner& lexer, Sta CallParams callParams; if (lexer.EatIfEqual('(')) { - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto result = exprParser.ParseCallParams(lexer); if (!result) return result.get_unexpected(); @@ -606,7 +611,7 @@ StatementsParser::ParseResult StatementsParser::ParseInclude(LexScanner& lexer, // auto operTok = lexer.NextToken(); ExpressionEvaluatorPtr<> valueExpr; - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto expr = exprParser.ParseFullExpression(lexer); if (!expr) return expr.get_unexpected(); @@ -662,7 +667,7 @@ StatementsParser::ParseResult StatementsParser::ParseInclude(LexScanner& lexer, StatementsParser::ParseResult StatementsParser::ParseImport(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok) { ExpressionEvaluatorPtr<> valueExpr; - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto expr = exprParser.ParseFullExpression(lexer); if (!expr) return expr.get_unexpected(); @@ -709,7 +714,7 @@ StatementsParser::ParseResult StatementsParser::ParseImport(LexScanner& lexer, S StatementsParser::ParseResult StatementsParser::ParseFrom(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok) { ExpressionEvaluatorPtr<> valueExpr; - ExpressionParser exprParser; + ExpressionParser exprParser(m_settings); auto expr = exprParser.ParseFullExpression(lexer); if (!expr) return expr.get_unexpected(); @@ -795,4 +800,19 @@ StatementsParser::ParseResult StatementsParser::ParseFrom(LexScanner& lexer, Sta return ParseResult(); } +StatementsParser::ParseResult StatementsParser::ParseDo(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok) +{ + ExpressionEvaluatorPtr<> valueExpr; + ExpressionParser exprParser(m_settings); + auto expr = exprParser.ParseFullExpression(lexer); + if (!expr) + return expr.get_unexpected(); + valueExpr = *expr; + + auto renderer = std::make_shared(valueExpr); + statementsInfo.back().currentComposition->AddRenderer(renderer); + + return jinja2::StatementsParser::ParseResult(); +} + } diff --git a/src/template_parser.h b/src/template_parser.h index e2974efa..9a5e3552 100644 --- a/src/template_parser.h +++ b/src/template_parser.h @@ -49,7 +49,7 @@ template struct ParserTraitsBase { static Token::Type s_keywords[]; - static KeywordsInfo s_keywordsInfo[39]; + static KeywordsInfo s_keywordsInfo[40]; static std::unordered_map s_tokens; }; @@ -194,6 +194,10 @@ class StatementsParser public: using ParseResult = nonstd::expected; + StatementsParser(const Settings& settings) + : m_settings(settings) + {} + ParseResult Parse(LexScanner& lexer, StatementInfoList& statementsInfo); private: @@ -216,6 +220,10 @@ class StatementsParser ParseResult ParseInclude(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok); ParseResult ParseImport(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok); ParseResult ParseFrom(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok); + ParseResult ParseDo(LexScanner& lexer, StatementInfoList& statementsInfo, const Token& stmtTok); + +private: + Settings m_settings; }; template @@ -532,7 +540,7 @@ class TemplateParser : public LexerHelper if (!lexer.Preprocess()) return MakeParseError(ErrorCode::Unspecified, MakeToken(Token::Unknown, {range.startOffset, range.startOffset + 1})); - P praser; + P praser(m_settings); LexScanner scanner(lexer); auto result = praser.Parse(scanner, std::forward(args)...); if (!result) @@ -773,7 +781,7 @@ class TemplateParser : public LexerHelper }; template -KeywordsInfo ParserTraitsBase::s_keywordsInfo[39] = { +KeywordsInfo ParserTraitsBase::s_keywordsInfo[40] = { {UNIVERSAL_STR("for"), Keyword::For}, {UNIVERSAL_STR("endfor"), Keyword::Endfor}, {UNIVERSAL_STR("in"), Keyword::In}, @@ -813,6 +821,7 @@ KeywordsInfo ParserTraitsBase::s_keywordsInfo[39] = { {UNIVERSAL_STR("context"), Keyword::Context}, {UNIVERSAL_STR("from"), Keyword::From}, {UNIVERSAL_STR("as"), Keyword::As}, + {UNIVERSAL_STR("do"), Keyword::Do}, }; template @@ -878,6 +887,7 @@ std::unordered_map ParserTraitsBase::s_tokens = { {Token::Context, UNIVERSAL_STR("context")}, {Token::From, UNIVERSAL_STR("form")}, {Token::As, UNIVERSAL_STR("as")}, + {Token::Do, UNIVERSAL_STR("do")}, {Token::CommentBegin, UNIVERSAL_STR("{#")}, {Token::CommentEnd, UNIVERSAL_STR("#}")}, {Token::StmtBegin, UNIVERSAL_STR("{%")}, diff --git a/test/expressions_test.cpp b/test/expressions_test.cpp index 31a58cd9..d6b6d98d 100644 --- a/test/expressions_test.cpp +++ b/test/expressions_test.cpp @@ -86,6 +86,42 @@ TEST(ExpressionsTest, IfExpression) EXPECT_STREQ(expectedResult.c_str(), result.c_str()); } +TEST(ExpressionTest, DoStatement) +{ + std::string source = R"( +{{ data.strValue }}{% do setData('Inner Value') %} +{{ data.strValue }} +)"; + + TemplateEnv env; + env.GetSettings().extensions.Do = true; + + TestInnerStruct innerStruct; + innerStruct.strValue = "Outer Value"; + + ValuesMap params = { + {"data", Reflect(&innerStruct)}, + {"setData", MakeCallable( + [&innerStruct](const std::string& val) -> Value { + innerStruct.strValue = val; + return "String not to be shown"; + }, + ArgInfo{"val"}) + }, + }; + + Template tpl(&env); + + ASSERT_TRUE(tpl.Load(source)); + std::string result = tpl.RenderAsString(params).value(); + std::cout << result << std::endl; + std::string expectedResult = R"( +Outer ValueInner Value +)"; + + EXPECT_STREQ(expectedResult.c_str(), result.c_str()); +} + struct LogicalExprTestTag; using LogicalExprTest = InputOutputPairTest; From 0a318c177e261954354c100ff2b2b7e832c8d9c0 Mon Sep 17 00:00:00 2001 From: Flex Ferrum Date: Tue, 28 May 2019 11:59:12 +0300 Subject: [PATCH 2/2] Add parsing errors test for 'do' statement --- test/errors_test.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/errors_test.cpp b/test/errors_test.cpp index a71d3f73..c073beb9 100644 --- a/test/errors_test.cpp +++ b/test/errors_test.cpp @@ -7,7 +7,9 @@ using namespace jinja2; struct ErrorsGenericTestTag; +struct ErrorsGenericExtensionTestTag; using ErrorsGenericTest = InputOutputPairTest; +using ErrorsGenericExtensionsTest = InputOutputPairTest; TEST_P(ErrorsGenericTest, Test) @@ -27,6 +29,26 @@ TEST_P(ErrorsGenericTest, Test) EXPECT_EQ(expectedResult, result); } +TEST_P(ErrorsGenericExtensionsTest, Test) +{ + auto& testParam = GetParam(); + std::string source = testParam.tpl; + + TemplateEnv env; + env.GetSettings().extensions.Do = true; + + Template tpl(&env); + auto parseResult = tpl.Load(source); + EXPECT_FALSE(parseResult.has_value()); + + std::ostringstream errorDescr; + errorDescr << parseResult.error(); + std::string result = errorDescr.str(); + std::cout << result << std::endl; + std::string expectedResult = testParam.result; + EXPECT_EQ(expectedResult, result); +} + INSTANTIATE_TEST_CASE_P(BasicTest, ErrorsGenericTest, ::testing::Values( InputOutputPair{"{{}}", "noname.j2tpl:1:3: error: Unexpected token: '<>'\n{{}}\n--^-------"}, @@ -236,6 +258,15 @@ INSTANTIATE_TEST_CASE_P(StatementsTest_2, ErrorsGenericTest, ::testing::Values( "noname.j2tpl:1:20: error: Unexpected token: '*'\n{% call name(param=*) %}{% endcall %}\n ---^-------"}, InputOutputPair{"{% block b %}{% endcall %}", "noname.j2tpl:1:17: error: Unexpected statement: 'endcall'\n{% block b %}{% endcall %}\n ---^-------"}, + InputOutputPair{"{% do 'Hello World' %}", + "noname.j2tpl:1:4: error: Extension disabled\n{% do 'Hello World' %}\n---^-------"}, InputOutputPair{"{{}}", "noname.j2tpl:1:3: error: Unexpected token: '<>'\n{{}}\n--^-------"} )); + +INSTANTIATE_TEST_CASE_P(ExtensionStatementsTest, ErrorsGenericExtensionsTest, ::testing::Values( + InputOutputPair{"{% do %}", + "noname.j2tpl:1:7: error: Unexpected token: '<>'\n{% do %}\n ---^-------"}, + InputOutputPair{"{% do 1 + %}", + "noname.j2tpl:1:11: error: Unexpected token: '<>'\n{% do 1 + %}\n ---^-------"} + ));