From 6e49e0cddb3f7b0e154064a37111bfac6a0dedac Mon Sep 17 00:00:00 2001 From: Eugene Palchukovsky Date: Fri, 23 Aug 2019 22:43:41 +0300 Subject: [PATCH 1/3] Adds block statement "set". Resolves #45. --- src/internal_value.cpp | 6 ++++ src/lexer.h | 3 +- src/statements.cpp | 53 ++++++++++++++++++++-------- src/statements.h | 65 ++++++++++++++++++++++++++++++---- src/template_parser.cpp | 67 +++++++++++++++++++++++++---------- test/errors_test.cpp | 4 +-- test/statements_tets.cpp | 75 +++++++++++++++++++++++++++++++++++++--- 7 files changed, 226 insertions(+), 47 deletions(-) diff --git a/src/internal_value.cpp b/src/internal_value.cpp index f8e6db19..4c184253 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -34,6 +34,12 @@ struct SubscriptionVisitor : public visitors::BaseVisitor<> return values.GetValueByName(field); } + template + InternalValue operator() (std::basic_string value, const std::basic_string& /*fieldName*/) const + { + return TargetString(std::move(value)); + } + InternalValue operator() (const ListAdapter& values, int64_t index) const { if (index < 0 || static_cast(index) >= values.GetSize()) diff --git a/src/lexer.h b/src/lexer.h index 78ea48b4..1b15623b 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -348,8 +348,7 @@ class LexScanner static const Token& EofToken() { - static Token eof; - eof.type = Token::Eof; + static const Token eof{Token::Eof}; return eof; } }; diff --git a/src/statements.cpp b/src/statements.cpp index 70126a19..5e0b58a6 100644 --- a/src/statements.cpp +++ b/src/statements.cpp @@ -211,19 +211,44 @@ void ElseBranchStatement::Render(OutStream& os, RenderContext& values) m_mainBody->Render(os, values); } -void SetStatement::Render(OutStream&, RenderContext& values) -{ - if (m_expr) - { - InternalValue val = m_expr->Evaluate(values); - if (m_fields.size() == 1) - values.GetCurrentScope()[m_fields[0]] = val; - else - { - for (auto& name : m_fields) - values.GetCurrentScope()[name] = Subscript(val, name, &values); - } - } +void SetStatement::AssingBody(InternalValue body, RenderContext& values) +{ + auto &scope = values.GetCurrentScope(); + if (m_fields.size() == 1) + scope[m_fields.front()] = body; + else + { + for (const auto& name : m_fields) + scope[name] = Subscript(body, name, &values); + } +} + +void SetLineStatement::Render(OutStream&, RenderContext& values) +{ + if (!m_expr) + return; + AssingBody(m_expr->Evaluate(values), values); +} + +InternalValue SetBlockStatement::RenderBody(RenderContext& values) +{ + TargetString result; + auto stream = values.GetRendererCallback()->GetStreamOnString(result); + auto innerValues = values.Clone(true); + m_body->Render(stream, innerValues); + return result; +} + +void SetRawBlockStatement::Render(OutStream&, RenderContext& values) +{ + AssingBody(RenderBody(values), values); +} + +void SetFilteredBlockStatement::Render(OutStream&, RenderContext& values) +{ + if (!m_expr) + return; + AssingBody(m_expr->Evaluate(RenderBody(values), values), values); } class BlocksRenderer : public RendererBase @@ -740,7 +765,7 @@ void FilterStatement::Render(OutStream& os, RenderContext& values) auto argStream = values.GetRendererCallback()->GetStreamOnString(arg); auto innerValues = values.Clone(true); m_body->Render(argStream, innerValues); - const auto result = m_expr->Evaluate(arg, values); + const auto result = m_expr->Evaluate(std::move(arg), values); os.WriteValue(result); } } // jinja2 diff --git a/src/statements.h b/src/statements.h index 7aeacd41..1e029fcb 100644 --- a/src/statements.h +++ b/src/statements.h @@ -124,22 +124,75 @@ class ElseBranchStatement : public Statement class SetStatement : public Statement { public: - VISITABLE_STATEMENT(); - SetStatement(std::vector fields) : m_fields(std::move(fields)) { } - void SetAssignmentExpr(ExpressionEvaluatorPtr<> expr) +protected: + void AssingBody(InternalValue, RenderContext&); + +private: + const std::vector m_fields; +}; + +class SetLineStatement final : public SetStatement +{ +public: + VISITABLE_STATEMENT(); + + SetLineStatement(std::vector fields, ExpressionEvaluatorPtr<> expr) + : SetStatement(std::move(fields)), m_expr(std::move(expr)) { - m_expr = std::move(expr); } + void Render(OutStream& os, RenderContext& values) override; private: - std::vector m_fields; - ExpressionEvaluatorPtr<> m_expr; + const ExpressionEvaluatorPtr<> m_expr; +}; + +class SetBlockStatement : public SetStatement +{ +public: + using SetStatement::SetStatement; + + void SetBody(RendererPtr renderer) + { + m_body = std::move(renderer); + } + +protected: + InternalValue RenderBody(RenderContext&); + +private: + RendererPtr m_body; +}; + +class SetRawBlockStatement final : public SetBlockStatement +{ +public: + VISITABLE_STATEMENT(); + + using SetBlockStatement::SetBlockStatement; + + void Render(OutStream&, RenderContext&) override; +}; + +class SetFilteredBlockStatement final : public SetBlockStatement +{ +public: + VISITABLE_STATEMENT(); + + explicit SetFilteredBlockStatement(std::vector fields, ExpressionEvaluatorPtr expr) + : SetBlockStatement(std::move(fields)), m_expr(std::move(expr)) + { + } + + void Render(OutStream&, RenderContext&) override; + +private: + const ExpressionEvaluatorPtr m_expr; }; class ParentBlockStatement : public Statement diff --git a/src/template_parser.cpp b/src/template_parser.cpp index eed5bd19..7ad31310 100644 --- a/src/template_parser.cpp +++ b/src/template_parser.cpp @@ -32,6 +32,9 @@ StatementsParser::ParseResult StatementsParser::Parse(LexScanner& lexer, Stateme case Keyword::Set: result = ParseSet(lexer, statementsInfo, tok); break; + case Keyword::EndSet: + result = ParseEndSet(lexer, statementsInfo, tok); + break; case Keyword::Block: result = ParseBlock(lexer, statementsInfo, tok); break; @@ -79,8 +82,6 @@ StatementsParser::ParseResult StatementsParser::Parse(LexScanner& lexer, Stateme case Keyword::EndFilter: result = ParseEndFilter(lexer, statementsInfo, tok); break; - case Keyword::EndSet: - return MakeParseError(ErrorCode::YetUnsupported, tok); default: return MakeParseError(ErrorCode::UnexpectedToken, tok); } @@ -320,30 +321,60 @@ StatementsParser::ParseResult StatementsParser::ParseSet(LexScanner& lexer, Stat if (vars.empty()) return MakeParseError(ErrorCode::ExpectedIdentifier, lexer.PeekNextToken()); - auto operTok = lexer.NextToken(); - ExpressionEvaluatorPtr<> valueExpr; - if (operTok == '=') + ExpressionParser exprParser(m_settings); + if (lexer.EatIfEqual('=')) { - ExpressionParser exprParser(m_settings); - auto expr = exprParser.ParseFullExpression(lexer); + const auto expr = exprParser.ParseFullExpression(lexer); if (!expr) return expr.get_unexpected(); - valueExpr = *expr; + statementsInfo.back().currentComposition->AddRenderer( + std::make_shared(std::move(vars), *expr)); + } + else if (lexer.EatIfEqual('|')) + { + const auto expr = exprParser.ParseFilterExpression(lexer); + if (!expr) + return expr.get_unexpected(); + auto statementInfo = StatementInfo::Create( + StatementInfo::SetStatement, stmtTok); + statementInfo.renderer = std::make_shared( + std::move(vars), *expr); + statementsInfo.push_back(std::move(statementInfo)); } else - return MakeParseError(ErrorCode::YetUnsupported, operTok, {stmtTok}); // TODO: Add handling of the block assignments - - auto renderer = std::make_shared(vars); - renderer->SetAssignmentExpr(valueExpr); - statementsInfo.back().currentComposition->AddRenderer(renderer); + { + auto operTok = lexer.NextToken(); + if (lexer.NextToken() != Token::Eof) + return MakeParseError(ErrorCode::YetUnsupported, operTok, {std::move(stmtTok)}); + auto statementInfo = StatementInfo::Create( + StatementInfo::SetStatement, stmtTok); + statementInfo.renderer = std::make_shared( + std::move(vars)); + statementsInfo.push_back(std::move(statementInfo)); + } - return ParseResult(); + return {}; } -StatementsParser::ParseResult StatementsParser::ParseEndSet(LexScanner& /*lexer*/, StatementInfoList& /*statementsInfo*/ +StatementsParser::ParseResult StatementsParser::ParseEndSet(LexScanner& + , StatementInfoList& statementsInfo , const Token& stmtTok) { - return MakeParseError(ErrorCode::YetUnsupported, stmtTok); + if (statementsInfo.size() <= 1) + return MakeParseError(ErrorCode::UnexpectedStatement, stmtTok); + + const auto info = statementsInfo.back(); + if (info.type != StatementInfo::SetStatement) + return MakeParseError(ErrorCode::UnexpectedStatement, stmtTok); + + auto &renderer = *boost::polymorphic_downcast( + info.renderer.get()); + renderer.SetBody(info.compositions[0]); + + statementsInfo.pop_back(); + statementsInfo.back().currentComposition->AddRenderer(info.renderer); + + return {}; } StatementsParser::ParseResult StatementsParser::ParseBlock(LexScanner& lexer, StatementInfoList& statementsInfo @@ -915,10 +946,10 @@ StatementsParser::ParseResult StatementsParser::ParseFilter(LexScanner& lexer, S StatementsParser::ParseResult StatementsParser::ParseEndFilter(LexScanner&, StatementInfoList& statementsInfo, const Token& stmtTok) { - if (statementsInfo.size() <= 1) + if (statementsInfo.size() <= 1) return MakeParseError(ErrorCode::UnexpectedStatement, stmtTok); - auto info = statementsInfo.back(); + const auto info = statementsInfo.back(); if (info.type != StatementInfo::FilterStatement) { return MakeParseError(ErrorCode::UnexpectedStatement, stmtTok); diff --git a/test/errors_test.cpp b/test/errors_test.cpp index 4faabc76..513832e0 100644 --- a/test/errors_test.cpp +++ b/test/errors_test.cpp @@ -380,14 +380,12 @@ INSTANTIATE_TEST_CASE_P(StatementsTest_1, ErrorsGenericTest, ::testing::Values( "noname.j2tpl:2:4: error: Unexpected statement: 'endfor'\n{% endfor %}\n---^-------"}, InputOutputPair{"{% set %}", "noname.j2tpl:1:8: error: Identifier expected\n{% set %}\n ---^-------"}, - InputOutputPair{"{% set id%}", - "noname.j2tpl:1:10: error: This feature has not been supported yet\n{% set id%}\n ---^-------"}, InputOutputPair{"{% set 10%}", "noname.j2tpl:1:8: error: Identifier expected\n{% set 10%}\n ---^-------"}, InputOutputPair{"{% set i = {key] %}", "noname.j2tpl:1:13: error: String expected\n{% set i = {key] %}\n ---^-------"}, InputOutputPair{"{% set id=10%}\n{% endset %}", - "noname.j2tpl:2:4: error: This feature has not been supported yet\n{% endset %}\n---^-------"}, + "noname.j2tpl:2:4: error: Unexpected statement: 'endset'\n{% endset %}\n---^-------"}, InputOutputPair{"{% extends %}", "noname.j2tpl:1:12: error: Unexpected token '<>'. Expected: '<>', '<>'\n{% extends %}\n ---^-------"}, InputOutputPair{"{% extends 10 %}", diff --git a/test/statements_tets.cpp b/test/statements_tets.cpp index f78a4815..f5e14ce6 100644 --- a/test/statements_tets.cpp +++ b/test/statements_tets.cpp @@ -202,14 +202,13 @@ TEST(FilterStatement, General) ASSERT_TRUE(tpl.Load(source)); const auto result = tpl.RenderAsString({}).value(); - std::cout << result << std::endl; EXPECT_STREQ("\n THIS TEXT BECOMES UPPERCASE\n", result.c_str()); } TEST(FilterStatement, ChainAndParams) { const std::string source = R"( -{% filter list | sort(reverse=true) | unique | join("+") %} +{% filter trim | list | sort(reverse=true) | unique | join("+") %} 11222333445556677890 {% endfilter %} )"; @@ -218,6 +217,74 @@ TEST(FilterStatement, ChainAndParams) ASSERT_TRUE(tpl.Load(source)); const auto result = tpl.RenderAsString({}).value(); - std::cout << result << std::endl; - EXPECT_STREQ("\n9+8+7+6+5+4+3+2+1+0+\n", result.c_str()); + EXPECT_STREQ("\n9+8+7+6+5+4+3+2+1+0", result.c_str()); } + +TEST(SetBlockStatement, OneVar) +{ + const std::string source = R"( +{% set foo %} +11222333445556677890 +{% endset %} +|{{foo}}| +)"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + + const auto result = tpl.RenderAsString({}).value(); + EXPECT_STREQ("\n|11222333445556677890\n|\n", result.c_str()); +} + +TEST(SetBlockStatement, MoreVars) +{ + const std::string source = R"( +{% set foo1,foo2,foo3,foo4,foo5 %} +11222333445556677890 +{% endset %} +|{{foo1}}| +|{{foo2}}| +|{{foo5}}| +)"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + + const auto result = tpl.RenderAsString({}).value(); + EXPECT_STREQ("\n|11222333445556677890\n|\n|11222333445556677890\n|\n|11222333445556677890\n|\n", result.c_str()); +} + +TEST(SetBlockStatement, OneVarFiltered) +{ + const std::string source = R"( +{% set foo | trim | list | sort(reverse=true) | unique | join("+") %} +11222333445556677890 +{% endset %} +|{{foo}}| +)"; + + Template tpl; + const auto load = tpl.Load(source); + ASSERT_TRUE(load) << load.error(); + + const auto result = tpl.RenderAsString({}).value(); + EXPECT_STREQ("\n|9+8+7+6+5+4+3+2+1+0|\n", result.c_str()); +} + +TEST(SetBlockStatement, MoreVarsFiltered) +{ + const std::string source = R"( +{% set foo1,foo2,foo3,foo4,foo5 | trim | list | sort(reverse=true) | unique | join("+") %} +11222333445556677890 +{% endset %} +|{{foo1}}| +|{{foo2}}| +|{{foo5}}| +)"; + + Template tpl; + ASSERT_TRUE(tpl.Load(source)); + + const auto result = tpl.RenderAsString({}).value(); + EXPECT_STREQ("\n|9+8+7+6+5+4+3+2+1+0|\n|9+8+7+6+5+4+3+2+1+0|\n|9+8+7+6+5+4+3+2+1+0|\n", result.c_str()); +} \ No newline at end of file From 969e0c0dcc881348cccf877cc6c4ed7b6fb4dac5 Mon Sep 17 00:00:00 2001 From: Eugene Palchukovsky Date: Sat, 24 Aug 2019 08:32:48 +0300 Subject: [PATCH 2/3] Fixes minor typos. --- src/statements.cpp | 10 +++++----- src/statements.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/statements.cpp b/src/statements.cpp index 5e0b58a6..aa595cae 100644 --- a/src/statements.cpp +++ b/src/statements.cpp @@ -211,11 +211,11 @@ void ElseBranchStatement::Render(OutStream& os, RenderContext& values) m_mainBody->Render(os, values); } -void SetStatement::AssingBody(InternalValue body, RenderContext& values) +void SetStatement::AssignBody(InternalValue body, RenderContext& values) { auto &scope = values.GetCurrentScope(); if (m_fields.size() == 1) - scope[m_fields.front()] = body; + scope[m_fields.front()] = std::move(body); else { for (const auto& name : m_fields) @@ -227,7 +227,7 @@ void SetLineStatement::Render(OutStream&, RenderContext& values) { if (!m_expr) return; - AssingBody(m_expr->Evaluate(values), values); + AssignBody(m_expr->Evaluate(values), values); } InternalValue SetBlockStatement::RenderBody(RenderContext& values) @@ -241,14 +241,14 @@ InternalValue SetBlockStatement::RenderBody(RenderContext& values) void SetRawBlockStatement::Render(OutStream&, RenderContext& values) { - AssingBody(RenderBody(values), values); + AssignBody(RenderBody(values), values); } void SetFilteredBlockStatement::Render(OutStream&, RenderContext& values) { if (!m_expr) return; - AssingBody(m_expr->Evaluate(RenderBody(values), values), values); + AssignBody(m_expr->Evaluate(RenderBody(values), values), values); } class BlocksRenderer : public RendererBase diff --git a/src/statements.h b/src/statements.h index 1e029fcb..eaeba8c9 100644 --- a/src/statements.h +++ b/src/statements.h @@ -130,7 +130,7 @@ class SetStatement : public Statement } protected: - void AssingBody(InternalValue, RenderContext&); + void AssignBody(InternalValue, RenderContext&); private: const std::vector m_fields; From b0a5d81ee02161d56989a639cef4ae95bb6c9030 Mon Sep 17 00:00:00 2001 From: Eugene Palchukovsky Date: Sat, 24 Aug 2019 22:22:53 +0300 Subject: [PATCH 3/3] Fixes compiling error with VS 2015. --- src/lexer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lexer.h b/src/lexer.h index 1b15623b..78ea48b4 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -348,7 +348,8 @@ class LexScanner static const Token& EofToken() { - static const Token eof{Token::Eof}; + static Token eof; + eof.type = Token::Eof; return eof; } };