Make the most of the keywords context-dependent #73 (#88)

Author: flexferrum

README.md

@@ -28,6 +28,7 @@ C++ implementation of big subset of Jinja2 template engine features. This librar
     - ['set' statement](#set-statement)
     - ['extends' statement](#extends-statement)
     - [Macros](#macros)
+    - [User-defined callables](#user-defined-callables)
     - [Error reporting](#error-reporting)
   - [Other features](#other-features)
 - [Current Jinja2 support](#current-jinja2-support)
@@ -437,6 +438,40 @@ inline {{ resultType }} {{ methodName }}({{ methodParams | join(', ') }} )
 
 Here is an `InlineMacro` which just describe the inline method definition skeleton. This macro doesn't contain the actual method body. Instead of this it calls `caller` special function. This function invokes the special **callback** macro which is a body of `call` statement. And this macro can have parameters as well. More detailed this mechanics described in the [Jinja2 documentation](http://jinja.pocoo.org/docs/2.10/templates/#macros).
 
+### 'applymacro' filter
+
+With help of `applymacro` filter macro can be called in filtering context. `applymacro` works similar to `map` (or `test`) filter with one exception: instead of name of other filter it takes name of macro via `macro` param and pass the rest of arguments to it. The object which is been filtered is passed as the first positional argument. For example:
+
+```
+{% macro toUpper(str) %}{{ str | upper }}{% endmacro %}
+{{ 'Hello World!' | applymacro(macro='toUpper') }}
+```
+
+produces the result `HELLO WORLD`. `applymacro` can be applied to the sequences via `map` filter. Also, macro name can be `caller`. In this case outer `call` statement will be invoked during macro application.
+
+## User-defined callables
+
+Not only C++ types can be reflected into Jinja2 template context, but the functions (and lambdas, and any other callable objects) as well. These refelected callable objects are called 'user-defined callables' and can be accessed from Jinja2 templates in the same manner as any other callables (like macros or global functions). In order to reflect callable object into Jinja2 context the `jinja2::MakeCallable` method should be used:
+
+```c++
+jinja2::ValuesMap params;
+    params["concat"] = MakeCallable(
+                [](const std::string& str1, const std::string& str2) {
+                    return str1 + " " + str2;
+                },
+                ArgInfo{"str1"}, ArgInfo{"str2", false, "default"}
+    );
+```
+
+As a first parameter this method takes the callable itself. It can be lambda, the std::function<> instance or pointer to function. The rest of params are callable arguments descriptors, which are provided via `ArgInfo` structure. In the sample above user-defined callable `concat` is introduced, which take two argument: `str1` and `str2`. This callable can be accessed from the template in the following ways:
+
+```
+{{ concat('Hello', 'World!') }}
+{{ concat(str2='World!', str1='Hello') }}
+{{ concat(str2='World!') }}
+{{ concat('Hello') }}
+```
+
 ## Error reporting
 It's difficult to write complex template completely without errors. Missed braces, wrong characters, incorrect names... Everything is possible. So, it's crucial to be able to get informative error report from the template engine. Jinja2Cpp provides such kind of report. ```Template::Load``` method (and TemplateEnv::LoadTemplate respectively) return instance of ```ErrorInfo``` class which contains details about the error. These details include:
 - Error code

src/expression_parser.cpp

@@ -1,6 +1,7 @@
 #include "expression_parser.h"
 
 #include <sstream>
+#include <unordered_set>
 
 namespace jinja2
 {
@@ -59,7 +60,7 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<FullExpressionEvaluator>> E
         evaluator->SetFilter(*filter);
     }
 
-    if (includeIfPart && lexer.EatIfEqual(Token::If))
+    if (includeIfPart && lexer.EatIfEqual(Keyword::If))
     {
         if (includeIfPart)
         {
@@ -79,7 +80,7 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionPars
 {
     auto left = ParseLogicalAnd(lexer);
 
-    if (left && lexer.EatIfEqual(Token::LogicalOr))
+    if (left && lexer.EatIfEqual(Keyword::LogicalOr))
     {
         auto right = ParseLogicalOr(lexer);
         if (!right)
@@ -95,7 +96,7 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionPars
 {
     auto left = ParseLogicalCompare(lexer);
 
-    if (left && lexer.EatIfEqual(Token::LogicalAnd))
+    if (left && lexer.EatIfEqual(Keyword::LogicalAnd))
     {
         auto right = ParseLogicalAnd(lexer);
         if (!right)
@@ -135,29 +136,33 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionPars
     case Token::LessEqual:
         operation = BinaryExpression::LogicalLe;
         break;
-    case Token::In:
-        operation = BinaryExpression::In;
-        break;
-    case Token::Is:
-    {
-        Token nextTok = lexer.NextToken();
-        if (nextTok != Token::Identifier)
-            return MakeParseError(ErrorCode::ExpectedIdentifier, nextTok);
-
-        std::string name = AsString(nextTok.value);
-        ParseResult<CallParams> params;
-
-        if (lexer.EatIfEqual('('))
-            params = ParseCallParams(lexer);
-
-        if (!params)
-            return params.get_unexpected();
-
-        return std::make_shared<IsExpression>(*left, std::move(name), std::move(*params));
-    }
     default:
-        lexer.ReturnToken();
-        return left;
+        switch (lexer.GetAsKeyword(tok))
+        {
+        case Keyword::In:
+            operation = BinaryExpression::In;
+            break;
+        case Keyword::Is:
+        {
+            Token nextTok = lexer.NextToken();
+            if (nextTok != Token::Identifier)
+                return MakeParseError(ErrorCode::ExpectedIdentifier, nextTok);
+    
+            std::string name = AsString(nextTok.value);
+            ParseResult<CallParams> params;
+    
+            if (lexer.EatIfEqual('('))
+                params = ParseCallParams(lexer);
+    
+            if (!params)
+                return params.get_unexpected();
+    
+            return std::make_shared<IsExpression>(*left, std::move(name), std::move(*params));
+        }
+        default:
+            lexer.ReturnToken();
+            return left;            
+        }
     }
 
     auto right = ParseStringConcat(lexer);
@@ -263,7 +268,7 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionPars
 ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseUnaryPlusMinus(LexScanner& lexer)
 {
     Token tok = lexer.NextToken();
-    if (tok != '+' && tok != '-' && tok != Token::LogicalNot)
+    if (tok != '+' && tok != '-' && lexer.GetAsKeyword(tok) != Keyword::LogicalNot)
     {
         lexer.ReturnToken();
         return ParseValueExpression(lexer);
@@ -279,14 +284,21 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionPars
 ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseValueExpression(LexScanner& lexer)
 {
     Token tok = lexer.NextToken();
+    static const std::unordered_set<Keyword> forbiddenKw = {Keyword::Is, Keyword::In, Keyword::If, Keyword::Else};
 
     ParseResult<ExpressionEvaluatorPtr<Expression>> valueRef;
 
     switch (tok.type)
     {
     case Token::Identifier:
+    {
+        auto kwType = lexer.GetAsKeyword(tok);
+        if (forbiddenKw.count(kwType) != 0)
+            return MakeParseError(ErrorCode::UnexpectedToken, tok);
+            
         valueRef = std::make_shared<ValueRefExpression>(AsString(tok.value));
         break;
+    }
     case Token::IntegerNum:
     case Token::FloatNum:
     case Token::String:
@@ -565,7 +577,7 @@ ExpressionParser::ParseResult<ExpressionEvaluatorPtr<IfExpression>> ExpressionPa
             return testExpr.get_unexpected();
 
         ParseResult<ExpressionEvaluatorPtr<>> altValue;
-        if (lexer.PeekNextToken() == Token::Else)
+        if (lexer.GetAsKeyword(lexer.PeekNextToken()) == Keyword::Else)
         {
             lexer.EatToken();
             auto value = ParseFullExpression(lexer);

src/filters.cpp

@@ -319,14 +319,14 @@ InternalValue GroupBy::Filter(const InternalValue& baseVal, RenderContext& conte
 
 ApplyMacro::ApplyMacro(FilterParams params)
 {
-    ParseParams({{"name", true}}, params);
+    ParseParams({{"macro", true}}, params);
     m_mappingParams.kwParams = m_args.extraKwArgs;
     m_mappingParams.posParams = m_args.extraPosArgs;
 }
 
 InternalValue ApplyMacro::Filter(const InternalValue& baseVal, RenderContext& context)
 {
-    InternalValue macroName = GetArgumentValue("name", context);
+    InternalValue macroName = GetArgumentValue("macro", context);
     if (IsEmpty(macroName))
         return InternalValue();
 

src/lexer.cpp

@@ -80,7 +80,25 @@ bool Lexer::ProcessNumber(const lexertk::token&, Token& newToken)
 
 bool Lexer::ProcessSymbolOrKeyword(const lexertk::token&, Token& newToken)
 {
-    Token::Type tokType = m_helper->GetKeyword(newToken.range);
+    Keyword kwType = m_helper->GetKeyword(newToken.range);
+    Token::Type tokType = Token::Unknown;
+    
+    switch (kwType)
+    {
+    case Keyword::None:
+        tokType = Token::None;
+        break;
+    case Keyword::True:
+        tokType = Token::True;
+        break;
+    case Keyword::False:
+        tokType = Token::False;
+        break;
+    default:
+        tokType = Token::Unknown;
+        break;
+    }
+    
     if (tokType == Token::Unknown)
     {
         newToken.type = Token::Identifier;

src/lexer.h

@@ -52,18 +52,18 @@ struct Token
         GreaterEqual,
         StarStar,
         DashDash,
-        LogicalOr,
-        LogicalAnd,
-        LogicalNot,
         MulMul,
         DivDiv,
         True,
         False,
         None,
-        In,
-        Is,
 
         // Keywords
+        LogicalOr,
+        LogicalAnd,
+        LogicalNot,
+        In,
+        Is,
         For,
         Endfor,
         If,
@@ -83,6 +83,8 @@ struct Token
         EndSet,
         Include,
         Import,
+        Recursive,
+        Scoped,
 
         // Template control
         CommentBegin,
@@ -92,7 +94,7 @@ struct Token
         ExprBegin,
         ExprEnd,
     };
-
+    
     Type type = Unknown;
     CharRange range = {0, 0};
     InternalValue value;
@@ -119,11 +121,47 @@ struct Token
     }
 };
 
+enum class Keyword
+{
+    Unknown,
+
+    // Keywords
+    LogicalOr,
+    LogicalAnd,
+    LogicalNot,
+    True,
+    False,
+    None,
+    In,
+    Is,
+    For,
+    Endfor,
+    If,
+    Else,
+    ElIf,
+    EndIf,
+    Block,
+    EndBlock,
+    Extends,
+    Macro,
+    EndMacro,
+    Call,
+    EndCall,
+    Filter,
+    EndFilter,
+    Set,
+    EndSet,
+    Include,
+    Import,
+    Recursive,
+    Scoped
+};
+
 struct LexerHelper
 {
     virtual std::string GetAsString(const CharRange& range) = 0;
     virtual InternalValue GetAsValue(const CharRange& range, Token::Type type) = 0;
-    virtual Token::Type GetKeyword(const CharRange& range) = 0;
+    virtual Keyword GetKeyword(const CharRange& range) = 0;
     virtual char GetCharAt(size_t pos) = 0;
 };
 
@@ -142,6 +180,8 @@ class Lexer
     {
         return m_tokens;
     }
+    
+    auto GetHelper() const {return m_helper;}
 
 private:
     bool ProcessNumber(const lexertk::token& token, Token& newToken);
@@ -188,6 +228,7 @@ class LexScanner
     };
 
     LexScanner(const Lexer& lexer)
+        : m_helper(lexer.GetHelper())
     {
         m_state.m_begin = lexer.GetTokens().begin();
         m_state.m_end = lexer.GetTokens().end();
@@ -252,7 +293,27 @@ class LexScanner
             return type == Token::Type::Eof;
         }
 
-        if (m_state.m_cur->type == type)
+        return EatIfEqualImpl(tok, [type](const Token& t) {return t.type == type;});
+    }
+    
+    auto GetAsKeyword(const Token& tok) const
+    {
+        return m_helper->GetKeyword(tok.range);
+    }
+    
+    bool EatIfEqual(Keyword kwType, Token* tok = nullptr)
+    {
+        if (m_state.m_cur == m_state.m_end)
+            return false;
+
+        return EatIfEqualImpl(tok, [this, kwType](const Token& t) {return GetAsKeyword(t) == kwType;});
+    }
+    
+private:
+    template<typename Fn>
+    bool EatIfEqualImpl(Token* tok, Fn&& predicate)
+    {
+        if (predicate(*m_state.m_cur))
         {
             if (tok)
                 *tok = *m_state.m_cur;
@@ -265,6 +326,8 @@ class LexScanner
 
 private:
     State m_state;
+    LexerHelper* m_helper;
+    
     static const Token& EofToken()
     {
         static Token eof;
@@ -275,4 +338,16 @@ class LexScanner
 
 } // jinja2
 
+namespace std
+{
+template<>
+struct hash<jinja2::Keyword>
+{
+    size_t operator()(jinja2::Keyword kw) const
+    {
+        return std::hash<int>{}(static_cast<int>(kw));
+    }
+};
+}
+
 #endif // LEXER_H