Add TokenType and ExpressionType and put in separate files to allow i…

…mport of symbol.h from literal.h

include/yaramod/types/expression.h

@@ -13,6 +13,7 @@
 
 #include "yaramod/utils/visitor_result.h"
 #include "yaramod/types/token_stream.h"
+#include "yaramod/types/expression_type.h"
 
 namespace yaramod {
 
@@ -28,16 +29,7 @@ class Expression
 	using Ptr = std::shared_ptr<Expression>;
 
 	///< Type of the expression.
-	enum class Type
-	{
-		Undefined,
-		Bool,
-		Int,
-		String,
-		Regexp,
-		Object,
-		Float
-	};
+	using Type = ExpressionType;
 
 	/// @name Constructors
 	/// @{

include/yaramod/types/literal.h

@@ -15,11 +15,10 @@
 #include <variant>
 
 #include "yaramod/yaramod_error.h"
+#include "yaramod/types/symbol.h"
 
 namespace yaramod {
 
-class Symbol;
-
 /**
  * Class representing literal. Literal can be either
  * string, integer or boolean. This class can only bear
@@ -80,7 +79,9 @@ class Literal
 	void setValue(std::uint64_t i, const std::optional<std::string>& integral_formatted_value = std::nullopt);
 	void setValue(double f, const std::optional<std::string>& integral_formatted_value = std::nullopt);
 	void setValue(const std::shared_ptr<Symbol>& s, const std::string& symbol_name);
+	//TODO delete this
 	void setValue(std::shared_ptr<Symbol>&& s, std::string&& symbol_name);
+	void setValue(std::shared_ptr<Symbol>&& s);
 	/// @}
 
 	/// @name String representation

include/yaramod/types/symbol.h

@@ -11,7 +11,8 @@
 #include <string>
 #include <unordered_map>
 
-#include "yaramod/types/expression.h"
+#include "yaramod/types/expression_type.h"
+#include "yaramod/types/token_type.h"
 
 namespace yaramod {
 
@@ -43,7 +44,7 @@ class Symbol
 	/// @name Getter methods
 	/// @{
 	const std::string& getName() const { return _name; }
-	Expression::Type getDataType() const { return _dataType; }
+	ExpressionType getDataType() const { return _dataType; }
 	Symbol::Type getType() const { return _type; }
 	TokenType getTokenType() const
 	{
@@ -71,13 +72,13 @@ class Symbol
 protected:
 	/// @name Constructors
 	/// @{
-	Symbol(Symbol::Type type, const std::string& name, Expression::Type dataType)
+	Symbol(Symbol::Type type, const std::string& name, ExpressionType dataType)
 		: _type(type), _name(name), _dataType(dataType) {}
 	/// @}
 
 	Symbol::Type _type; ///< Type of the symbol
 	std::string _name; ///< Name
-	Expression::Type _dataType; ///< Data type of the symbol
+	ExpressionType _dataType; ///< Data type of the symbol
 };
 
 }

include/yaramod/types/symbols.h

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <optional>
+#include <vector>
 
 #include "yaramod/types/symbol.h"
 
@@ -19,30 +20,30 @@ namespace yaramod {
 class ValueSymbol : public Symbol
 {
 public:
-	ValueSymbol(const std::string& name, Expression::Type dataType) : Symbol(Symbol::Type::Value, name, dataType) {}
+	ValueSymbol(const std::string& name, ExpressionType dataType) : Symbol(Symbol::Type::Value, name, dataType) {}
 };
 
 /**
  * Abstract class representing iterable symbol. Iterable symbol may be
  * array or dictionary symbol. Iterable symbols store data type of the elements
- * they are iterating over. If the element type is @c Expression::Type::Object then
+ * they are iterating over. If the element type is @c ExpressionType::Object then
  * iterable symbol also carries the symbol representing structured type of the element.
  */
 class IterableSymbol : public Symbol
 {
 public:
-	Expression::Type getElementType() const { return _elementType; }
+	ExpressionType getElementType() const { return _elementType; }
 	const std::shared_ptr<Symbol>& getStructuredElementType() const { return _structuredType; }
 
-	bool isStructured() const { return _elementType == Expression::Type::Object && _structuredType; }
+	bool isStructured() const { return _elementType == ExpressionType::Object && _structuredType; }
 
 protected:
-	IterableSymbol(Symbol::Type type, const std::string& name, Expression::Type elementType)
-		: Symbol(type, name, Expression::Type::Object), _elementType(elementType), _structuredType() {}
+	IterableSymbol(Symbol::Type type, const std::string& name, ExpressionType elementType)
+		: Symbol(type, name, ExpressionType::Object), _elementType(elementType), _structuredType() {}
 	IterableSymbol(Symbol::Type type, const std::string& name, const std::shared_ptr<Symbol>& structuredType)
-		: Symbol(type, name, Expression::Type::Object), _elementType(Expression::Type::Object), _structuredType(structuredType) {}
+		: Symbol(type, name, ExpressionType::Object), _elementType(ExpressionType::Object), _structuredType(structuredType) {}
 
-	Expression::Type _elementType; ///< Element of the iterated data
+	ExpressionType _elementType; ///< Element of the iterated data
 	std::shared_ptr<Symbol> _structuredType; ///< Structured type of the object elements
 };
 
@@ -53,7 +54,7 @@ class IterableSymbol : public Symbol
 class ArraySymbol : public IterableSymbol
 {
 public:
-	ArraySymbol(const std::string& name, Expression::Type elementType) : IterableSymbol(Symbol::Type::Array, name, elementType) {}
+	ArraySymbol(const std::string& name, ExpressionType elementType) : IterableSymbol(Symbol::Type::Array, name, elementType) {}
 	ArraySymbol(const std::string& name, const std::shared_ptr<Symbol>& structuredType) : IterableSymbol(Symbol::Type::Array, name, structuredType) {}
 };
 
@@ -64,7 +65,7 @@ class ArraySymbol : public IterableSymbol
 class DictionarySymbol : public IterableSymbol
 {
 public:
-	DictionarySymbol(const std::string& name, Expression::Type elementType) : IterableSymbol(Symbol::Type::Dictionary, name, elementType) {}
+	DictionarySymbol(const std::string& name, ExpressionType elementType) : IterableSymbol(Symbol::Type::Dictionary, name, elementType) {}
 	DictionarySymbol(const std::string& name, const std::shared_ptr<Symbol>& structuredType) : IterableSymbol(Symbol::Type::Array, name, structuredType) {}
 };
 
@@ -76,28 +77,28 @@ class FunctionSymbol : public Symbol
 {
 public:
 	template <typename... Args>
-	FunctionSymbol(const std::string& name, Expression::Type returnType, const Args&... args)
-		: Symbol(Symbol::Type::Function, name, Expression::Type::Object), _returnType(returnType), _argTypesOverloads(1)
+	FunctionSymbol(const std::string& name, ExpressionType returnType, const Args&... args)
+		: Symbol(Symbol::Type::Function, name, ExpressionType::Object), _returnType(returnType), _argTypesOverloads(1)
 	{
 		_initArgs(args...);
 	}
 
-	Expression::Type getReturnType() const { return _returnType; }
-	const std::vector<std::vector<Expression::Type>>& getAllOverloads() const { return _argTypesOverloads; }
+	ExpressionType getReturnType() const { return _returnType; }
+	const std::vector<std::vector<ExpressionType>>& getAllOverloads() const { return _argTypesOverloads; }
 
 	std::size_t getArgumentCount(std::size_t overloadIndex = 0) const
 	{
 		assert(overloadIndex < _argTypesOverloads.size());
 		return _argTypesOverloads[overloadIndex].size();
 	}
 
-	std::vector<Expression::Type> getArgumentTypes(std::size_t overloadIndex = 0) const
+	std::vector<ExpressionType> getArgumentTypes(std::size_t overloadIndex = 0) const
 	{
 		assert(overloadIndex < _argTypesOverloads.size());
 		return _argTypesOverloads[overloadIndex];
 	}
 
-	bool addOverload(const std::vector<Expression::Type>& argTypes)
+	bool addOverload(const std::vector<ExpressionType>& argTypes)
 	{
 		if (overloadExists(argTypes))
 			return false;
@@ -106,7 +107,7 @@ class FunctionSymbol : public Symbol
 		return true;
 	}
 
-	bool overloadExists(const std::vector<Expression::Type>& args) const
+	bool overloadExists(const std::vector<ExpressionType>& args) const
 	{
 		for (const auto& overload : _argTypesOverloads)
 		{
@@ -126,14 +127,14 @@ class FunctionSymbol : public Symbol
 	void _initArgs() {}
 
 	template <typename... Args>
-	void _initArgs(Expression::Type argType, const Args&... args)
+	void _initArgs(ExpressionType argType, const Args&... args)
 	{
 		_argTypesOverloads.front().push_back(argType);
 		_initArgs(args...);
 	}
 
-	Expression::Type _returnType; ///< Return type of the function
-	std::vector<std::vector<Expression::Type>> _argTypesOverloads; ///< All possible overloads of the function
+	ExpressionType _returnType; ///< Return type of the function
+	std::vector<std::vector<ExpressionType>> _argTypesOverloads; ///< All possible overloads of the function
 };
 
 /**
@@ -142,7 +143,7 @@ class FunctionSymbol : public Symbol
 class StructureSymbol : public Symbol
 {
 public:
-	StructureSymbol(const std::string& name) : Symbol(Symbol::Type::Structure, name, Expression::Type::Object) {}
+	StructureSymbol(const std::string& name) : Symbol(Symbol::Type::Structure, name, ExpressionType::Object) {}
 
 	std::optional<std::shared_ptr<Symbol>> getAttribute(const std::string& name) const
 	{

include/yaramod/types/token.h

@@ -10,6 +10,7 @@
 
 #include "yaramod/types/literal.h"
 #include "yaramod/types/location.h"
+#include "yaramod/types/token_type.h"
 #include "yaramod/yaramod_error.h"
 
 namespace yaramod {
@@ -20,140 +21,6 @@ using TokenConstIt = std::list<Token>::const_iterator;
 using TokenItReversed = std::reverse_iterator<TokenIt>;
 using TokenConstItReversed = std::reverse_iterator<TokenConstIt>;
 
-/**
- * Represents type of parsed tokens.
- */
-enum TokenType
-{
-	RULE_NAME,
-	TAG,
-	HEX_ALT, // '|'
-	HEX_NIBBLE,
-	HEX_WILDCARD,
-	HEX_WILDCARD_LOW,
-	HEX_WILDCARD_HIGH,
-	HEX_JUMP_LEFT_BRACKET, // '['
-	HEX_JUMP_RIGHT_BRACKET, // ']'
-	HEX_ALT_LEFT_BRACKET, // '('
-	HEX_ALT_RIGHT_BRACKET, // ')'
-	HEX_JUMP_FIXED,
-	HEX_START_BRACKET, // '{'
-	HEX_END_BRACKET, // '}'
-	NEW_LINE,
-	META, // 'meta'
-	LQUOTE,
-	RQUOTE,
-	RULE_END, // '}'
-	RULE_BEGIN, // '{'
-	RANGE,
-	DOT,
-	DOUBLE_DOT,
-	LT,
-	GT,
-	LE,
-	GE,
-	EQ,
-	NEQ,
-	SHIFT_LEFT,
-	SHIFT_RIGHT,
-	MINUS,
-	PLUS,
-	MULTIPLY,
-	DIVIDE,
-	MODULO,
-	BITWISE_XOR,
-	BITWISE_AND,
-	BITWISE_OR,
-	BITWISE_NOT,
-	LP,
-	RP,
-	LCB, // '{'
-	RCB, // '}'
-	ASSIGN,
-	COLON,
-	COLON_BEFORE_NEWLINE,
-	COMMA,
-	PRIVATE,
-	GLOBAL,
-	NONE,
-	RULE,
-	STRINGS,
-	CONDITION,
-	ASCII,
-	NOCASE,
-	WIDE,
-	FULLWORD,
-	PRIVATE_STRING_MODIFIER,
-	XOR,
-	IMPORT_MODULE,
-	IMPORT_KEYWORD,
-	NOT,
-	AND,
-	OR,
-	ALL,
-	ANY,
-	OF,
-	THEM,
-	FOR,
-	ENTRYPOINT,
-	OP_AT,
-	OP_IN,
-	FILESIZE,
-	CONTAINS,
-	MATCHES,
-	SLASH,
-	STRING_LITERAL,
-	INTEGER,
-	DOUBLE,
-	STRING_ID,
-	STRING_ID_BEFORE_NEWLINE,
-	STRING_ID_WILDCARD,
-	STRING_LENGTH,
-	STRING_OFFSET,
-	STRING_COUNT,
-	ID,
-	INTEGER_FUNCTION,
-	LSQB, // '['
-	RSQB, // ']'
-	DASH, // '-'
-	REGEXP_OR,
-	REGEXP_ITER,
-	REGEXP_PITER,
-	REGEXP_OPTIONAL,
-	REGEXP_START_SLASH,
-	REGEXP_END_SLASH,
-	REGEXP_CHAR,
-	REGEXP_RANGE,
-	REGEXP_TEXT,
-	REGEXP_CLASS_NEGATIVE,
-	REGEXP_MODIFIERS,
-	REGEXP_GREEDY,
-	UNARY_MINUS,
-	META_KEY,
-	META_VALUE,
-	STRING_KEY,
-	VALUE_SYMBOL,
-	FUNCTION_SYMBOL,
-	ARRAY_SYMBOL,
-	DICTIONARY_SYMBOL,
-	STRUCTURE_SYMBOL,
-	LP_ENUMERATION,
-	RP_ENUMERATION,
-	LP_WITH_SPACE_AFTER,
-	RP_WITH_SPACE_BEFORE,
-	LP_WITH_SPACES,
-	RP_WITH_SPACES,
-	BOOL_TRUE,
-	BOOL_FALSE,
-	ONELINE_COMMENT,
-	COMMENT,
-	INCLUDE_DIRECTIVE,
-	INCLUDE_PATH,
-	FUNCTION_CALL_LP,
-	FUNCTION_CALL_RP,
-	INVALID,
-};
-
 class TokenStream;
 
 /**
@@ -200,6 +67,7 @@ class Token
 	void setValue(double value, const std::optional<std::string>& integral_formated_value = std::nullopt) { _value->setValue(value, integral_formated_value); }
 	void setValue(const std::shared_ptr<Symbol>& value, const std::string& symbol_name) { _value->setValue(value, symbol_name); }
 	void setValue(std::shared_ptr<Symbol>&& value, std::string&& symbol_name) { _value->setValue(std::move(value), std::move(symbol_name)); }
+	void setValue(std::shared_ptr<Symbol>&& value);
 
 	void setType(TokenType type) { _type = type; }
 	void setFlag(bool flag) { _flag = flag; }