Token locations python bindings (#120)

* bindings: Expose TokenType for Token

* bindings: Introduce `Position` and expose location

* location: Merge Rule::Location and Location

In order to preserve backwards compatibility the `line_number` property
has been kept, but will be deprecated. Now any other `Location` instance
will include `line_number`, that should not be mentioned in the documentation.

* string: Add location to strings

* token: Expose common tokens for strings and rules

* TokenType: convert to cpp11 enum class

* Revert "TokenType: convert to cpp11 enum class"

This reverts commit 5e6dc9c.

* tests: Add location python bindings test

include/yaramod/parser/location.h

@@ -15,10 +15,27 @@ namespace yaramod {
 class Location
 {
 public:
+
+	/**
+	 * Class representing position (line, column) in the file.
+	 */
+	struct Position {
+
+		Position(std::size_t line, std::size_t column) : line(line), column(column) {}
+
+		std::size_t getLine() { return line; }
+		std::size_t getColumn() { return column; }
+
+		std::size_t line;
+		std::size_t column;
+	};
+
 	Location() : Location(std::string{}) {}
 	Location(const std::string& filePath) : Location(filePath, 1, 0) {}
 	Location(const std::string& filePath, std::size_t line, std::size_t column)
 		: _filePath(filePath), _begin(line, column), _end(line, column) {}
+	Location(const std::string& filePath, const Position &begin, const Position& end)
+		: _filePath(filePath), _begin(begin), _end(end) {}
 	Location(const Location&) = default;
 	Location(Location&&) noexcept = default;
 
@@ -30,15 +47,15 @@ class Location
 	void addLine(std::size_t count = 1)
 	{
 		std::swap(_begin, _end);
-		_end.first = _begin.first + count; // line
-		_end.second = 0; // column
+		_end.line = _begin.line + count; // line
+		_end.column = 0; // column
 	}
 
 	void addColumn(std::size_t count)
 	{
-		_begin.first = _end.first;
-		_begin.second = _end.second;
-		_end.second += count;
+		_begin.line = _end.line;
+		_begin.column = _end.column;
+		_end.column += count;
 	}
 
 	void reset()
@@ -51,25 +68,30 @@ class Location
 	/// @name Getters
 	/// @{
 	bool isUnnamed() const { return _filePath == "[stream]"; }
+	/**
+	 * Returns the absolute path of a file in which this rule was located.
+	 * Returns "[stream]" in case this rule was parsed from input stream and not a file,
+	 * or if this file was created with `YaraRuleBuilder`.
+	 */
 	const std::string& getFilePath() const { return _filePath; }
-	std::pair<std::size_t, std::size_t> begin() const { return {_begin.first, _begin.second + 1}; }
-	const std::pair<std::size_t, std::size_t>& end() const { return _end; }
+	Position begin() const { return {_begin.line, _begin.column + 1}; }
+	const Position& end() const { return _end; }
 	/// @}
 
 	friend std::ostream& operator<<(std::ostream& os, const Location& location)
 	{
 		if (!location.isUnnamed())
 			os << location.getFilePath() << ':';
-		os << location.begin().first << '.' << location.begin().second;
-		if (location.begin().second < location.end().second)
-			os << '-' << location.end().second;
+		os << location.begin().line << '.' << location.begin().column;
+		if (location.begin().column < location.end().column)
+			os << '-' << location.end().column;
 		return os;
 	}
 
 private:
 	std::string _filePath;
-	std::pair<std::size_t, std::size_t> _begin; // (line, column)
-	std::pair<std::size_t, std::size_t> _end; // (line, column)
+	Position _begin;
+	Position _end;
 };
 
 } //namespace yaramod

include/yaramod/types/rule.h

@@ -10,6 +10,7 @@
 #include <optional>
 #include <vector>
 
+#include "yaramod/parser/location.h"
 #include "yaramod/types/expression.h"
 #include "yaramod/types/meta.h"
 #include "yaramod/types/string.h"
@@ -26,12 +27,6 @@ class YaraFileBuilder;
 class Rule
 {
 public:
-	struct Location
-	{
-		std::string filePath;
-		std::uint64_t lineNumber;
-	};
-
 	using StringsTrie = Trie<std::shared_ptr<String>>;
 
 	/**
@@ -79,7 +74,7 @@ class Rule
 	const std::shared_ptr<Symbol>& getSymbol() const;
 	Meta* getMetaWithName(const std::string& key);
 	const Meta* getMetaWithName(const std::string& key) const;
-	const Location& getLocation() const;
+	const Location& getLocation() const { return _location; }
 	TokenStream* getTokenStream() const { return _tokenStream.get(); }
 	TokenIt getFirstTokenIt() const;
 	TokenIt getLastTokenIt() const;
@@ -91,7 +86,7 @@ class Rule
 	void setMetas(const std::vector<Meta>& metas);
 	void setTags(const std::vector<std::string>& tags);
 	void setCondition(const Expression::Ptr& condition);
-	void setLocation(const std::string& filePath, std::uint64_t lineNumber);
+	void setLocation(const Location& location) { _location = location; }
 	void setModifier(const Modifier& modifier);
 	/// @}
 

include/yaramod/types/string.h

@@ -80,13 +80,15 @@ class String
 	/// @name Getter methods
 	/// @{
 	Type getType() const { return _type; }
+
 	const Literal* getIdentifierTokenIt() const
 	{
 		if (_id)
 			return &(_id.value()->getLiteral());
 		else
 			return nullptr;
 	}
+
 	std::string getIdentifier() const
 	{
 		if (_id)
@@ -95,6 +97,20 @@ class String
 			return std::string();
 	}
 
+	std::optional<Token> getIdentifierToken() const
+	{
+		if (_id)
+			return *_id.value();
+		return {};
+	}
+
+	std::optional<Token> getAssignToken() const
+	{
+		if (_assignToken)
+			return *_assignToken.value();
+		return {};
+	}
+
 	std::string getModifiersText() const
 	{
 		if (_mods.empty())
@@ -126,6 +142,14 @@ class String
 
 	const std::shared_ptr<TokenStream>& getTokenStream() const { return _tokenStream; }
 
+	const Location getLocation() const { 
+		return {
+			_id.value_or(getFirstTokenIt())->getLocation().getFilePath(), 
+			_id.value_or(getFirstTokenIt())->getLocation().begin(), 
+			getLastTokenIt()->getLocation().end()
+		}; 
+	}
+
 	virtual TokenIt getFirstTokenIt() const = 0;
 	virtual TokenIt getLastTokenIt() const = 0;
 	/// @}

src/parser/parser_driver.cpp

@@ -1867,7 +1867,7 @@ void ParserDriver::addRule(Rule&& rule)
  */
 void ParserDriver::addRule(std::unique_ptr<Rule>&& rule)
 {
-	rule->setLocation(_lastRuleLocation.getFilePath(), _lastRuleLocation.begin().first);
+	rule->setLocation(_lastRuleLocation);
 
 	if (ruleExists(rule->getName()))
 		throw ParserError("Error: Redefinition of rule " + rule->getName());