From 9fc4d2bb75029531078b5b2de5e72f6d4f47d12b Mon Sep 17 00:00:00 2001
From: David Grudl <david@grudl.com>
Date: Wed, 27 Sep 2023 11:58:05 +0200
Subject: [PATCH] added Position to Token

---
 src/Neon/Lexer.php       | 30 ++++++++++++++++++++++++------
 src/Neon/Position.php    | 27 +++++++++++++++++++++++++++
 src/Neon/Token.php       |  1 +
 src/Neon/TokenStream.php | 15 ++-------------
 4 files changed, 54 insertions(+), 19 deletions(-)
 create mode 100644 src/Neon/Position.php

diff --git a/src/Neon/Lexer.php b/src/Neon/Lexer.php
index ae2e734..4791f9f 100644
--- a/src/Neon/Lexer.php
+++ b/src/Neon/Lexer.php
@@ -56,20 +56,20 @@ public function tokenize(string $input): TokenStream
 		}
 
 		$types = array_keys(self::Patterns);
-		$offset = 0;
+		$position = new Position;
 
 		$tokens = [];
 		foreach ($matches as $match) {
 			$type = $types[count($match) - 2];
-			$tokens[] = new Token($type === Token::Char ? $match[0] : $type, $match[0]);
-			$offset += strlen($match[0]);
+			$tokens[] = new Token($type === Token::Char ? $match[0] : $type, $match[0], $position);
+			$position = $this->advance($position, $match[0]);
 		}
 
-		$tokens[] = new Token(Token::End, '');
+		$tokens[] = new Token(Token::End, '', $position);
 
 		$stream = new TokenStream($tokens);
-		if ($offset !== strlen($input)) {
-			$s = str_replace("\n", '\n', substr($input, $offset, 40));
+		if ($position->offset !== strlen($input)) {
+			$s = str_replace("\n", '\n', substr($input, $position->offset, 40));
 			$stream->error("Unexpected '$s'", count($tokens) - 1);
 		}
 
@@ -77,6 +77,24 @@ public function tokenize(string $input): TokenStream
 	}
 
 
+	private function advance(Position $position, string $str): Position
+	{
+		if ($lines = substr_count($str, "\n")) {
+			return new Position(
+				$position->line + $lines,
+				strlen($str) - strrpos($str, "\n"),
+				$position->offset + strlen($str),
+			);
+		} else {
+			return new Position(
+				$position->line,
+				$position->column + strlen($str),
+				$position->offset + strlen($str),
+			);
+		}
+	}
+
+
 	public static function requiresDelimiters(string $s): bool
 	{
 		return preg_match('~[\x00-\x1F]|^[+-.]?\d|^(true|false|yes|no|on|off|null)$~Di', $s)
diff --git a/src/Neon/Position.php b/src/Neon/Position.php
new file mode 100644
index 0000000..a172bd1
--- /dev/null
+++ b/src/Neon/Position.php
@@ -0,0 +1,27 @@
+<?php
+
+/**
+ * This file is part of the Nette Framework (https://nette.org)
+ * Copyright (c) 2004 David Grudl (https://davidgrudl.com)
+ */
+
+declare(strict_types=1);
+
+namespace Nette\Neon;
+
+
+final class Position
+{
+	public function __construct(
+		public /*readonly*/ int $line = 1,
+		public /*readonly*/ int $column = 1,
+		public /*readonly*/ int $offset = 0,
+	) {
+	}
+
+
+	public function __toString(): string
+	{
+		return "on line $this->line" . ($this->column ? " at column $this->column" : '');
+	}
+}
diff --git a/src/Neon/Token.php b/src/Neon/Token.php
index 9985185..d92eee4 100644
--- a/src/Neon/Token.php
+++ b/src/Neon/Token.php
@@ -25,6 +25,7 @@ final class Token
 	public function __construct(
 		public int|string $type,
 		public string $text,
+		public Position $position,
 	) {
 	}
 
diff --git a/src/Neon/TokenStream.php b/src/Neon/TokenStream.php
index e756ba2..2b5c460 100644
--- a/src/Neon/TokenStream.php
+++ b/src/Neon/TokenStream.php
@@ -74,21 +74,10 @@ public function getIndentation(): string
 	public function error(?string $message = null, ?int $pos = null): void
 	{
 		$pos ??= $this->index;
-		$input = '';
-		foreach ($this->tokens as $i => $token) {
-			if ($i >= $pos) {
-				break;
-			}
-
-			$input .= $token->text;
-		}
-
-		$line = substr_count($input, "\n") + 1;
-		$col = strlen($input) - strrpos("\n" . $input, "\n") + 1;
 		$token = $this->tokens[$pos];
 		$message ??= 'Unexpected ' . ($token->type === Token::End
 			? 'end'
-			: "'" . str_replace("\n", '<new line>', substr($this->tokens[$pos]->text, 0, 40)) . "'");
-		throw new Exception("$message on line $line at column $col");
+			: "'" . str_replace("\n", '<new line>', substr($token->text, 0, 40)) . "'");
+		throw new Exception("$message $token->position");
 	}
 }