added Position to Token

src/Neon/Lexer.php

@@ -56,27 +56,45 @@ public function tokenize(string $input): TokenStream
 		}
 
 		$types = array_keys(self::Patterns);
-		$offset = 0;
+		$position = new Position;
 
 		$tokens = [];
 		foreach ($matches as $match) {
 			$type = $types[count($match) - 2];
-			$tokens[] = new Token($type === Token::Char ? $match[0] : $type, $match[0]);
-			$offset += strlen($match[0]);
+			$tokens[] = new Token($type === Token::Char ? $match[0] : $type, $match[0], $position);
+			$position = $this->advance($position, $match[0]);
 		}
 
-		$tokens[] = new Token(Token::End, '');
+		$tokens[] = new Token(Token::End, '', $position);
 
 		$stream = new TokenStream($tokens);
-		if ($offset !== strlen($input)) {
-			$s = str_replace("\n", '\n', substr($input, $offset, 40));
+		if ($position->offset !== strlen($input)) {
+			$s = str_replace("\n", '\n', substr($input, $position->offset, 40));
 			$stream->error("Unexpected '$s'", count($tokens) - 1);
 		}
 
 		return $stream;
 	}
 
 
+	private function advance(Position $position, string $str): Position
+	{
+		if ($lines = substr_count($str, "\n")) {
+			return new Position(
+				$position->line + $lines,
+				strlen($str) - strrpos($str, "\n"),
+				$position->offset + strlen($str),
+			);
+		} else {
+			return new Position(
+				$position->line,
+				$position->column + strlen($str),
+				$position->offset + strlen($str),
+			);
+		}
+	}
+
+
 	public static function requiresDelimiters(string $s): bool
 	{
 		return preg_match('~[\x00-\x1F]|^[+-.]?\d|^(true|false|yes|no|on|off|null)$~Di', $s)

src/Neon/Position.php

@@ -0,0 +1,27 @@
+<?php
+
+/**
+ * This file is part of the Nette Framework (https://nette.org)
+ * Copyright (c) 2004 David Grudl (https://davidgrudl.com)
+ */
+
+declare(strict_types=1);
+
+namespace Nette\Neon;
+
+
+final class Position
+{
+	public function __construct(
+		public /*readonly*/ int $line = 1,
+		public /*readonly*/ int $column = 1,
+		public /*readonly*/ int $offset = 0,
+	) {
+	}
+
+
+	public function __toString(): string
+	{
+		return "on line $this->line" . ($this->column ? " at column $this->column" : '');
+	}
+}

src/Neon/Token.php

@@ -25,6 +25,7 @@ final class Token
 	public function __construct(
 		public int|string $type,
 		public string $text,
+		public Position $position,
 	) {
 	}
 

src/Neon/TokenStream.php

@@ -74,21 +74,10 @@ public function getIndentation(): string
 	public function error(?string $message = null, ?int $pos = null): void
 	{
 		$pos ??= $this->index;
-		$input = '';
-		foreach ($this->tokens as $i => $token) {
-			if ($i >= $pos) {
-				break;
-			}
-
-			$input .= $token->text;
-		}
-
-		$line = substr_count($input, "\n") + 1;
-		$col = strlen($input) - strrpos("\n" . $input, "\n") + 1;
 		$token = $this->tokens[$pos];
 		$message ??= 'Unexpected ' . ($token->type === Token::End
 			? 'end'
-			: "'" . str_replace("\n", '<new line>', substr($this->tokens[$pos]->text, 0, 40)) . "'");
-		throw new Exception("$message on line $line at column $col");
+			: "'" . str_replace("\n", '<new line>', substr($token->text, 0, 40)) . "'");
+		throw new Exception("$message $token->position");
 	}
 }