Skip to content

Commit

Permalink
TokenStream: renamed some methods
Browse files Browse the repository at this point in the history
  • Loading branch information
@dg
dg committed Apr 28, 2024
1 parent d035c40 commit 4cca7ad
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 57 deletions.
80 changes: 40 additions & 40 deletions src/Neon/Parser.php
View file
Expand Up @@ -24,11 +24,11 @@ public function parse(TokenStream $tokens): Node
$this->tokens = $tokens;
$this->initLines();

while ($this->tokens->consume(Token::Newline));
while ($this->tokens->tryConsume(Token::Newline));
$node = $this->parseBlock($this->tokens->getIndentation());

while ($this->tokens->consume(Token::Newline));
if ($this->tokens->isNext()) {
while ($this->tokens->tryConsume(Token::Newline));
if ($this->tokens->is()) {
$this->tokens->error();
}

Expand All @@ -45,16 +45,16 @@ private function parseBlock(string $indent, bool $onlyBullets = false): Node
loop:
$item = new Node\ArrayItemNode;
$this->injectPos($item);
if ($this->tokens->consume('-')) {
if ($this->tokens->tryConsume('-')) {
// continue
} elseif (!$this->tokens->isNext() || $onlyBullets) {
} elseif (!$this->tokens->is() || $onlyBullets) {
return $res->items
? $res
: $this->injectPos(new Node\LiteralNode(null));

} else {
$value = $this->parseValue();
if ($this->tokens->consume(':', '=')) {
if ($this->tokens->tryConsume(':', '=')) {
$this->checkArrayKey($value, $keyCheck);
$item->key = $value;
} else {
Expand All @@ -70,8 +70,8 @@ private function parseBlock(string $indent, bool $onlyBullets = false): Node
$item->value = new Node\LiteralNode(null);
$this->injectPos($item->value);

if ($this->tokens->consume(Token::Newline)) {
while ($this->tokens->consume(Token::Newline));
if ($this->tokens->tryConsume(Token::Newline)) {
while ($this->tokens->tryConsume(Token::Newline));
$nextIndent = $this->tokens->getIndentation();

if (strncmp($nextIndent, $indent, min(strlen($nextIndent), strlen($indent)))) {
Expand All @@ -83,20 +83,20 @@ private function parseBlock(string $indent, bool $onlyBullets = false): Node
} elseif (strlen($nextIndent) < strlen($indent)) { // close block
return $res;

} elseif ($item->key !== null && $this->tokens->isNext('-')) { // special dash subblock
} elseif ($item->key !== null && $this->tokens->is('-')) { // special dash subblock
$item->value = $this->parseBlock($indent, onlyBullets: true);
}
} elseif ($item->key === null) { // open new block after dash
$save = $this->tokens->getPos();
$save = $this->tokens->getIndex();
try {
$item->value = $this->parseBlock($indent . "\t");
} catch (Exception) {
$this->tokens->seek($save);
$item->value = $this->parseBlock($indent . ' ');
}
} elseif ($this->tokens->isNext()) {
} elseif ($this->tokens->is()) {
$item->value = $this->parseValue();
if ($this->tokens->isNext() && !$this->tokens->isNext(Token::Newline)) {
if ($this->tokens->is() && !$this->tokens->is(Token::Newline)) {
$this->tokens->error();
}
}
Expand All @@ -108,8 +108,8 @@ private function parseBlock(string $indent, bool $onlyBullets = false): Node
$this->injectPos($res, $res->startTokenPos, $item->value->endTokenPos);
$this->injectPos($item, $item->startTokenPos, $item->value->endTokenPos);

while ($this->tokens->consume(Token::Newline));
if (!$this->tokens->isNext()) {
while ($this->tokens->tryConsume(Token::Newline));
if (!$this->tokens->is()) {
return $res;
}

Expand All @@ -130,19 +130,19 @@ private function parseBlock(string $indent, bool $onlyBullets = false): Node

private function parseValue(): Node
{
if ($token = $this->tokens->consume(Token::String)) {
if ($token = $this->tokens->tryConsume(Token::String)) {
try {
$node = new Node\StringNode(Node\StringNode::parse($token->value));
$this->injectPos($node, $this->tokens->getPos() - 1);
$this->injectPos($node, $this->tokens->getIndex() - 1);
} catch (Exception $e) {
$this->tokens->error($e->getMessage(), $this->tokens->getPos() - 1);
$this->tokens->error($e->getMessage(), $this->tokens->getIndex() - 1);
}
} elseif ($token = $this->tokens->consume(Token::Literal)) {
$pos = $this->tokens->getPos() - 1;
$node = new Node\LiteralNode(Node\LiteralNode::parse($token->value, $this->tokens->isNext(':', '=')));
} elseif ($token = $this->tokens->tryConsume(Token::Literal)) {
$pos = $this->tokens->getIndex() - 1;
$node = new Node\LiteralNode(Node\LiteralNode::parse($token->value, $this->tokens->is(':', '=')));
$this->injectPos($node, $pos);

} elseif ($this->tokens->isNext('[', '(', '{')) {
} elseif ($this->tokens->is('[', '(', '{')) {
$node = $this->parseBraces();

} else {
Expand All @@ -155,17 +155,17 @@ private function parseValue(): Node

private function parseEntity(Node $node): Node
{
if (!$this->tokens->isNext('(')) {
if (!$this->tokens->is('(')) {
return $node;
}

$attributes = $this->parseBraces();
$entities[] = $this->injectPos(new Node\EntityNode($node, $attributes->items), $node->startTokenPos, $attributes->endTokenPos);

while ($token = $this->tokens->consume(Token::Literal)) {
while ($token = $this->tokens->tryConsume(Token::Literal)) {
$valueNode = new Node\LiteralNode(Node\LiteralNode::parse($token->value));
$this->injectPos($valueNode, $this->tokens->getPos() - 1);
if ($this->tokens->isNext('(')) {
$this->injectPos($valueNode, $this->tokens->getIndex() - 1);
if ($this->tokens->is('(')) {
$attributes = $this->parseBraces();
$entities[] = $this->injectPos(new Node\EntityNode($valueNode, $attributes->items), $valueNode->startTokenPos, $attributes->endTokenPos);
} else {
Expand All @@ -182,41 +182,41 @@ private function parseEntity(Node $node): Node

private function parseBraces(): Node\InlineArrayNode
{
$token = $this->tokens->consume();
$token = $this->tokens->tryConsume();
$endBrace = ['[' => ']', '{' => '}', '(' => ')'][$token->value];
$res = new Node\InlineArrayNode($token->value);
$this->injectPos($res, $this->tokens->getPos() - 1);
$this->injectPos($res, $this->tokens->getIndex() - 1);
$keyCheck = [];

loop:
while ($this->tokens->consume(Token::Newline));
if ($this->tokens->consume($endBrace)) {
$this->injectPos($res, $res->startTokenPos, $this->tokens->getPos() - 1);
while ($this->tokens->tryConsume(Token::Newline));
if ($this->tokens->tryConsume($endBrace)) {
$this->injectPos($res, $res->startTokenPos, $this->tokens->getIndex() - 1);
return $res;
}

$res->items[] = $item = new Node\ArrayItemNode;
$this->injectPos($item, $this->tokens->getPos());
$this->injectPos($item, $this->tokens->getIndex());
$value = $this->parseValue();

if ($this->tokens->consume(':', '=')) {
if ($this->tokens->tryConsume(':', '=')) {
$this->checkArrayKey($value, $keyCheck);
$item->key = $value;
$item->value = $this->tokens->isNext(Token::Newline, ',', $endBrace)
? $this->injectPos(new Node\LiteralNode(null), $this->tokens->getPos())
$item->value = $this->tokens->is(Token::Newline, ',', $endBrace)
? $this->injectPos(new Node\LiteralNode(null), $this->tokens->getIndex())
: $this->parseValue();
} else {
$item->value = $value;
}

$this->injectPos($item, $item->startTokenPos, $item->value->endTokenPos);

$old = $this->tokens->getPos();
while ($this->tokens->consume(Token::Newline));
$this->tokens->consume(',');
if ($old !== $this->tokens->getPos()) {
$old = $this->tokens->getIndex();
while ($this->tokens->tryConsume(Token::Newline));
$this->tokens->tryConsume(',');
if ($old !== $this->tokens->getIndex()) {
goto loop;
} elseif (!$this->tokens->isNext($endBrace)) {
} elseif (!$this->tokens->is($endBrace)) {
$this->tokens->error();
}

Expand All @@ -242,7 +242,7 @@ private function checkArrayKey(Node $key, array &$arr): void

private function injectPos(Node $node, int $start = null, int $end = null): Node
{
$node->startTokenPos = $start ?? $this->tokens->getPos();
$node->startTokenPos = $start ?? $this->tokens->getIndex();
$node->startLine = $this->posToLine[$node->startTokenPos];
$node->endTokenPos = $end ?? $node->startTokenPos;
$node->endLine = $this->posToLine[$node->endTokenPos + 1] ?? end($this->posToLine);
Expand Down
34 changes: 17 additions & 17 deletions src/Neon/TokenStream.php
View file
Expand Up @@ -13,7 +13,7 @@
/** @internal */
final class TokenStream
{
private int $pos = 0;
private int $index = 0;


public function __construct(
Expand All @@ -23,15 +23,15 @@ public function __construct(
}


public function getPos(): int
public function getIndex(): int
{
return $this->pos;
return $this->index;
}


public function seek(int $position): void
public function seek(int $index): void
{
$this->pos = $position;
$this->index = $index;
}


Expand All @@ -42,39 +42,39 @@ public function getTokens(): array
}


public function isNext(int|string ...$types): bool
public function is(int|string ...$types): bool
{
while (in_array($this->tokens[$this->pos]->type ?? null, [Token::Comment, Token::Whitespace], strict: true)) {
$this->pos++;
while (in_array($this->tokens[$this->index]->type ?? null, [Token::Comment, Token::Whitespace], strict: true)) {
$this->index++;
}

return $types
? in_array($this->tokens[$this->pos]->type ?? null, $types, strict: true)
: isset($this->tokens[$this->pos]);
? in_array($this->tokens[$this->index]->type ?? null, $types, strict: true)
: isset($this->tokens[$this->index]);
}


public function consume(int|string ...$types): ?Token
public function tryConsume(int|string ...$types): ?Token
{
return $this->isNext(...$types)
? $this->tokens[$this->pos++]
return $this->is(...$types)
? $this->tokens[$this->index++]
: null;
}


public function getIndentation(): string
{
return in_array($this->tokens[$this->pos - 2]->type ?? null, [Token::Newline, null], strict: true)
&& ($this->tokens[$this->pos - 1]->type ?? null) === Token::Whitespace
? $this->tokens[$this->pos - 1]->value
return in_array($this->tokens[$this->index - 2]->type ?? null, [Token::Newline, null], strict: true)
&& ($this->tokens[$this->index - 1]->type ?? null) === Token::Whitespace
? $this->tokens[$this->index - 1]->value
: '';
}


/** @return never */
public function error(?string $message = null, ?int $pos = null): void
{
$pos ??= $this->pos;
$pos ??= $this->index;
$input = '';
foreach ($this->tokens as $i => $token) {
if ($i >= $pos) {
Expand Down

0 comments on commit 4cca7ad

Please sign in to comment.