Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Fix tokenizer hitting tail-recursion limit #125

Merged
merged 3 commits into from
Mar 8, 2024
Merged

fix: Fix tokenizer hitting tail-recursion limit #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6fc9cb9
Split `skipIgnored` back out of tail-recursive tokenizer
kitten Mar 8, 2024
6eeebc9
Add changeset
kitten Mar 8, 2024
3a757fd
Restore prior structure of parser
kitten Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/eight-yaks-provide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'gql.tada': patch
---

Fix tokenizer hitting tail recursion limit by recursing on each ignored token.
96 changes: 42 additions & 54 deletions src/parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ export interface _match<Out, In extends any[]> {
in: In;
}

export interface _match2<Out1, Out2, In extends any[]> {
out1: Out1;
out2: Out2;
in: In;
}

type takeOptionalName<In extends any[]> = In extends [
{ kind: Token.Name; name: infer Name },
...infer In,
Expand Down Expand Up @@ -112,19 +118,28 @@ export type takeDirectives<
? takeDirectives<In, Const, [...Directives, Directive]>
: _match<Directives, In>;

type _takeField<In extends any[], Alias> = In extends [
Token.Colon,
{ kind: Token.Name; name: infer Name },
type _takeFieldName<In extends any[]> = In extends [
{ kind: Token.Name; name: infer MaybeAlias },
...infer In,
]
? In extends [Token.Colon, { kind: Token.Name; name: infer Name }, ...infer In]
? _match2<{ kind: Kind.NAME; value: MaybeAlias }, { kind: Kind.NAME; value: Name }, In>
: _match2<undefined, { kind: Kind.NAME; value: MaybeAlias }, In>
: void;

type _takeField<In extends any[]> = _takeFieldName<In> extends _match2<
infer Alias,
infer Name,
infer In
>
? takeArguments<In, false> extends _match<infer Arguments, infer In>
? takeDirectives<In, false> extends _match<infer Directives, infer In>
? takeSelectionSet<In> extends _match<infer SelectionSet, infer In>
? _match<
{
kind: Kind.FIELD;
alias: Alias;
name: { kind: Kind.NAME; value: Name };
name: Name;
arguments: Arguments;
directives: Directives;
selectionSet: SelectionSet;
Expand All @@ -135,7 +150,7 @@ type _takeField<In extends any[], Alias> = In extends [
{
kind: Kind.FIELD;
alias: Alias;
name: { kind: Kind.NAME; value: Name };
name: Name;
arguments: Arguments;
directives: Directives;
selectionSet: undefined;
Expand All @@ -144,33 +159,7 @@ type _takeField<In extends any[], Alias> = In extends [
>
: void
: void
: takeArguments<In, false> extends _match<infer Arguments, infer In>
? takeDirectives<In, false> extends _match<infer Directives, infer In>
? takeSelectionSet<In> extends _match<infer SelectionSet, infer In>
? _match<
{
kind: Kind.FIELD;
alias: undefined;
name: Alias;
arguments: Arguments;
directives: Directives;
selectionSet: SelectionSet;
},
In
>
: _match<
{
kind: Kind.FIELD;
alias: undefined;
name: Alias;
arguments: Arguments;
directives: Directives;
selectionSet: undefined;
},
In
>
: void
: void;
: void;

export type takeType<In extends any[]> = In extends [Token.BracketOpen, ...infer In]
? takeType<In> extends _match<infer Subtype, infer In>
Expand All @@ -193,6 +182,7 @@ export type takeType<In extends any[]> = In extends [Token.BracketOpen, ...infer
: void;

type _takeFragmentSpread<In extends any[]> = In extends [
Token.Spread,
{ kind: Token.Name; name: 'on' },
{ kind: Token.Name; name: infer Type },
...infer In,
Expand All @@ -210,7 +200,7 @@ type _takeFragmentSpread<In extends any[]> = In extends [
>
: void
: void
: In extends [{ kind: Token.Name; name: infer Name }, ...infer In]
: In extends [Token.Spread, { kind: Token.Name; name: infer Name }, ...infer In]
? takeDirectives<In, false> extends _match<infer Directives, infer In>
? _match<
{
Expand All @@ -221,31 +211,29 @@ type _takeFragmentSpread<In extends any[]> = In extends [
In
>
: void
: takeDirectives<In, false> extends _match<infer Directives, infer In>
? takeSelectionSet<In> extends _match<infer SelectionSet, infer In>
? _match<
{
kind: Kind.INLINE_FRAGMENT;
typeCondition: undefined;
directives: Directives;
selectionSet: SelectionSet;
},
In
>
: In extends [Token.Spread, ...infer In]
? takeDirectives<In, false> extends _match<infer Directives, infer In>
? takeSelectionSet<In> extends _match<infer SelectionSet, infer In>
? _match<
{
kind: Kind.INLINE_FRAGMENT;
typeCondition: undefined;
directives: Directives;
selectionSet: SelectionSet;
},
In
>
: void
: void
: void;

type _takeSelectionRec<Selections extends any[], In extends any[]> = In extends [
Token.Spread,
...infer In,
]
? _takeFragmentSpread<In> extends _match<infer Selection, infer In>
type _takeSelectionRec<Selections extends any[], In extends any[]> = _takeField<In> extends _match<
infer Selection,
infer In
>
? _takeSelectionRec<[...Selections, Selection], In>
: _takeFragmentSpread<In> extends _match<infer Selection, infer In>
? _takeSelectionRec<[...Selections, Selection], In>
: void
: In extends [{ kind: Token.Name; name: infer Alias }, ...infer In]
? _takeField<In, { kind: Kind.NAME; value: Alias }> extends _match<infer Selection, infer In>
? _takeSelectionRec<[...Selections, Selection], In>
: void
: In extends [Token.BraceClose, ...infer In]
? _match<{ kind: Kind.SELECTION_SET; selections: Selections }, In>
: void;
Expand Down
8 changes: 7 additions & 1 deletion src/tokenizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ type letter =
| 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm'
| 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z';

type skipIgnored<In> = In extends `#${infer _}\n${infer In}`
? skipIgnored<In>
: In extends `${ignored}${infer In}`
? skipIgnored<In>
: In;

type skipDigits<In> = In extends `${digit}${infer In}` ? skipDigits<In> : In;

type skipFloat<In> = In extends `${'.'}${infer In}`
Expand Down Expand Up @@ -96,7 +102,7 @@ type tokenizeRec<State> =
: State extends _state<infer In, infer Out>
? tokenizeRec<
In extends `#${string}\n${infer In}` ? _state<In, Out>
: In extends `${ignored}${infer In}` ? _state<In, Out>
: In extends `${ignored}${infer In}` ? _state<skipIgnored<In>, Out>
: In extends `...${infer In}` ? _state<In, [...Out, Token.Spread]>
: In extends `!${infer In}` ? _state<In, [...Out, Token.Exclam]>
: In extends `=${infer In}` ? _state<In, [...Out, Token.Equal]>
Expand Down
Loading