Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Extended lexer.Definition to support directly lexing strings and []byte slices. Remove ebnf and regex lexers. An adapter has been added for v0 lexers.
- Loading branch information
1 parent
2403858
commit 1444519
Showing
35 changed files
with
202 additions
and
1,875 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| package participle | ||
| package participle_test | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package lexer | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "io" | ||
| "strings" | ||
| ) | ||
|
|
||
| type legacy struct { | ||
| legacy interface { | ||
| Lex(io.Reader) (Lexer, error) | ||
| Symbols() map[string]rune | ||
| } | ||
| } | ||
|
|
||
| func (l legacy) LexReader(r io.Reader) (Lexer, error) { return l.legacy.Lex(r) } | ||
| func (l legacy) LexString(s string) (Lexer, error) { return l.legacy.Lex(strings.NewReader(s)) } | ||
| func (l legacy) LexBytes(b []byte) (Lexer, error) { return l.legacy.Lex(bytes.NewReader(b)) } | ||
| func (l legacy) Symbols() map[string]rune { return l.legacy.Symbols() } | ||
|
|
||
| // Legacy is a shim for Participle v0 lexer definitions. | ||
| func Legacy(def interface { | ||
| Lex(io.Reader) (Lexer, error) | ||
| Symbols() map[string]rune | ||
| }) Definition { | ||
| return legacy{def} | ||
| } | ||
|
|
||
| // Simple upgrades a lexer that only implements LexReader() by using | ||
| // strings/bytes.NewReader(). | ||
| func Simple(def interface { | ||
| Symbols() map[string]rune | ||
| LexReader(io.Reader) (Lexer, error) | ||
| }) Definition { | ||
| return simple{def} | ||
| } | ||
|
|
||
| type simplei interface { | ||
| Symbols() map[string]rune | ||
| LexReader(io.Reader) (Lexer, error) | ||
| } | ||
|
|
||
| type simple struct{ simplei } | ||
|
|
||
| func (s simple) LexString(str string) (Lexer, error) { return s.LexReader(strings.NewReader(str)) } | ||
| func (s simple) LexBytes(b []byte) (Lexer, error) { return s.LexReader(bytes.NewReader(b)) } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,19 +1,5 @@ | ||
| // Package lexer defines interfaces and implementations used by Participle to perform lexing. | ||
| // | ||
| // The primary interfaces are Definition and Lexer. There are three implementations of these | ||
| // interfaces: | ||
| // | ||
| // TextScannerLexer is based on text/scanner. This is the fastest, but least flexible, in that | ||
| // tokens are restricted to those supported by that package. It can scan about 5M tokens/second on a | ||
| // late 2013 15" MacBook Pro. | ||
| // | ||
| // The second lexer is constructed via the Regexp() function, mapping regexp capture groups | ||
| // to tokens. The complete input source is read into memory, so it is unsuitable for large inputs. | ||
| // | ||
| // The final lexer provided accepts a lexical grammar in EBNF. Each capitalised production is a | ||
| // lexical token supported by the resulting Lexer. This is very flexible, but a bit slower, scanning | ||
| // around 730K tokens/second on the same machine, though it is currently completely unoptimised. | ||
| // This could/should be converted to a table-based lexer. | ||
| // | ||
| // Lexer implementations must use Panic/Panicf to report errors. | ||
| // The primary interfaces are Definition and Lexer. There is one concrete implementation included, | ||
| // the stateful lexer. | ||
| package lexer |
Oops, something went wrong.