Skip to content

Commit

Permalink
caddyfile: Export Tokenize function for lexing (#3549)
Browse files Browse the repository at this point in the history
  • Loading branch information
@francislavoie
francislavoie committed Jul 20, 2020
1 parent 6cea1f2 commit fb9d874
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 54 deletions.
19 changes: 19 additions & 0 deletions caddyconfig/caddyfile/lexer.go
View file
Expand Up @@ -16,6 +16,7 @@ package caddyfile

import (
"bufio"
"bytes"
"io"
"unicode"
)
Expand Down Expand Up @@ -168,3 +169,21 @@ func (l *lexer) next() bool {
val = append(val, ch)
}
}

// Tokenize takes bytes as input and lexes it into
// a list of tokens that can be parsed as a Caddyfile.
// Also takes a filename to fill the token's File as
// the source of the tokens, which is important to
// determine relative paths for `import` directives.
func Tokenize(input []byte, filename string) ([]Token, error) {
l := lexer{}
if err := l.load(bytes.NewReader(input)); err != nil {
return nil, err
}
var tokens []Token
for l.next() {
l.token.File = filename
tokens = append(tokens, l.token)
}
return tokens, nil
}
84 changes: 37 additions & 47 deletions caddyconfig/caddyfile/lexer_test.go
View file
Expand Up @@ -15,37 +15,35 @@
package caddyfile

import (
"log"
"strings"
"testing"
)

type lexerTestCase struct {
input string
input []byte
expected []Token
}

func TestLexer(t *testing.T) {
testCases := []lexerTestCase{
{
input: `host:123`,
input: []byte(`host:123`),
expected: []Token{
{Line: 1, Text: "host:123"},
},
},
{
input: `host:123
input: []byte(`host:123
directive`,
directive`),
expected: []Token{
{Line: 1, Text: "host:123"},
{Line: 3, Text: "directive"},
},
},
{
input: `host:123 {
input: []byte(`host:123 {
directive
}`,
}`),
expected: []Token{
{Line: 1, Text: "host:123"},
{Line: 1, Text: "{"},
Expand All @@ -54,7 +52,7 @@ func TestLexer(t *testing.T) {
},
},
{
input: `host:123 { directive }`,
input: []byte(`host:123 { directive }`),
expected: []Token{
{Line: 1, Text: "host:123"},
{Line: 1, Text: "{"},
Expand All @@ -63,12 +61,12 @@ func TestLexer(t *testing.T) {
},
},
{
input: `host:123 {
input: []byte(`host:123 {
#comment
directive
# comment
foobar # another comment
}`,
}`),
expected: []Token{
{Line: 1, Text: "host:123"},
{Line: 1, Text: "{"},
Expand All @@ -78,10 +76,10 @@ func TestLexer(t *testing.T) {
},
},
{
input: `host:123 {
input: []byte(`host:123 {
# hash inside string is not a comment
redir / /some/#/path
}`,
}`),
expected: []Token{
{Line: 1, Text: "host:123"},
{Line: 1, Text: "{"},
Expand All @@ -92,14 +90,14 @@ func TestLexer(t *testing.T) {
},
},
{
input: "# comment at beginning of file\n# comment at beginning of line\nhost:123",
input: []byte("# comment at beginning of file\n# comment at beginning of line\nhost:123"),
expected: []Token{
{Line: 3, Text: "host:123"},
},
},
{
input: `a "quoted value" b
foobar`,
input: []byte(`a "quoted value" b
foobar`),
expected: []Token{
{Line: 1, Text: "a"},
{Line: 1, Text: "quoted value"},
Expand All @@ -108,15 +106,15 @@ func TestLexer(t *testing.T) {
},
},
{
input: `A "quoted \"value\" inside" B`,
input: []byte(`A "quoted \"value\" inside" B`),
expected: []Token{
{Line: 1, Text: "A"},
{Line: 1, Text: `quoted "value" inside`},
{Line: 1, Text: "B"},
},
},
{
input: "An escaped \"newline\\\ninside\" quotes",
input: []byte("An escaped \"newline\\\ninside\" quotes"),
expected: []Token{
{Line: 1, Text: "An"},
{Line: 1, Text: "escaped"},
Expand All @@ -125,7 +123,7 @@ func TestLexer(t *testing.T) {
},
},
{
input: "An escaped newline\\\noutside quotes",
input: []byte("An escaped newline\\\noutside quotes"),
expected: []Token{
{Line: 1, Text: "An"},
{Line: 1, Text: "escaped"},
Expand All @@ -135,7 +133,7 @@ func TestLexer(t *testing.T) {
},
},
{
input: "line1\\\nescaped\nline2\nline3",
input: []byte("line1\\\nescaped\nline2\nline3"),
expected: []Token{
{Line: 1, Text: "line1"},
{Line: 1, Text: "escaped"},
Expand All @@ -144,7 +142,7 @@ func TestLexer(t *testing.T) {
},
},
{
input: "line1\\\nescaped1\\\nescaped2\nline4\nline5",
input: []byte("line1\\\nescaped1\\\nescaped2\nline4\nline5"),
expected: []Token{
{Line: 1, Text: "line1"},
{Line: 1, Text: "escaped1"},
Expand All @@ -154,34 +152,34 @@ func TestLexer(t *testing.T) {
},
},
{
input: `"unescapable\ in quotes"`,
input: []byte(`"unescapable\ in quotes"`),
expected: []Token{
{Line: 1, Text: `unescapable\ in quotes`},
},
},
{
input: `"don't\escape"`,
input: []byte(`"don't\escape"`),
expected: []Token{
{Line: 1, Text: `don't\escape`},
},
},
{
input: `"don't\\escape"`,
input: []byte(`"don't\\escape"`),
expected: []Token{
{Line: 1, Text: `don't\\escape`},
},
},
{
input: `un\escapable`,
input: []byte(`un\escapable`),
expected: []Token{
{Line: 1, Text: `un\escapable`},
},
},
{
input: `A "quoted value with line
input: []byte(`A "quoted value with line
break inside" {
foobar
}`,
}`),
expected: []Token{
{Line: 1, Text: "A"},
{Line: 1, Text: "quoted value with line\n\t\t\t\t\tbreak inside"},
Expand All @@ -191,21 +189,21 @@ func TestLexer(t *testing.T) {
},
},
{
input: `"C:\php\php-cgi.exe"`,
input: []byte(`"C:\php\php-cgi.exe"`),
expected: []Token{
{Line: 1, Text: `C:\php\php-cgi.exe`},
},
},
{
input: `empty "" string`,
input: []byte(`empty "" string`),
expected: []Token{
{Line: 1, Text: `empty`},
{Line: 1, Text: ``},
{Line: 1, Text: `string`},
},
},
{
input: "skip those\r\nCR characters",
input: []byte("skip those\r\nCR characters"),
expected: []Token{
{Line: 1, Text: "skip"},
{Line: 1, Text: "those"},
Expand All @@ -214,37 +212,37 @@ func TestLexer(t *testing.T) {
},
},
{
input: "\xEF\xBB\xBF:8080", // test with leading byte order mark
input: []byte("\xEF\xBB\xBF:8080"), // test with leading byte order mark
expected: []Token{
{Line: 1, Text: ":8080"},
},
},
{
input: "simple `backtick quoted` string",
input: []byte("simple `backtick quoted` string"),
expected: []Token{
{Line: 1, Text: `simple`},
{Line: 1, Text: `backtick quoted`},
{Line: 1, Text: `string`},
},
},
{
input: "multiline `backtick\nquoted\n` string",
input: []byte("multiline `backtick\nquoted\n` string"),
expected: []Token{
{Line: 1, Text: `multiline`},
{Line: 1, Text: "backtick\nquoted\n"},
{Line: 3, Text: `string`},
},
},
{
input: "nested `\"quotes inside\" backticks` string",
input: []byte("nested `\"quotes inside\" backticks` string"),
expected: []Token{
{Line: 1, Text: `nested`},
{Line: 1, Text: `"quotes inside" backticks`},
{Line: 1, Text: `string`},
},
},
{
input: "reverse-nested \"`backticks` inside\" quotes",
input: []byte("reverse-nested \"`backticks` inside\" quotes"),
expected: []Token{
{Line: 1, Text: `reverse-nested`},
{Line: 1, Text: "`backticks` inside"},
Expand All @@ -254,22 +252,14 @@ func TestLexer(t *testing.T) {
}

for i, testCase := range testCases {
actual := tokenize(testCase.input)
actual, err := Tokenize(testCase.input, "")
if err != nil {
t.Errorf("%v", err)
}
lexerCompare(t, i, testCase.expected, actual)
}
}

func tokenize(input string) (tokens []Token) {
l := lexer{}
if err := l.load(strings.NewReader(input)); err != nil {
log.Printf("[ERROR] load failed: %v", err)
}
for l.next() {
tokens = append(tokens, l.token)
}
return
}

func lexerCompare(t *testing.T, n int, expected, actual []Token) {
if len(expected) != len(actual) {
t.Errorf("Test case %d: expected %d token(s) but got %d", n, len(expected), len(actual))
Expand Down
8 changes: 1 addition & 7 deletions caddyconfig/caddyfile/parse.go
View file
Expand Up @@ -87,16 +87,10 @@ func allTokens(filename string, input []byte) ([]Token, error) {
if err != nil {
return nil, err
}
l := new(lexer)
err = l.load(bytes.NewReader(input))
tokens, err := Tokenize(input, filename)
if err != nil {
return nil, err
}
var tokens []Token
for l.next() {
l.token.File = filename
tokens = append(tokens, l.token)
}
return tokens, nil
}

Expand Down

0 comments on commit fb9d874

Please sign in to comment.