This repository has been archived by the owner on Mar 8, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
preserve raw string literals in the native ast; fixes #32
Signed-off-by: Denys Smirnov <denys@sourced.tech>
- Loading branch information
Denys Smirnov
committed
Jan 3, 2019
1 parent
aebcf20
commit 676f7ea
Showing
41 changed files
with
464 additions
and
86 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| package normalizer | ||
|
|
||
| import ( | ||
| "strconv" | ||
| "unicode/utf8" | ||
| ) | ||
|
|
||
| // Functions below are copied from strconv.Unquote and strconv.Quote. | ||
| // Original functions are unable to escape/unescape values containing | ||
| // multiple characters since in Go single quotes represent a rune literal | ||
|
|
||
| // unquoteSingle is the same as strconv.Unquote, but uses ' as a quote. | ||
| func unquoteSingle(s string) (string, error) { | ||
| var runeTmp [utf8.UTFMax]byte | ||
| buf := make([]byte, 0, 3*len(s)/2) | ||
| for len(s) > 0 { | ||
| c, multibyte, ss, err := strconv.UnquoteChar(s, '\'') | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| s = ss | ||
| if c < utf8.RuneSelf || !multibyte { | ||
| buf = append(buf, byte(c)) | ||
| } else { | ||
| n := utf8.EncodeRune(runeTmp[:], c) | ||
| buf = append(buf, runeTmp[:n]...) | ||
| } | ||
| } | ||
| return string(buf), nil | ||
| } | ||
|
|
||
| const lowerhex = "0123456789abcdef" | ||
|
|
||
| // quoteSingle is the same as strconv.Quote, but uses ' as a quote. | ||
| func quoteSingle(s string) string { | ||
| const ( | ||
| quote = '\'' | ||
| ) | ||
|
|
||
| buf := make([]byte, 0, 3*len(s)/2) | ||
| buf = append(buf, quote) | ||
| for width := 0; len(s) > 0; s = s[width:] { | ||
| r := rune(s[0]) | ||
| width = 1 | ||
| if r >= utf8.RuneSelf { | ||
| r, width = utf8.DecodeRuneInString(s) | ||
| } | ||
| if width == 1 && r == utf8.RuneError { | ||
| buf = append(buf, `\x`...) | ||
| buf = append(buf, lowerhex[s[0]>>4]) | ||
| buf = append(buf, lowerhex[s[0]&0xF]) | ||
| continue | ||
| } | ||
| buf = appendEscapedRune(buf, r, quote) | ||
| } | ||
| buf = append(buf, quote) | ||
| return string(buf) | ||
| } | ||
|
|
||
| func appendEscapedRune(buf []byte, r rune, quote byte) []byte { | ||
| var runeTmp [utf8.UTFMax]byte | ||
| if r == rune(quote) || r == '\\' { // always backslashed | ||
| buf = append(buf, '\\') | ||
| buf = append(buf, byte(r)) | ||
| return buf | ||
| } | ||
| if strconv.IsPrint(r) { | ||
| n := utf8.EncodeRune(runeTmp[:], r) | ||
| buf = append(buf, runeTmp[:n]...) | ||
| return buf | ||
| } | ||
| switch r { | ||
| case '\a': | ||
| buf = append(buf, `\a`...) | ||
| case '\b': | ||
| buf = append(buf, `\b`...) | ||
| case '\f': | ||
| buf = append(buf, `\f`...) | ||
| case '\n': | ||
| buf = append(buf, `\n`...) | ||
| case '\r': | ||
| buf = append(buf, `\r`...) | ||
| case '\t': | ||
| buf = append(buf, `\t`...) | ||
| case '\v': | ||
| buf = append(buf, `\v`...) | ||
| default: | ||
| switch { | ||
| case r < ' ': | ||
| buf = append(buf, `\x`...) | ||
| buf = append(buf, lowerhex[byte(r)>>4]) | ||
| buf = append(buf, lowerhex[byte(r)&0xF]) | ||
| case r > utf8.MaxRune: | ||
| r = 0xFFFD | ||
| fallthrough | ||
| case r < 0x10000: | ||
| buf = append(buf, `\u`...) | ||
| for s := 12; s >= 0; s -= 4 { | ||
| buf = append(buf, lowerhex[r>>uint(s)&0xF]) | ||
| } | ||
| default: | ||
| buf = append(buf, `\U`...) | ||
| for s := 28; s >= 0; s -= 4 { | ||
| buf = append(buf, lowerhex[r>>uint(s)&0xF]) | ||
| } | ||
| } | ||
| } | ||
| return buf | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -425,7 +425,7 @@ | |
| col: 10, | ||
| }, | ||
| }, | ||
| Format: "", | ||
| Format: "single", | ||
| Value: "", | ||
| }, | ||
| ], | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -71,7 +71,7 @@ | |
| col: 26, | ||
| }, | ||
| }, | ||
| Format: "", | ||
| Format: "single", | ||
| Value: "Hello World", | ||
| }, | ||
| ], | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.