Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Byte/raw binary literal fixes
  • Loading branch information
emberian committed Jul 21, 2014
1 parent 9fc5cf9 commit f8fd32e
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 4 deletions.
4 changes: 4 additions & 0 deletions src/grammar/.gitignore
@@ -0,0 +1,4 @@
verify
*.class
*.java
*.tokens
9 changes: 7 additions & 2 deletions src/grammar/RustLexer.g4
Expand Up @@ -44,6 +44,7 @@ SHR : '>>' ;

BINOP
: PLUS
| SLASH
| MINUS
| STAR
| PERCENT
Expand Down Expand Up @@ -95,6 +96,10 @@ LIT_CHAR
: '\'' ( '\\' CHAR_ESCAPE | ~[\\'\n\t\r] ) '\''
;

LIT_BYTE
: 'b\'' ( '\\' ( [xX] HEXIT HEXIT | [nrt\\'"0] ) | ~[\\'\n\t\r] ) '\''
;

fragment INT_SUFFIX
: 'i'
| 'i8'
Expand Down Expand Up @@ -130,7 +135,7 @@ LIT_STR
;

LIT_BINARY : 'b' LIT_STR ;
LIT_BINARY_RAW : 'b' LIT_STR_RAW ;
LIT_BINARY_RAW : 'rb' LIT_STR_RAW ;

/* this is a bit messy */

Expand Down Expand Up @@ -159,7 +164,7 @@ OUTER_DOC_COMMENT : '//!' ~[\r\n]* -> type(DOC_COMMENT) ;
LINE_COMMENT : '//' ~[\r\n]* -> type(COMMENT) ;

DOC_BLOCK_COMMENT
: ('/**' | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/' -> type(DOC_COMMENT)
: ('/**' ~[*] | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/' -> type(DOC_COMMENT)
;

BLOCK_COMMENT : '/*' (BLOCK_COMMENT | .)*? '*/' -> type(COMMENT) ;
15 changes: 13 additions & 2 deletions src/grammar/verify.rs
Expand Up @@ -71,6 +71,7 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"IDENT" => id(),
"PLUS" => BINOP(PLUS),
"LIT_CHAR" => LIT_CHAR(Name(0)),
"LIT_BYTE" => LIT_BYTE(Name(0)),
"EQ" => EQ,
"RBRACKET" => RBRACKET,
"COMMENT" => COMMENT,
Expand Down Expand Up @@ -124,7 +125,7 @@ fn str_to_binop(s: &str) -> BinOp {
}
}

/// Assuming a raw string/binary literal, strip out the leading/trailing
/// Assuming a string/binary literal, strip out the leading/trailing
/// hashes and surrounding quotes/raw/binary prefix.
fn fix(mut lit: &str) -> ast::Name {
if lit.char_at(0) == 'r' {
Expand All @@ -143,6 +144,15 @@ fn fix(mut lit: &str) -> ast::Name {
parse::token::intern(lit.slice(leading_hashes + 1, lit.len() - leading_hashes - 1))
}

/// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
fn fixchar(mut lit: &str) -> ast::Name {
if lit.char_at(0) == 'b' {
lit = lit.slice_from(1);
}

parse::token::intern(lit.slice(1, lit.len() - 1))
}

fn count(lit: &str) -> uint {
lit.chars().take_while(|c| *c == '#').count()
}
Expand All @@ -167,7 +177,8 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, Token>) -> TokenAndSpan {
BINOPEQ(..) => BINOPEQ(str_to_binop(content.slice_to(content.len() - 1))),
LIT_STR(..) => LIT_STR(fix(content)),
LIT_STR_RAW(..) => LIT_STR_RAW(fix(content), count(content)),
LIT_CHAR(..) => LIT_CHAR(nm),
LIT_CHAR(..) => LIT_CHAR(fixchar(content)),
LIT_BYTE(..) => LIT_BYTE(fixchar(content)),
DOC_COMMENT(..) => DOC_COMMENT(nm),
LIT_INTEGER(..) => LIT_INTEGER(nm),
LIT_FLOAT(..) => LIT_FLOAT(nm),
Expand Down

0 comments on commit f8fd32e

Please sign in to comment.