Skip to content
Permalink
Browse files

libsyntax: Forbid escapes in the inclusive range `\x80`-`\xff` in

Unicode characters and strings.

Use `\u0080`-`\u00ff` instead. ASCII/byte literals are unaffected.

This PR introduces a new function, `escape_default`, into the ASCII
module. This was necessary for the pretty printer to continue to
function.

RFC #326.

Closes #18062.

[breaking-change]
  • Loading branch information
pcwalton committed Nov 4, 2014
1 parent bb70ee5 commit e8d6031c71a5ab42648f26a253671ba17584407a
@@ -283,7 +283,7 @@ def load_east_asian_width(want_widths, except_cats):
return widths

def escape_char(c):
if c <= 0xff:
if c <= 0x7f:
return "'\\x%2.2x'" % c
if c <= 0xffff:
return "'\\u%4.4x'" % c
@@ -810,15 +810,15 @@ mod tests {
assert_eq!("".len(), 0u);
assert_eq!("hello world".len(), 11u);
assert_eq!("\x63".len(), 1u);
assert_eq!("\xa2".len(), 2u);
assert_eq!("\u00a2".len(), 2u);
assert_eq!("\u03c0".len(), 2u);
assert_eq!("\u2620".len(), 3u);
assert_eq!("\U0001d11e".len(), 4u);

assert_eq!("".char_len(), 0u);
assert_eq!("hello world".char_len(), 11u);
assert_eq!("\x63".char_len(), 1u);
assert_eq!("\xa2".char_len(), 1u);
assert_eq!("\u00a2".char_len(), 1u);
assert_eq!("\u03c0".char_len(), 1u);
assert_eq!("\u2620".char_len(), 1u);
assert_eq!("\U0001d11e".char_len(), 1u);
@@ -1499,7 +1499,8 @@ mod tests {
assert_eq!("a c".escape_unicode(), String::from_str("\\x61\\x20\\x63"));
assert_eq!("\r\n\t".escape_unicode(), String::from_str("\\x0d\\x0a\\x09"));
assert_eq!("'\"\\".escape_unicode(), String::from_str("\\x27\\x22\\x5c"));
assert_eq!("\x00\x01\xfe\xff".escape_unicode(), String::from_str("\\x00\\x01\\xfe\\xff"));
assert_eq!("\x00\x01\u00fe\u00ff".escape_unicode(),
String::from_str("\\x00\\x01\\u00fe\\u00ff"));
assert_eq!("\u0100\uffff".escape_unicode(), String::from_str("\\u0100\\uffff"));
assert_eq!("\U00010000\U0010ffff".escape_unicode(),
String::from_str("\\U00010000\\U0010ffff"));
@@ -1783,11 +1784,11 @@ mod tests {
t!("\u2126", "\u03a9");
t!("\u1e0b\u0323", "\u1e0d\u0307");
t!("\u1e0d\u0307", "\u1e0d\u0307");
t!("a\u0301", "\xe1");
t!("a\u0301", "\u00e1");
t!("\u0301a", "\u0301a");
t!("\ud4db", "\ud4db");
t!("\uac1c", "\uac1c");
t!("a\u0300\u0305\u0315\u05aeb", "\xe0\u05ae\u0305\u0315b");
t!("a\u0300\u0305\u0315\u05aeb", "\u00e0\u05ae\u0305\u0315b");
}

#[test]
@@ -1803,11 +1804,11 @@ mod tests {
t!("\u2126", "\u03a9");
t!("\u1e0b\u0323", "\u1e0d\u0307");
t!("\u1e0d\u0307", "\u1e0d\u0307");
t!("a\u0301", "\xe1");
t!("a\u0301", "\u00e1");
t!("\u0301a", "\u0301a");
t!("\ud4db", "\ud4db");
t!("\uac1c", "\uac1c");
t!("a\u0300\u0305\u0315\u05aeb", "\xe0\u05ae\u0305\u0315b");
t!("a\u0300\u0305\u0315\u05aeb", "\u00e0\u05ae\u0305\u0315b");
}

#[test]
@@ -176,7 +176,7 @@ pub fn escape_unicode(c: char, f: |char|) {
// here.
f('\\');
let pad = match () {
_ if c <= '\xff' => { f('x'); 2 }
_ if c <= '\x7f' => { f('x'); 2 }
_ if c <= '\uffff' => { f('u'); 4 }
_ => { f('U'); 8 }
};
@@ -140,8 +140,8 @@ fn test_escape_default() {
assert_eq!(s.as_slice(), "\\x1f");
let s = string('\x7f');
assert_eq!(s.as_slice(), "\\x7f");
let s = string('\xff');
assert_eq!(s.as_slice(), "\\xff");
let s = string('\u00ff');
assert_eq!(s.as_slice(), "\\u00ff");
let s = string('\u011b');
assert_eq!(s.as_slice(), "\\u011b");
let s = string('\U0001d4b6');
@@ -211,8 +211,8 @@ fn test_width() {
assert_eq!('h'.width(false),Some(2));
assert_eq!('h'.width(true),Some(2));

assert_eq!('\xAD'.width(false),Some(1));
assert_eq!('\xAD'.width(true),Some(1));
assert_eq!('\u00AD'.width(false),Some(1));
assert_eq!('\u00AD'.width(true),Some(1));

assert_eq!('\u1160'.width(false),Some(0));
assert_eq!('\u1160'.width(true),Some(0));
@@ -209,14 +209,16 @@ mat!(match_flag_ungreedy_greedy, "(?U)a+?", "aa", Some((0, 2)))
mat!(match_flag_ungreedy_noop, "(?U)(?-U)a+", "aa", Some((0, 2)))

// Some Unicode tests.
mat!(uni_literal, r"Ⅰ", "Ⅰ", Some((0, 3)))
// A couple of these are commented out because something in the guts of macro expansion is creating
// invalid byte strings.
//mat!(uni_literal, r"Ⅰ", "Ⅰ", Some((0, 3)))
mat!(uni_one, r"\pN", "Ⅰ", Some((0, 3)))
mat!(uni_mixed, r"\pN+", "Ⅰ1Ⅱ2", Some((0, 8)))
mat!(uni_not, r"\PN+", "abⅠ", Some((0, 2)))
mat!(uni_not_class, r"[\PN]+", "abⅠ", Some((0, 2)))
mat!(uni_not_class_neg, r"[^\PN]+", "abⅠ", Some((2, 5)))
mat!(uni_case, r"(?i)Δ", "δ", Some((0, 2)))
mat!(uni_case_not, r"Δ", "δ", None)
//mat!(uni_case_not, r"Δ", "δ", None)
mat!(uni_case_upper, r"\p{Lu}+", "ΛΘΓΔα", Some((0, 8)))
mat!(uni_case_upper_nocase_flag, r"(?i)\p{Lu}+", "ΛΘΓΔα", Some((0, 10)))
mat!(uni_case_upper_nocase, r"\p{L}+", "ΛΘΓΔα", Some((0, 10)))
@@ -461,6 +461,38 @@ impl OwnedAsciiExt for Vec<u8> {
}
}

/// Returns a 'default' ASCII and C++11-like literal escape of a `u8`
///
/// The default is chosen with a bias toward producing literals that are
/// legal in a variety of languages, including C++11 and similar C-family
/// languages. The exact rules are:
///
/// - Tab, CR and LF are escaped as '\t', '\r' and '\n' respectively.
/// - Single-quote, double-quote and backslash chars are backslash-escaped.
/// - Any other chars in the range [0x20,0x7e] are not escaped.
/// - Any other chars are given hex escapes.
/// - Unicode escapes are never generated by this function.
pub fn escape_default(c: u8, f: |u8|) {
match c {
b'\t' => { f(b'\\'); f(b't'); }
b'\r' => { f(b'\\'); f(b'r'); }
b'\n' => { f(b'\\'); f(b'n'); }
b'\\' => { f(b'\\'); f(b'\\'); }
b'\'' => { f(b'\\'); f(b'\''); }
b'"' => { f(b'\\'); f(b'"'); }
b'\x20' ... b'\x7e' => { f(c); }
_ => {
f(b'\\');
f(b'x');
for &offset in [4u, 0u].iter() {
match ((c as i32) >> offset) & 0xf {
i @ 0 ... 9 => f(b'0' + (i as u8)),
i => f(b'a' + (i as u8 - 10)),
}
}
}
}
}

pub static ASCII_LOWER_MAP: [u8, ..256] = [
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
@@ -720,7 +720,11 @@ impl<'a> StringReader<'a> {

/// Scan over `n_digits` hex digits, stopping at `delim`, reporting an
/// error if too many or too few digits are encountered.
fn scan_hex_digits(&mut self, n_digits: uint, delim: char) -> bool {
fn scan_hex_digits(&mut self,
n_digits: uint,
delim: char,
below_0x7f_only: bool)
-> bool {
debug!("scanning {} digits until {}", n_digits, delim);
let start_bpos = self.last_pos;
let mut accum_int = 0;
@@ -745,6 +749,13 @@ impl<'a> StringReader<'a> {
self.bump();
}

if below_0x7f_only && accum_int >= 0x80 {
self.err_span_(start_bpos,
self.last_pos,
"this form of character escape may only be used \
with characters in the range [\\x00-\\x7f]");
}

match char::from_u32(accum_int) {
Some(_) => true,
None => {
@@ -773,9 +784,13 @@ impl<'a> StringReader<'a> {
Some(e) => {
return match e {
'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true,
'x' => self.scan_hex_digits(2u, delim),
'u' if !ascii_only => self.scan_hex_digits(4u, delim),
'U' if !ascii_only => self.scan_hex_digits(8u, delim),
'x' => self.scan_hex_digits(2u, delim, !ascii_only),
'u' if !ascii_only => {
self.scan_hex_digits(4u, delim, false)
}
'U' if !ascii_only => {
self.scan_hex_digits(8u, delim, false)
}
'\n' if delim == '"' => {
self.consume_whitespace();
true
@@ -30,6 +30,7 @@ use print::pp::{Breaks, Consistent, Inconsistent, eof};
use print::pp;
use ptr::P;

use std::ascii;
use std::io::{IoResult, MemWriter};
use std::io;
use std::mem;
@@ -2776,7 +2777,7 @@ impl<'a> State<'a> {
ast::LitStr(ref st, style) => self.print_string(st.get(), style),
ast::LitByte(byte) => {
let mut res = String::from_str("b'");
(byte as char).escape_default(|c| res.push(c));
ascii::escape_default(byte, |c| res.push(c as char));
res.push('\'');
word(&mut self.s, res.as_slice())
}
@@ -2821,8 +2822,12 @@ impl<'a> State<'a> {
if val { word(&mut self.s, "true") } else { word(&mut self.s, "false") }
}
ast::LitBinary(ref v) => {
let escaped: String = v.iter().map(|&b| b as char).collect();
word(&mut self.s, format!("b\"{}\"", escaped.escape_default()).as_slice())
let mut escaped: String = String::new();
for &ch in v.iter() {
ascii::escape_default(ch as u8,
|ch| escaped.push(ch as char));
}
word(&mut self.s, format!("b\"{}\"", escaped).as_slice())
}
}
}

5 comments on commit e8d6031

@bors

This comment has been minimized.

Copy link
Contributor

@bors bors replied Nov 5, 2014

saw approval from pcwalton
at pcwalton@e8d6031

@bors

This comment has been minimized.

Copy link
Contributor

@bors bors replied Nov 5, 2014

merging pcwalton/rust/small-escapes = e8d6031 into auto

@bors

This comment has been minimized.

Copy link
Contributor

@bors bors replied Nov 5, 2014

pcwalton/rust/small-escapes = e8d6031 merged ok, testing candidate = 4375b32

@bors

This comment has been minimized.

Copy link
Contributor

@bors bors replied Nov 5, 2014

fast-forwarding master to auto = 4375b32

Please sign in to comment.
You can’t perform that action at this time.