From 4fe2326ae2c355e3a12452f50eada9746a75911a Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 6 Oct 2023 00:07:51 -0400 Subject: [PATCH 1/2] Add source_text() test with mulitbyte char --- tests/test.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test.rs b/tests/test.rs index 8e47b46..a1ea72a 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -325,6 +325,15 @@ fn literal_span() { assert!(positive.subspan(1..4).is_none()); } +#[cfg(span_locations)] +#[test] +fn source_text() { + let input = " 𓀕 "; + let tokens = input.parse::().unwrap(); + let ident = tokens.into_iter().next().unwrap(); + assert_eq!("𓀕", ident.span().source_text().unwrap()); // FIXME +} + #[test] fn roundtrip() { fn roundtrip(p: &str) { From 7f5533d6cc9ff783d174aec4e3be2caa202b62ca Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Fri, 6 Oct 2023 00:04:32 -0400 Subject: [PATCH 2/2] Account for multibyte chars in source_text() computation --- src/fallback.rs | 9 +++++++-- tests/test.rs | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/fallback.rs b/src/fallback.rs index db52959..352b459 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -364,8 +364,13 @@ impl FileInfo { fn source_text(&self, span: Span) -> String { let lo = (span.lo - self.span.lo) as usize; - let hi = (span.hi - self.span.lo) as usize; - self.source_text[lo..hi].to_owned() + let trunc_lo = &self.source_text[lo..]; + let char_len = (span.hi - span.lo) as usize; + let source_text = match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => &trunc_lo[..offset], + None => trunc_lo, + }; + source_text.to_owned() } } diff --git a/tests/test.rs b/tests/test.rs index a1ea72a..4a6ca75 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -331,7 +331,7 @@ fn source_text() { let input = " 𓀕 "; let tokens = input.parse::().unwrap(); let ident = tokens.into_iter().next().unwrap(); - assert_eq!("𓀕", ident.span().source_text().unwrap()); // FIXME + assert_eq!("𓀕", ident.span().source_text().unwrap()); } #[test]