fix(span): track unnormalized source len for dep-info

Add `unnormalized_source_len` field to track the byte length
of source files before normalization (the original length).

`unnormalized_source_len` is for writing the correct file length
to dep-info for `-Zchecksum-hash-algorithm`

Author: weihanglo

compiler/rustc_interface/src/passes.rs

@@ -596,7 +596,9 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
             .map(|fmap| {
                 (
                     escape_dep_filename(&fmap.name.prefer_local().to_string()),
-                    fmap.normalized_source_len.0 as u64,
+                    // This needs to be unnormalized,
+                    // as external tools wouldn't know how rustc normalizes them
+                    fmap.unnormalized_source_len as u64,
                     fmap.checksum_hash,
                 )
             })

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1745,6 +1745,7 @@ impl<'a> CrateMetadataRef<'a> {
                     checksum_hash,
                     start_pos: original_start_pos,
                     normalized_source_len,
+                    unnormalized_source_len,
                     lines,
                     multibyte_chars,
                     normalized_pos,
@@ -1805,6 +1806,7 @@ impl<'a> CrateMetadataRef<'a> {
                     checksum_hash,
                     stable_id,
                     normalized_source_len.to_u32(),
+                    unnormalized_source_len,
                     self.cnum,
                     lines,
                     multibyte_chars,

compiler/rustc_query_system/src/ich/impls_syntax.rs

@@ -55,6 +55,7 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
             external_src: _,
             start_pos: _,
             normalized_source_len: _,
+            unnormalized_source_len: _,
             lines: _,
             ref multibyte_chars,
             ref normalized_pos,

compiler/rustc_span/src/lib.rs

@@ -1725,6 +1725,8 @@ pub struct SourceFile {
     pub start_pos: BytePos,
     /// The byte length of this source after normalization.
     pub normalized_source_len: RelativeBytePos,
+    /// The byte length of this source before normalization.
+    pub unnormalized_source_len: u32,
     /// Locations of lines beginnings in the source code.
     pub lines: FreezeLock<SourceFileLines>,
     /// Locations of multi-byte characters in the source code.
@@ -1749,6 +1751,7 @@ impl Clone for SourceFile {
             external_src: self.external_src.clone(),
             start_pos: self.start_pos,
             normalized_source_len: self.normalized_source_len,
+            unnormalized_source_len: self.unnormalized_source_len,
             lines: self.lines.clone(),
             multibyte_chars: self.multibyte_chars.clone(),
             normalized_pos: self.normalized_pos.clone(),
@@ -1765,6 +1768,7 @@ impl<S: SpanEncoder> Encodable<S> for SourceFile {
         self.checksum_hash.encode(s);
         // Do not encode `start_pos` as it's global state for this session.
         self.normalized_source_len.encode(s);
+        self.unnormalized_source_len.encode(s);
 
         // We are always in `Lines` form by the time we reach here.
         assert!(self.lines.read().is_lines());
@@ -1838,6 +1842,7 @@ impl<D: SpanDecoder> Decodable<D> for SourceFile {
         let src_hash: SourceFileHash = Decodable::decode(d);
         let checksum_hash: Option<SourceFileHash> = Decodable::decode(d);
         let normalized_source_len: RelativeBytePos = Decodable::decode(d);
+        let unnormalized_source_len = Decodable::decode(d);
         let lines = {
             let num_lines: u32 = Decodable::decode(d);
             if num_lines > 0 {
@@ -1860,6 +1865,7 @@ impl<D: SpanDecoder> Decodable<D> for SourceFile {
             name,
             start_pos: BytePos::from_u32(0),
             normalized_source_len,
+            unnormalized_source_len,
             src: None,
             src_hash,
             checksum_hash,
@@ -1959,6 +1965,12 @@ impl SourceFile {
                 SourceFileHash::new_in_memory(checksum_hash_kind, src.as_bytes())
             }
         });
+        // Capture the original source length before normalization.
+        let unnormalized_source_len = u32::try_from(src.len()).map_err(|_| OffsetOverflowError)?;
+        if unnormalized_source_len > Self::MAX_FILE_SIZE {
+            return Err(OffsetOverflowError);
+        }
+
         let normalized_pos = normalize_src(&mut src);
 
         let stable_id = StableSourceFileId::from_filename_in_current_crate(&name);
@@ -1977,6 +1989,7 @@ impl SourceFile {
             external_src: FreezeLock::frozen(ExternalSource::Unneeded),
             start_pos: BytePos::from_u32(0),
             normalized_source_len: RelativeBytePos::from_u32(normalized_source_len),
+            unnormalized_source_len,
             lines: FreezeLock::frozen(SourceFileLines::Lines(lines)),
             multibyte_chars,
             normalized_pos,

compiler/rustc_span/src/source_map.rs

@@ -354,6 +354,7 @@ impl SourceMap {
         checksum_hash: Option<SourceFileHash>,
         stable_id: StableSourceFileId,
         normalized_source_len: u32,
+        unnormalized_source_len: u32,
         cnum: CrateNum,
         file_local_lines: FreezeLock<SourceFileLines>,
         multibyte_chars: Vec<MultiByteChar>,
@@ -373,6 +374,7 @@ impl SourceMap {
             }),
             start_pos: BytePos(0),
             normalized_source_len,
+            unnormalized_source_len,
             lines: file_local_lines,
             multibyte_chars,
             normalized_pos,

compiler/rustc_span/src/source_map/tests.rs

@@ -231,6 +231,7 @@ fn t10() {
         src_hash,
         checksum_hash,
         normalized_source_len,
+        unnormalized_source_len,
         lines,
         multibyte_chars,
         normalized_pos,
@@ -244,6 +245,7 @@ fn t10() {
         checksum_hash,
         stable_id,
         normalized_source_len.to_u32(),
+        unnormalized_source_len,
         CrateNum::ZERO,
         FreezeLock::new(lines.read().clone()),
         multibyte_chars,

compiler/rustc_span/src/tests.rs

@@ -103,3 +103,17 @@ fn test_trim() {
 
     assert_eq!(span(well_before, before).trim_start(other), None);
 }
+
+#[test]
+fn test_unnormalized_source_length() {
+    let source = "\u{feff}hello\r\nferries\r\n".to_owned();
+    let sf = SourceFile::new(
+        FileName::Anon(Hash64::ZERO),
+        source,
+        SourceFileHashAlgorithm::Sha256,
+        Some(SourceFileHashAlgorithm::Sha256),
+    )
+    .unwrap();
+    assert_eq!(sf.unnormalized_source_len, 19);
+    assert_eq!(sf.normalized_source_len.0, 14);
+}