Skip to content

Commit

Permalink
Call FileEncoder::finish in rmeta encoding
Browse files Browse the repository at this point in the history
  • Loading branch information
saethlin committed Oct 28, 2023
1 parent cf226e9 commit 80648b8
Show file tree
Hide file tree
Showing 14 changed files with 87 additions and 45 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -4069,6 +4069,7 @@ dependencies = [
"rustc_query_impl",
"rustc_query_system",
"rustc_resolve",
"rustc_serialize",
"rustc_session",
"rustc_span",
"rustc_symbol_mangling",
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_ssa/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ impl CodegenResults {
encoder.emit_raw_bytes(&RLINK_VERSION.to_be_bytes());
encoder.emit_str(sess.cfg_version);
Encodable::encode(codegen_results, &mut encoder);
encoder.finish()
encoder.finish().map_err(|(_path, err)| err)
}

pub fn deserialize_rlink(sess: &Session, data: Vec<u8>) -> Result<Self, CodegenErrors> {
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_incremental/src/persist/file_format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ where
);
debug!("save: data written to disk successfully");
}
Err(err) => {
sess.emit_err(errors::WriteNew { name, path: path_buf, err });
Err((path, err)) => {
sess.emit_err(errors::WriteNew { name, path, err });
}
}
}
Expand Down
3 changes: 0 additions & 3 deletions compiler/rustc_incremental/src/persist/save.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
join(
move || {
sess.time("incr_comp_persist_dep_graph", || {
if let Err(err) = tcx.dep_graph.encode(&tcx.sess.prof) {
sess.emit_err(errors::WriteDepGraph { path: &staging_dep_graph_path, err });
}
if let Err(err) = fs::rename(&staging_dep_graph_path, &dep_graph_path) {
sess.emit_err(errors::MoveDepGraph {
from: &staging_dep_graph_path,
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_interface/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ rustc_fs_util = { path = "../rustc_fs_util" }
rustc_macros = { path = "../rustc_macros" }
rustc_parse = { path = "../rustc_parse" }
rustc_session = { path = "../rustc_session" }
rustc_serialize = { path = "../rustc_serialize" }
rustc_span = { path = "../rustc_span" }
rustc_middle = { path = "../rustc_middle" }
rustc_ast_lowering = { path = "../rustc_ast_lowering" }
Expand Down
11 changes: 10 additions & 1 deletion compiler/rustc_interface/src/queries.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::errors::{FailedWritingFile, RustcErrorFatal, RustcErrorUnexpectedAnnotation};
use crate::interface::{Compiler, Result};
use crate::{passes, util};
use crate::{errors, passes, util};

use rustc_ast as ast;
use rustc_codegen_ssa::traits::CodegenBackend;
Expand All @@ -15,6 +15,7 @@ use rustc_metadata::creader::CStore;
use rustc_middle::arena::Arena;
use rustc_middle::dep_graph::DepGraph;
use rustc_middle::ty::{GlobalCtxt, TyCtxt};
use rustc_serialize::opaque::FileEncodeResult;
use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
use rustc_session::cstore::Untracked;
use rustc_session::{output::find_crate_name, Session};
Expand Down Expand Up @@ -101,6 +102,10 @@ impl<'tcx> Queries<'tcx> {
}
}

pub fn finish(&self) -> FileEncodeResult {
if let Some(gcx) = self.gcx_cell.get() { gcx.finish() } else { Ok(0) }
}

fn session(&self) -> &Lrc<Session> {
&self.compiler.sess
}
Expand Down Expand Up @@ -347,6 +352,10 @@ impl Compiler {
.time("serialize_dep_graph", || gcx.enter(rustc_incremental::save_dep_graph));
}

if let Err((path, error)) = queries.finish() {
self.session().emit_err(errors::FailedWritingFile { path: &path, error });
}

_timer = Some(self.session().timer("free_global_ctxt"));

ret
Expand Down
5 changes: 1 addition & 4 deletions compiler/rustc_metadata/messages.ftl
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,8 @@ metadata_extern_location_not_file =
metadata_fail_create_file_encoder =
failed to create file encoder: {$err}
metadata_fail_seek_file =
failed to seek the file: {$err}
metadata_fail_write_file =
failed to write to the file: {$err}
failed to write to `{$path}`: {$err}
metadata_failed_copy_to_stdout =
failed to copy {$filename} to stdout: {$err}
Expand Down
9 changes: 2 additions & 7 deletions compiler/rustc_metadata/src/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -307,15 +307,10 @@ pub struct FailCreateFileEncoder {
pub err: Error,
}

#[derive(Diagnostic)]
#[diag(metadata_fail_seek_file)]
pub struct FailSeekFile {
pub err: Error,
}

#[derive(Diagnostic)]
#[diag(metadata_fail_write_file)]
pub struct FailWriteFile {
pub struct FailWriteFile<'a> {
pub path: &'a Path,
pub err: Error,
}

Expand Down
42 changes: 26 additions & 16 deletions compiler/rustc_metadata/src/rmeta/encoder.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::errors::{FailCreateFileEncoder, FailSeekFile, FailWriteFile};
use crate::errors::{FailCreateFileEncoder, FailWriteFile};
use crate::rmeta::def_path_hash_map::DefPathHashMapRef;
use crate::rmeta::table::TableBuilder;
use crate::rmeta::*;
Expand Down Expand Up @@ -41,6 +41,7 @@ use rustc_span::symbol::{sym, Symbol};
use rustc_span::{self, ExternalSource, FileName, SourceFile, Span, SpanData, SyntaxContext};
use std::borrow::Borrow;
use std::collections::hash_map::Entry;
use std::fs::File;
use std::hash::Hash;
use std::io::{Read, Seek, Write};
use std::num::NonZeroUsize;
Expand Down Expand Up @@ -2251,22 +2252,17 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
// culminating in the `CrateRoot` which points to all of it.
let root = ecx.encode_crate_root();

ecx.opaque.flush();

let mut file = ecx.opaque.file();
// We will return to this position after writing the root position.
let pos_before_seek = file.stream_position().unwrap();

// Encode the root position.
let header = METADATA_HEADER.len();
file.seek(std::io::SeekFrom::Start(header as u64))
.unwrap_or_else(|err| tcx.sess.emit_fatal(FailSeekFile { err }));
let pos = root.position.get();
file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8])
.unwrap_or_else(|err| tcx.sess.emit_fatal(FailWriteFile { err }));
// Make sure we report any errors from writing to the file.
// If we forget this, compilation can succeed with an incomplete rmeta file,
// causing an ICE when the rmeta file is read by another compilation.
if let Err((path, err)) = ecx.opaque.finish() {
tcx.sess.emit_err(FailWriteFile { path: &path, err });
}

// Return to the position where we are before writing the root position.
file.seek(std::io::SeekFrom::Start(pos_before_seek)).unwrap();
let file = ecx.opaque.file();
if let Err(err) = encode_root_position(file, root.position.get()) {
tcx.sess.emit_err(FailWriteFile { path: ecx.opaque.path(), err });
}

// Record metadata size for self-profiling
tcx.prof.artifact_size(
Expand All @@ -2276,6 +2272,20 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
);
}

fn encode_root_position(mut file: &File, pos: usize) -> Result<(), std::io::Error> {
// We will return to this position after writing the root position.
let pos_before_seek = file.stream_position().unwrap();

// Encode the root position.
let header = METADATA_HEADER.len();
file.seek(std::io::SeekFrom::Start(header as u64))?;
file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8])?;

// Return to the position where we are before writing the root position.
file.seek(std::io::SeekFrom::Start(pos_before_seek))?;
Ok(())
}

pub fn provide(providers: &mut Providers) {
*providers = Providers {
doc_link_resolutions: |tcx, def_id| {
Expand Down
3 changes: 1 addition & 2 deletions compiler/rustc_middle/src/query/on_disk_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ use rustc_span::source_map::{SourceMap, StableSourceFileId};
use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, RelativeBytePos, SourceFile, Span};
use rustc_span::{CachingSourceMapView, Symbol};
use std::collections::hash_map::Entry;
use std::io;
use std::mem;

const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE;
Expand Down Expand Up @@ -867,7 +866,7 @@ impl<'a, 'tcx> CacheEncoder<'a, 'tcx> {
}

#[inline]
fn finish(self) -> Result<usize, io::Error> {
fn finish(mut self) -> FileEncodeResult {
self.encoder.finish()
}
}
Expand Down
4 changes: 4 additions & 0 deletions compiler/rustc_middle/src/ty/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,10 @@ impl<'tcx> GlobalCtxt<'tcx> {
let icx = tls::ImplicitCtxt::new(self);
tls::enter_context(&icx, || f(icx.tcx))
}

pub fn finish(&self) -> FileEncodeResult {
self.dep_graph.finish_encoding(&self.sess.prof)
}
}

impl<'tcx> TyCtxt<'tcx> {
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_query_system/src/dep_graph/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -989,7 +989,7 @@ impl<D: Deps> DepGraph<D> {
}
}

pub fn encode(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
pub fn finish_encoding(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
if let Some(data) = &self.data {
data.current.encoder.steal().finish(profiler)
} else {
Expand Down
43 changes: 36 additions & 7 deletions compiler/rustc_serialize/src/opaque.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ use std::io::{self, Write};
use std::marker::PhantomData;
use std::ops::Range;
use std::path::Path;
use std::path::PathBuf;

// -----------------------------------------------------------------------------
// Encoder
// -----------------------------------------------------------------------------

pub type FileEncodeResult = Result<usize, io::Error>;
pub type FileEncodeResult = Result<usize, (PathBuf, io::Error)>;

/// The size of the buffer in `FileEncoder`.
const BUF_SIZE: usize = 8192;
Expand All @@ -34,21 +35,28 @@ pub struct FileEncoder {
// This is used to implement delayed error handling, as described in the
// comment on `trait Encoder`.
res: Result<(), io::Error>,
path: PathBuf,
#[cfg(debug_assertions)]
finished: bool,
}

impl FileEncoder {
pub fn new<P: AsRef<Path>>(path: P) -> io::Result<Self> {
// File::create opens the file for writing only. When -Zmeta-stats is enabled, the metadata
// encoder rewinds the file to inspect what was written. So we need to always open the file
// for reading and writing.
let file = File::options().read(true).write(true).create(true).truncate(true).open(path)?;
let file =
File::options().read(true).write(true).create(true).truncate(true).open(&path)?;

Ok(FileEncoder {
buf: vec![0u8; BUF_SIZE].into_boxed_slice().try_into().unwrap(),
path: path.as_ref().into(),
buffered: 0,
flushed: 0,
file,
res: Ok(()),
#[cfg(debug_assertions)]
finished: false,
})
}

Expand All @@ -63,6 +71,10 @@ impl FileEncoder {
#[cold]
#[inline(never)]
pub fn flush(&mut self) {
#[cfg(debug_assertions)]
{
self.finished = false;
}
if self.res.is_ok() {
self.res = self.file.write_all(&self.buf[..self.buffered]);
}
Expand All @@ -74,6 +86,10 @@ impl FileEncoder {
&self.file
}

pub fn path(&self) -> &Path {
&self.path
}

#[inline]
fn buffer_empty(&mut self) -> &mut [u8] {
// SAFETY: self.buffered is inbounds as an invariant of the type
Expand All @@ -97,6 +113,10 @@ impl FileEncoder {

#[inline]
fn write_all(&mut self, buf: &[u8]) {
#[cfg(debug_assertions)]
{
self.finished = false;
}
if let Some(dest) = self.buffer_empty().get_mut(..buf.len()) {
dest.copy_from_slice(buf);
self.buffered += buf.len();
Expand All @@ -121,6 +141,10 @@ impl FileEncoder {
/// with one instruction, so while this does in some sense do wasted work, we come out ahead.
#[inline]
pub fn write_with<const N: usize>(&mut self, visitor: impl FnOnce(&mut [u8; N]) -> usize) {
#[cfg(debug_assertions)]
{
self.finished = false;
}
let flush_threshold = const { BUF_SIZE.checked_sub(N).unwrap() };
if std::intrinsics::unlikely(self.buffered > flush_threshold) {
self.flush();
Expand Down Expand Up @@ -152,20 +176,25 @@ impl FileEncoder {
})
}

pub fn finish(mut self) -> Result<usize, io::Error> {
pub fn finish(&mut self) -> FileEncodeResult {
self.flush();
#[cfg(debug_assertions)]
{
self.finished = true;
}
match std::mem::replace(&mut self.res, Ok(())) {
Ok(()) => Ok(self.position()),
Err(e) => Err(e),
Err(e) => Err((self.path.clone(), e)),
}
}
}

#[cfg(debug_assertions)]
impl Drop for FileEncoder {
fn drop(&mut self) {
// Likely to be a no-op, because `finish` should have been called and
// it also flushes. But do it just in case.
self.flush();
if !std::thread::panicking() {
assert!(self.finished);
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/librustdoc/scrape_examples.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ pub(crate) fn run(
// Save output to provided path
let mut encoder = FileEncoder::new(options.output_path).map_err(|e| e.to_string())?;
calls.encode(&mut encoder);
encoder.finish().map_err(|e| e.to_string())?;
encoder.finish().map_err(|(_path, e)| e.to_string())?;

Ok(())
};
Expand Down

0 comments on commit 80648b8

Please sign in to comment.