Skip to content

Commit

Permalink
run optimization and codegen on worker threads
Browse files Browse the repository at this point in the history 
Refactor the code in `llvm::back` that invokes LLVM optimization and codegen
passes so that it can be called from worker threads.  (Previously, it used
`&Session` extensively, and `Session` is not `Share`.)  The new code can handle
multiple compilation units, by compiling each unit to `crate.0.o`, `crate.1.o`,
etc., and linking together all the `crate.N.o` files into a single `crate.o`
using `ld -r`.  The later linking steps can then be run unchanged.

The new code preserves the behavior of `--emit`/`-o` when building a single
compilation unit.  With multiple compilation units, the `--emit=asm/ir/bc`
options produce multiple files, so combinations like `--emit=ir -o foo.ll` will
not actually produce `foo.ll` (they instead produce several `foo.N.ll` files).

The new code supports `-Z lto` only when using a single compilation unit.
Compiling with multiple compilation units and `-Z lto` will produce an error.
(I can't think of any good reason to do such a thing.)  Linking with `-Z lto`
against a library that was built as multiple compilation units will also fail,
because the rlib does not contain a `crate.bytecode.deflate` file.  This could
be supported in the future by linking together the `crate.N.bc` files produced
when compiling the library into a single `crate.bc`, or by making the LTO code
support multiple `crate.N.bytecode.deflate` files.
  • Loading branch information
@spernsteiner
spernsteiner committed Sep 5, 2014
1 parent e29aa14 commit cf67285
Show file tree
Hide file tree
Showing 10 changed files with 773 additions and 285 deletions.
13 changes: 4 additions & 9 deletions src/compiletest/runtest.rs
View file
Expand Up @@ -1577,10 +1577,6 @@ fn _arm_push_aux_shared_library(config: &Config, testfile: &Path) {

// codegen tests (vs. clang)

fn make_o_name(config: &Config, testfile: &Path) -> Path {
output_base_name(config, testfile).with_extension("o")
}

fn append_suffix_to_stem(p: &Path, suffix: &str) -> Path {
if suffix.len() == 0 {
(*p).clone()
Expand All @@ -1596,14 +1592,13 @@ fn compile_test_and_save_bitcode(config: &Config, props: &TestProps,
// FIXME (#9639): This needs to handle non-utf8 paths
let link_args = vec!("-L".to_string(),
aux_dir.as_str().unwrap().to_string());
let llvm_args = vec!("--emit=obj".to_string(),
"--crate-type=lib".to_string(),
"-C".to_string(),
"save-temps".to_string());
let llvm_args = vec!("--emit=bc,obj".to_string(),
"--crate-type=lib".to_string());
let args = make_compile_args(config,
props,
link_args.append(llvm_args.as_slice()),
|a, b| ThisFile(make_o_name(a, b)), testfile);
|a, b| ThisDirectory(output_base_name(a, b).dir_path()),
testfile);
compose_and_run_compiler(config, props, testfile, args, None)
}

Expand Down
87 changes: 46 additions & 41 deletions src/librustc/back/link.rs
View file
Expand Up @@ -662,51 +662,56 @@ fn link_rlib<'a>(sess: &'a Session,
ab.add_file(&metadata).unwrap();
remove(sess, &metadata);

// For LTO purposes, the bytecode of this library is also inserted
// into the archive.
//
// Note that we make sure that the bytecode filename in the archive
// is never exactly 16 bytes long by adding a 16 byte extension to
// it. This is to work around a bug in LLDB that would cause it to
// crash if the name of a file in an archive was exactly 16 bytes.
let bc_filename = obj_filename.with_extension("bc");
let bc_deflated_filename = obj_filename.with_extension("bytecode.deflate");

let bc_data = match fs::File::open(&bc_filename).read_to_end() {
Ok(buffer) => buffer,
Err(e) => sess.fatal(format!("failed to read bytecode: {}",
e).as_slice())
};
if sess.opts.cg.codegen_units == 1 {
// For LTO purposes, the bytecode of this library is also
// inserted into the archive. We currently do this only when
// codegen_units == 1, so we don't have to deal with multiple
// bitcode files per crate.
//
// Note that we make sure that the bytecode filename in the
// archive is never exactly 16 bytes long by adding a 16 byte
// extension to it. This is to work around a bug in LLDB that
// would cause it to crash if the name of a file in an archive
// was exactly 16 bytes.
let bc_filename = obj_filename.with_extension("bc");
let bc_deflated_filename = obj_filename.with_extension("bytecode.deflate");

let bc_data = match fs::File::open(&bc_filename).read_to_end() {
Ok(buffer) => buffer,
Err(e) => sess.fatal(format!("failed to read bytecode: {}",
e).as_slice())
};

let bc_data_deflated = match flate::deflate_bytes(bc_data.as_slice()) {
Some(compressed) => compressed,
None => sess.fatal(format!("failed to compress bytecode from {}",
bc_filename.display()).as_slice())
};
let bc_data_deflated = match flate::deflate_bytes(bc_data.as_slice()) {
Some(compressed) => compressed,
None => sess.fatal(format!("failed to compress bytecode from {}",
bc_filename.display()).as_slice())
};

let mut bc_file_deflated = match fs::File::create(&bc_deflated_filename) {
Ok(file) => file,
Err(e) => {
sess.fatal(format!("failed to create compressed bytecode \
file: {}", e).as_slice())
}
};
let mut bc_file_deflated = match fs::File::create(&bc_deflated_filename) {
Ok(file) => file,
Err(e) => {
sess.fatal(format!("failed to create compressed bytecode \
file: {}", e).as_slice())
}
};

match write_rlib_bytecode_object_v1(&mut bc_file_deflated,
bc_data_deflated.as_slice()) {
Ok(()) => {}
Err(e) => {
sess.err(format!("failed to write compressed bytecode: \
{}", e).as_slice());
sess.abort_if_errors()
}
};
match write_rlib_bytecode_object_v1(&mut bc_file_deflated,
bc_data_deflated.as_slice()) {
Ok(()) => {}
Err(e) => {
sess.err(format!("failed to write compressed bytecode: \
{}", e).as_slice());
sess.abort_if_errors()
}
};

ab.add_file(&bc_deflated_filename).unwrap();
remove(sess, &bc_deflated_filename);
if !sess.opts.cg.save_temps &&
!sess.opts.output_types.contains(&OutputTypeBitcode) {
remove(sess, &bc_filename);
ab.add_file(&bc_deflated_filename).unwrap();
remove(sess, &bc_deflated_filename);
if !sess.opts.cg.save_temps &&
!sess.opts.output_types.contains(&OutputTypeBitcode) {
remove(sess, &bc_filename);
}
}
}

Expand Down
11 changes: 9 additions & 2 deletions src/librustc/back/lto.rs
View file
Expand Up @@ -67,7 +67,14 @@ pub fn run(sess: &session::Session, llmod: ModuleRef,
archive.read(format!("{}.bytecode.deflate",
file).as_slice())
});
let bc_encoded = bc_encoded.expect("missing compressed bytecode in archive!");
let bc_encoded = match bc_encoded {
Some(data) => data,
None => {
sess.fatal(format!("missing compressed bytecode in {} \
(perhaps it was compiled with -C codegen-units > 1)",
path.display()).as_slice());
},
};
let bc_extractor = if is_versioned_bytecode_format(bc_encoded) {
|_| {
// Read the version
Expand Down Expand Up @@ -120,7 +127,7 @@ pub fn run(sess: &session::Session, llmod: ModuleRef,
if !llvm::LLVMRustLinkInExternalBitcode(llmod,
ptr as *const libc::c_char,
bc_decoded.len() as libc::size_t) {
write::llvm_err(sess,
write::llvm_err(sess.diagnostic().handler(),
format!("failed to load bc of `{}`",
name.as_slice()));
}
Expand Down

0 comments on commit cf67285

Please sign in to comment.