Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rewrite LLVM's archive writer in Rust #97485

Merged
merged 4 commits into from Dec 3, 2022
Merged

Rewrite LLVM's archive writer in Rust #97485

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
be67084
Rewrite LLVM's archive writer in Rust
bjorn3 May 28, 2022
0673cde
Use LLVM for getting symbols from COFF bigobj files
bjorn3 Nov 26, 2022
e1edc13
Write to temp file before renaming to the final name
bjorn3 Dec 2, 2022
a99838a
Make sure all input archives are unmapped before persisting the outpu…
bjorn3 Dec 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cargo.lock
View file
Open in desktop
Expand Up @@ -92,6 +92,15 @@ version = "1.0.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"

[[package]]
name = "ar_archive_writer"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "276881980556fdadeb88aa1ffc667e4d2e8fe72531dfabcb7a82bb3c9ea9ba31"
dependencies = [
"object",
]

[[package]]
name = "array_tool"
version = "1.0.3"
Expand Down Expand Up @@ -3413,6 +3422,7 @@ dependencies = [
name = "rustc_codegen_ssa"
version = "0.0.0"
dependencies = [
"ar_archive_writer",
"bitflags",
"cc",
"itertools",
Expand Down
15 changes: 15 additions & 0 deletions LICENSES/LLVM-exception.txt
View file
Open in desktop
@@ -0,0 +1,15 @@
---- LLVM Exceptions to the Apache 2.0 License ----

As an exception, if, as a result of your compiling your source code, portions
of this Software are embedded into an Object form of such source code, you
may redistribute such embedded portions in such Object form without complying
with the conditions of Sections 4(a), 4(b) and 4(d) of the License.

In addition, if you combine or link compiled forms of this Software with
software that is licensed under the GPLv2 ("Combined Software") and if a
court of competent jurisdiction determines that the patent provision (Section
3), the indemnity provision (Section 9) or other Section of the License
conflicts with the conditions of the GPLv2, you may retroactively and
prospectively choose to deem waived or otherwise exclude such Section(s) of
the License, but only in their entirety and only with respect to the Combined
Software.
6 changes: 0 additions & 6 deletions compiler/rustc_codegen_cranelift/Cargo.lock
View file
Open in desktop
Expand Up @@ -19,11 +19,6 @@ version = "1.0.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c794e162a5eff65c72ef524dfe393eb923c354e350bb78b9c7383df13f3bc142"

[[package]]
name = "ar"
version = "0.8.0"
source = "git+https://github.com/bjorn3/rust-ar.git?branch=do_not_remove_cg_clif_ranlib#de9ab0e56bf3a208381d342aa5b60f9ff2891648"

[[package]]
name = "arrayvec"
version = "0.7.2"
Expand Down Expand Up @@ -324,7 +319,6 @@ dependencies = [
name = "rustc_codegen_cranelift"
version = "0.1.0"
dependencies = [
"ar",
"cranelift-codegen",
"cranelift-frontend",
"cranelift-jit",
Expand Down
1 change: 0 additions & 1 deletion compiler/rustc_codegen_cranelift/Cargo.toml
View file
Open in desktop
Expand Up @@ -18,7 +18,6 @@ target-lexicon = "0.12.0"
gimli = { version = "0.26.0", default-features = false, features = ["write"]}
object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
indexmap = "1.9.1"
libloading = { version = "0.7.3", optional = true }
once_cell = "1.10.0"
Expand Down
224 changes: 5 additions & 219 deletions compiler/rustc_codegen_cranelift/src/archive.rs
View file
Open in desktop
@@ -1,35 +1,15 @@
//! Creation of ar archives like for the lib and staticlib crate type

use std::collections::BTreeMap;
use std::fs::File;
use std::io::{self, Read, Seek};
use std::path::{Path, PathBuf};

use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
use rustc_codegen_ssa::back::archive::{
get_native_object_symbols, ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder,
};
use rustc_session::Session;

use object::read::archive::ArchiveFile;
use object::{Object, ObjectSymbol, ReadCache};

#[derive(Debug)]
enum ArchiveEntry {
FromArchive { archive_index: usize, file_range: (u64, u64) },
File(PathBuf),
}

pub(crate) struct ArArchiveBuilderBuilder;

impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
Box::new(ArArchiveBuilder {
sess,
use_gnu_style_archive: sess.target.archive_format == "gnu",
// FIXME fix builtin ranlib on macOS
no_builtin_ranlib: sess.target.is_like_osx,

src_archives: vec![],
entries: vec![],
})
Box::new(ArArchiveBuilder::new(sess, get_native_object_symbols))
}

fn create_dll_import_lib(
Expand All @@ -40,200 +20,6 @@ impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
_tmpdir: &Path,
_is_direct_dependency: bool,
) -> PathBuf {
bug!("creating dll imports is not supported");
}
}

pub(crate) struct ArArchiveBuilder<'a> {
sess: &'a Session,
use_gnu_style_archive: bool,
no_builtin_ranlib: bool,

src_archives: Vec<File>,
// Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
// the end of an archive for linkers to not get confused.
entries: Vec<(Vec<u8>, ArchiveEntry)>,
}

impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
fn add_file(&mut self, file: &Path) {
self.entries.push((
file.file_name().unwrap().to_str().unwrap().to_string().into_bytes(),
ArchiveEntry::File(file.to_owned()),
));
}

fn add_archive(
&mut self,
archive_path: &Path,
mut skip: Box<dyn FnMut(&str) -> bool + 'static>,
) -> std::io::Result<()> {
let read_cache = ReadCache::new(std::fs::File::open(&archive_path)?);
let archive = ArchiveFile::parse(&read_cache).unwrap();
let archive_index = self.src_archives.len();

for entry in archive.members() {
let entry = entry.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
let file_name = String::from_utf8(entry.name().to_vec())
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
if !skip(&file_name) {
self.entries.push((
file_name.into_bytes(),
ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() },
));
}
}

self.src_archives.push(read_cache.into_inner());
Ok(())
}

fn build(mut self: Box<Self>, output: &Path) -> bool {
enum BuilderKind {
Bsd(ar::Builder<File>),
Gnu(ar::GnuBuilder<File>),
}

let sess = self.sess;

let mut symbol_table = BTreeMap::new();

let mut entries = Vec::new();

for (mut entry_name, entry) in self.entries {
// FIXME only read the symbol table of the object files to avoid having to keep all
// object files in memory at once, or read them twice.
let data = match entry {
ArchiveEntry::FromArchive { archive_index, file_range } => {
// FIXME read symbols from symtab
let src_read_cache = &mut self.src_archives[archive_index];

src_read_cache.seek(io::SeekFrom::Start(file_range.0)).unwrap();
let mut data = std::vec::from_elem(0, usize::try_from(file_range.1).unwrap());
src_read_cache.read_exact(&mut data).unwrap();

data
}
ArchiveEntry::File(file) => std::fs::read(file).unwrap_or_else(|err| {
sess.fatal(&format!(
"error while reading object file during archive building: {}",
err
));
}),
};

if !self.no_builtin_ranlib {
if symbol_table.contains_key(&entry_name) {
// The ar crate can't handle creating a symbol table in case of multiple archive
// members with the same name. Work around this by prepending a number until we
// get a unique name.
for i in 1.. {
let new_name = format!("{}_", i)
.into_bytes()
.into_iter()
.chain(entry_name.iter().copied())
.collect::<Vec<_>>();
if !symbol_table.contains_key(&new_name) {
entry_name = new_name;
break;
}
}
}

match object::File::parse(&*data) {
Ok(object) => {
symbol_table.insert(
entry_name.to_vec(),
object
.symbols()
.filter_map(|symbol| {
if symbol.is_undefined() || symbol.is_local() {
None
} else {
symbol.name().map(|name| name.as_bytes().to_vec()).ok()
}
})
.collect::<Vec<_>>(),
);
}
Err(err) => {
let err = err.to_string();
if err == "Unknown file magic" {
// Not an object file; skip it.
} else if object::read::archive::ArchiveFile::parse(&*data).is_ok() {
// Nested archive file; skip it.
} else {
sess.fatal(&format!(
"error parsing `{}` during archive creation: {}",
String::from_utf8_lossy(&entry_name),
err
));
}
}
}
}

entries.push((entry_name, data));
}

let mut builder = if self.use_gnu_style_archive {
BuilderKind::Gnu(
ar::GnuBuilder::new(
File::create(output).unwrap_or_else(|err| {
sess.fatal(&format!(
"error opening destination during archive building: {}",
err
));
}),
entries.iter().map(|(name, _)| name.clone()).collect(),
ar::GnuSymbolTableFormat::Size32,
symbol_table,
)
.unwrap(),
)
} else {
BuilderKind::Bsd(
ar::Builder::new(
File::create(output).unwrap_or_else(|err| {
sess.fatal(&format!(
"error opening destination during archive building: {}",
err
));
}),
symbol_table,
)
.unwrap(),
)
};

let any_members = !entries.is_empty();

// Add all files
for (entry_name, data) in entries.into_iter() {
let header = ar::Header::new(entry_name, data.len() as u64);
match builder {
BuilderKind::Bsd(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
BuilderKind::Gnu(ref mut builder) => builder.append(&header, &mut &*data).unwrap(),
}
}

// Finalize archive
std::mem::drop(builder);

if self.no_builtin_ranlib {
let ranlib = crate::toolchain::get_toolchain_binary(self.sess, "ranlib");

// Run ranlib to be able to link the archive
let status = std::process::Command::new(ranlib)
.arg(output)
.status()
.expect("Couldn't run ranlib");

if !status.success() {
self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
}
}

any_members
unimplemented!("creating dll imports is not yet supported");
}
}
14 changes: 0 additions & 14 deletions compiler/rustc_codegen_gcc/Cargo.lock
View file
Open in desktop
Expand Up @@ -11,12 +11,6 @@ dependencies = [
"memchr",
]

[[package]]
name = "ar"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "450575f58f7bee32816abbff470cbc47797397c2a81e0eaced4b98436daf52e1"

[[package]]
name = "bitflags"
version = "1.3.2"
Expand Down Expand Up @@ -212,10 +206,8 @@ dependencies = [
name = "rustc_codegen_gcc"
version = "0.1.0"
dependencies = [
"ar",
"gccjit",
"lang_tester",
"target-lexicon",
"tempfile",
]

Expand All @@ -228,12 +220,6 @@ dependencies = [
"winapi-util",
]

[[package]]
name = "target-lexicon"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d"

[[package]]
name = "tempfile"
version = "3.2.0"
Expand Down
4 changes: 0 additions & 4 deletions compiler/rustc_codegen_gcc/Cargo.toml
View file
Open in desktop
Expand Up @@ -27,10 +27,6 @@ gccjit = { git = "https://github.com/antoyo/gccjit.rs" }
# Local copy.
#gccjit = { path = "../gccjit.rs" }

target-lexicon = "0.10.0"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This dependency was already unused before this PR.


ar = "0.8.0"

[dev-dependencies]
lang_tester = "0.3.9"
tempfile = "3.1.0"
Expand Down