[WIP] make nightly compilers able to parallelize

compiler/rustc_codegen_ssa/src/base.rs

@@ -17,10 +17,7 @@ use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_attr as attr;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
-
-use rustc_data_structures::sync::par_iter;
-#[cfg(parallel_compiler)]
-use rustc_data_structures::sync::ParallelIterator;
+use rustc_data_structures::sync::par_map;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::lang_items::LangItem;
@@ -682,7 +679,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
     // This likely is a temporary measure. Once we don't have to support the
     // non-parallel compiler anymore, we can compile CGUs end-to-end in
     // parallel and get rid of the complicated scheduling logic.
-    let mut pre_compiled_cgus = if cfg!(parallel_compiler) {
+    let mut pre_compiled_cgus = if rustc_data_structures::sync::active() {
         tcx.sess.time("compile_first_CGU_batch", || {
             // Try to find one CGU to compile per thread.
             let cgus: Vec<_> = cgu_reuse
@@ -695,12 +692,10 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
             // Compile the found CGUs in parallel.
             let start_time = Instant::now();
 
-            let pre_compiled_cgus = par_iter(cgus)
-                .map(|(i, _)| {
-                    let module = backend.compile_codegen_unit(tcx, codegen_units[i].name());
-                    (i, module)
-                })
-                .collect();
+            let pre_compiled_cgus = par_map(cgus, |(i, _)| {
+                let module = backend.compile_codegen_unit(tcx, codegen_units[i].name());
+                (i, module)
+            });
 
             total_codegen_time += start_time.elapsed();
 

compiler/rustc_data_structures/Cargo.toml

@@ -10,11 +10,11 @@ arrayvec = { version = "0.7", default-features = false }
 bitflags = "1.2.1"
 cfg-if = "1.0"
 ena = "0.14.2"
-indexmap = { version = "1.9.3" }
+indexmap = { version = "1.9.3", features = ["rustc-rayon"] }
 jobserver_crate = { version = "0.1.13", package = "jobserver" }
 libc = "0.2"
 measureme = "10.0.0"
-rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-rayon-core = { version = "0.5.0" }
 rustc-rayon = { version = "0.5.0", optional = true }
 rustc_graphviz = { path = "../rustc_graphviz" }
 rustc-hash = "1.1.0"
@@ -51,4 +51,4 @@ features = [
 memmap2 = "0.2.1"
 
 [features]
-rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon", "rustc-rayon-core"]
+rustc_use_parallel_compiler = ["rustc-rayon"]

compiler/rustc_data_structures/src/sharded.rs

@@ -1,5 +1,5 @@
 use crate::fx::{FxHashMap, FxHasher};
-use crate::sync::{Lock, LockGuard};
+use crate::sync::{active, Lock, LockGuard};
 use std::borrow::Borrow;
 use std::collections::hash_map::RawEntryMut;
 use std::hash::{Hash, Hasher};
@@ -9,20 +9,17 @@ use std::mem;
 #[cfg_attr(parallel_compiler, repr(align(64)))]
 struct CacheAligned<T>(T);
 
-#[cfg(parallel_compiler)]
 // 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700,
 // but this should be tested on higher core count CPUs. How the `Sharded` type gets used
 // may also affect the ideal number of shards.
 const SHARD_BITS: usize = 5;
 
-#[cfg(not(parallel_compiler))]
-const SHARD_BITS: usize = 0;
-
 pub const SHARDS: usize = 1 << SHARD_BITS;
 
 /// An array of cache-line aligned inner locked structures with convenience methods.
 pub struct Sharded<T> {
     shards: [CacheAligned<Lock<T>>; SHARDS],
+    single_thread: bool,
 }
 
 impl<T: Default> Default for Sharded<T> {
@@ -35,31 +32,41 @@ impl<T: Default> Default for Sharded<T> {
 impl<T> Sharded<T> {
     #[inline]
     pub fn new(mut value: impl FnMut() -> T) -> Self {
-        Sharded { shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))) }
+        Sharded {
+            shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))),
+            single_thread: !active(),
+        }
     }
 
     /// The shard is selected by hashing `val` with `FxHasher`.
     #[inline]
     pub fn get_shard_by_value<K: Hash + ?Sized>(&self, val: &K) -> &Lock<T> {
-        if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) }
+        if self.single_thread { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) }
     }
 
     #[inline]
     pub fn get_shard_by_hash(&self, hash: u64) -> &Lock<T> {
-        &self.shards[get_shard_index_by_hash(hash)].0
-    }
-
-    #[inline]
-    pub fn get_shard_by_index(&self, i: usize) -> &Lock<T> {
-        &self.shards[i].0
+        if self.single_thread {
+            &self.shards[0].0
+        } else {
+            &self.shards[get_shard_index_by_hash(hash)].0
+        }
     }
 
     pub fn lock_shards(&self) -> Vec<LockGuard<'_, T>> {
-        (0..SHARDS).map(|i| self.shards[i].0.lock()).collect()
+        if self.single_thread {
+            vec![self.shards[0].0.lock()]
+        } else {
+            (0..SHARDS).map(|i| self.shards[i].0.lock()).collect()
+        }
     }
 
     pub fn try_lock_shards(&self) -> Option<Vec<LockGuard<'_, T>>> {
-        (0..SHARDS).map(|i| self.shards[i].0.try_lock()).collect()
+        if self.single_thread {
+            Some(vec![self.shards[0].0.try_lock()?])
+        } else {
+            (0..SHARDS).map(|i| self.shards[i].0.try_lock()).collect()
+        }
     }
 }
 
@@ -141,7 +148,7 @@ pub fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 {
 /// consistently for each `Sharded` instance.
 #[inline]
 #[allow(clippy::modulo_one)]
-pub fn get_shard_index_by_hash(hash: u64) -> usize {
+fn get_shard_index_by_hash(hash: u64) -> usize {
     let hash_len = mem::size_of::<usize>();
     // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits.
     // hashbrown also uses the lowest bits, so we can't use those