Rollup of 8 pull requests

compiler/rustc_codegen_llvm/messages.ftl

@@ -1,4 +1,5 @@
 codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z autodiff=Enable
+codegen_llvm_autodiff_without_lto = using the autodiff feature requires setting `lto="fat"` in your Cargo.toml
 
 codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
 

compiler/rustc_codegen_llvm/src/errors.rs

@@ -32,6 +32,10 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
     }
 }
 
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_autodiff_without_lto)]
+pub(crate) struct AutoDiffWithoutLto;
+
 #[derive(Diagnostic)]
 #[diag(codegen_llvm_autodiff_without_enable)]
 pub(crate) struct AutoDiffWithoutEnable;

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -25,7 +25,7 @@ use crate::abi::FnAbiLlvmExt;
 use crate::builder::Builder;
 use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
 use crate::context::CodegenCx;
-use crate::errors::AutoDiffWithoutEnable;
+use crate::errors::{AutoDiffWithoutEnable, AutoDiffWithoutLto};
 use crate::llvm::{self, Metadata, Type, Value};
 use crate::type_of::LayoutLlvmExt;
 use crate::va_arg::emit_va_arg;
@@ -378,8 +378,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::ctpop
             | sym::bswap
             | sym::bitreverse
-            | sym::rotate_left
-            | sym::rotate_right
             | sym::saturating_add
             | sym::saturating_sub
             | sym::unchecked_funnel_shl
@@ -424,19 +422,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     sym::bitreverse => {
                         self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
                     }
-                    sym::rotate_left
-                    | sym::rotate_right
-                    | sym::unchecked_funnel_shl
-                    | sym::unchecked_funnel_shr => {
-                        let is_left = name == sym::rotate_left || name == sym::unchecked_funnel_shl;
+                    sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
+                        let is_left = name == sym::unchecked_funnel_shl;
                         let lhs = args[0].immediate();
-                        let (rhs, raw_shift) =
-                            if name == sym::rotate_left || name == sym::rotate_right {
-                                // rotate = funnel shift with first two args the same
-                                (lhs, args[1].immediate())
-                            } else {
-                                (args[1].immediate(), args[2].immediate())
-                            };
+                        let rhs = args[1].immediate();
+                        let raw_shift = args[2].immediate();
                         let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
 
                         // llvm expects shift to be the same type as the values, but rust
@@ -1146,6 +1136,9 @@ fn codegen_autodiff<'ll, 'tcx>(
     if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
         let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable);
     }
+    if tcx.sess.lto() != rustc_session::config::Lto::Fat {
+        let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutLto);
+    }
 
     let fn_args = instance.args;
     let callee_ty = instance.ty(tcx, bx.typing_env());

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -333,29 +333,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let r = self.read_immediate(&args[1])?;
                 self.exact_div(&l, &r, dest)?;
             }
-            sym::rotate_left | sym::rotate_right => {
-                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
-                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
-                let layout_val = self.layout_of(instance_args.type_at(0))?;
-                let val = self.read_scalar(&args[0])?;
-                let val_bits = val.to_bits(layout_val.size)?; // sign is ignored here
-
-                let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
-                let raw_shift = self.read_scalar(&args[1])?;
-                let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
-
-                let width_bits = u128::from(layout_val.size.bits());
-                let shift_bits = raw_shift_bits % width_bits;
-                let inv_shift_bits = (width_bits - shift_bits) % width_bits;
-                let result_bits = if intrinsic_name == sym::rotate_left {
-                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
-                } else {
-                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
-                };
-                let truncated_bits = layout_val.size.truncate(result_bits);
-                let result = Scalar::from_uint(truncated_bits, layout_val.size);
-                self.write_scalar(result, dest)?;
-            }
             sym::copy => {
                 self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
             }

compiler/rustc_session/src/session.rs

@@ -594,14 +594,6 @@ impl Session {
 
     /// Calculates the flavor of LTO to use for this compilation.
     pub fn lto(&self) -> config::Lto {
-        // Autodiff currently requires fat-lto to have access to the llvm-ir of all (indirectly) used functions and types.
-        // fat-lto is the easiest solution to this requirement, but quite expensive.
-        // FIXME(autodiff): Make autodiff also work with embed-bc instead of fat-lto.
-        // Don't apply fat-lto to proc-macro crates as they cannot use fat-lto without -Zdylib-lto
-        if self.opts.autodiff_enabled() && !self.opts.crate_types.contains(&CrateType::ProcMacro) {
-            return config::Lto::Fat;
-        }
-
         // If our target has codegen requirements ignore the command line
         if self.target.requires_lto {
             return config::Lto::Fat;

library/core/src/intrinsics/mod.rs

@@ -56,7 +56,7 @@
 
 use crate::ffi::va_list::{VaArgSafe, VaListImpl};
 use crate::marker::{ConstParamTy, Destruct, DiscriminantKind, PointeeSized, Tuple};
-use crate::ptr;
+use crate::{mem, ptr};
 
 mod bounds;
 pub mod fallback;
@@ -2013,7 +2013,14 @@ pub const unsafe fn unchecked_mul<T: Copy>(x: T, y: T) -> T;
 #[rustc_intrinsic_const_stable_indirect]
 #[rustc_nounwind]
 #[rustc_intrinsic]
-pub const fn rotate_left<T: Copy>(x: T, shift: u32) -> T;
+#[rustc_allow_const_fn_unstable(const_trait_impl, funnel_shifts)]
+#[miri::intrinsic_fallback_is_spec]
+pub const fn rotate_left<T: [const] fallback::FunnelShift>(x: T, shift: u32) -> T {
+    // Make sure to call the intrinsic for `funnel_shl`, not the fallback impl.
+    // SAFETY: we modulo `shift` so that the result is definitely less than the size of
+    // `T` in bits.
+    unsafe { unchecked_funnel_shl(x, x, shift % (mem::size_of::<T>() as u32 * 8)) }
+}
 
 /// Performs rotate right.
 ///
@@ -2028,7 +2035,14 @@ pub const fn rotate_left<T: Copy>(x: T, shift: u32) -> T;
 #[rustc_intrinsic_const_stable_indirect]
 #[rustc_nounwind]
 #[rustc_intrinsic]
-pub const fn rotate_right<T: Copy>(x: T, shift: u32) -> T;
+#[rustc_allow_const_fn_unstable(const_trait_impl, funnel_shifts)]
+#[miri::intrinsic_fallback_is_spec]
+pub const fn rotate_right<T: [const] fallback::FunnelShift>(x: T, shift: u32) -> T {
+    // Make sure to call the intrinsic for `funnel_shr`, not the fallback impl.
+    // SAFETY: we modulo `shift` so that the result is definitely less than the size of
+    // `T` in bits.
+    unsafe { unchecked_funnel_shr(x, x, shift % (mem::size_of::<T>() as u32 * 8)) }
+}
 
 /// Returns (a + b) mod 2<sup>N</sup>, where N is the width of T in bits.
 ///

library/core/src/mem/manually_drop.rs

@@ -217,8 +217,9 @@ impl<T> ManuallyDrop<T> {
     ///
     #[must_use = "if you don't need the value, you can use `ManuallyDrop::drop` instead"]
     #[stable(feature = "manually_drop_take", since = "1.42.0")]
+    #[rustc_const_unstable(feature = "const_manually_drop_take", issue = "148773")]
     #[inline]
-    pub unsafe fn take(slot: &mut ManuallyDrop<T>) -> T {
+    pub const unsafe fn take(slot: &mut ManuallyDrop<T>) -> T {
         // SAFETY: we are reading from a reference, which is guaranteed
         // to be valid for reads.
         unsafe { ptr::read(&slot.value) }

library/core/src/mem/mod.rs

@@ -808,7 +808,8 @@ pub const fn swap<T>(x: &mut T, y: &mut T) {
 /// ```
 #[inline]
 #[stable(feature = "mem_take", since = "1.40.0")]
-pub fn take<T: Default>(dest: &mut T) -> T {
+#[rustc_const_unstable(feature = "const_default", issue = "143894")]
+pub const fn take<T: [const] Default>(dest: &mut T) -> T {
     replace(dest, T::default())
 }
 

library/core/src/num/int_macros.rs

@@ -275,6 +275,10 @@ macro_rules! int_impl {
         /// Shifts the bits to the left by a specified amount, `n`,
         /// wrapping the truncated bits to the end of the resulting integer.
         ///
+        /// `rotate_left(n)` is equivalent to applying `rotate_left(1)` a total of `n` times. In
+        /// particular, a rotation by the number of bits in `self` returns the input value
+        /// unchanged.
+        ///
         /// Please note this isn't the same operation as the `<<` shifting operator!
         ///
         /// # Examples
@@ -284,6 +288,7 @@ macro_rules! int_impl {
         #[doc = concat!("let m = ", $rot_result, ";")]
         ///
         #[doc = concat!("assert_eq!(n.rotate_left(", $rot, "), m);")]
+        #[doc = concat!("assert_eq!(n.rotate_left(1024), n);")]
         /// ```
         #[stable(feature = "rust1", since = "1.0.0")]
         #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")]
@@ -298,6 +303,10 @@ macro_rules! int_impl {
         /// wrapping the truncated bits to the beginning of the resulting
         /// integer.
         ///
+        /// `rotate_right(n)` is equivalent to applying `rotate_right(1)` a total of `n` times. In
+        /// particular, a rotation by the number of bits in `self` returns the input value
+        /// unchanged.
+        ///
         /// Please note this isn't the same operation as the `>>` shifting operator!
         ///
         /// # Examples
@@ -307,6 +316,7 @@ macro_rules! int_impl {
         #[doc = concat!("let m = ", $rot_op, ";")]
         ///
         #[doc = concat!("assert_eq!(n.rotate_right(", $rot, "), m);")]
+        #[doc = concat!("assert_eq!(n.rotate_right(1024), n);")]
         /// ```
         #[stable(feature = "rust1", since = "1.0.0")]
         #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")]

library/core/src/num/uint_macros.rs

@@ -336,6 +336,10 @@ macro_rules! uint_impl {
         /// Shifts the bits to the left by a specified amount, `n`,
         /// wrapping the truncated bits to the end of the resulting integer.
         ///
+        /// `rotate_left(n)` is equivalent to applying `rotate_left(1)` a total of `n` times. In
+        /// particular, a rotation by the number of bits in `self` returns the input value
+        /// unchanged.
+        ///
         /// Please note this isn't the same operation as the `<<` shifting operator!
         ///
         /// # Examples
@@ -345,12 +349,14 @@ macro_rules! uint_impl {
         #[doc = concat!("let m = ", $rot_result, ";")]
         ///
         #[doc = concat!("assert_eq!(n.rotate_left(", $rot, "), m);")]
+        #[doc = concat!("assert_eq!(n.rotate_left(1024), n);")]
         /// ```
         #[stable(feature = "rust1", since = "1.0.0")]
         #[rustc_const_stable(feature = "const_math", since = "1.32.0")]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
         #[inline(always)]
+        #[rustc_allow_const_fn_unstable(const_trait_impl)] // for the intrinsic fallback
         pub const fn rotate_left(self, n: u32) -> Self {
             return intrinsics::rotate_left(self, n);
         }
@@ -359,6 +365,10 @@ macro_rules! uint_impl {
         /// wrapping the truncated bits to the beginning of the resulting
         /// integer.
         ///
+        /// `rotate_right(n)` is equivalent to applying `rotate_right(1)` a total of `n` times. In
+        /// particular, a rotation by the number of bits in `self` returns the input value
+        /// unchanged.
+        ///
         /// Please note this isn't the same operation as the `>>` shifting operator!
         ///
         /// # Examples
@@ -368,12 +378,14 @@ macro_rules! uint_impl {
         #[doc = concat!("let m = ", $rot_op, ";")]
         ///
         #[doc = concat!("assert_eq!(n.rotate_right(", $rot, "), m);")]
+        #[doc = concat!("assert_eq!(n.rotate_right(1024), n);")]
         /// ```
         #[stable(feature = "rust1", since = "1.0.0")]
         #[rustc_const_stable(feature = "const_math", since = "1.32.0")]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
         #[inline(always)]
+        #[rustc_allow_const_fn_unstable(const_trait_impl)] // for the intrinsic fallback
         pub const fn rotate_right(self, n: u32) -> Self {
             return intrinsics::rotate_right(self, n);
         }

library/core/src/str/mod.rs

@@ -1251,6 +1251,8 @@ impl str {
     /// ending will return the same lines as an otherwise identical string
     /// without a final line ending.
     ///
+    /// An empty string returns an empty iterator.
+    ///
     /// # Examples
     ///
     /// Basic usage:
@@ -1281,6 +1283,15 @@ impl str {
     ///
     /// assert_eq!(None, lines.next());
     /// ```
+    ///
+    /// An empty string returns an empty iterator:
+    ///
+    /// ```
+    /// let text = "";
+    /// let mut lines = text.lines();
+    ///
+    /// assert_eq!(lines.next(), None);
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
     #[inline]
     pub fn lines(&self) -> Lines<'_> {

src/tools/compiletest/src/runtest.rs

@@ -2766,12 +2766,11 @@ impl<'test> TestCx<'test> {
             .map_err(|err| format!("failed to load expected output from `{}`: {}", path, err))
     }
 
+    /// Attempts to delete a file, succeeding if the file does not exist.
     fn delete_file(&self, file: &Utf8Path) {
-        if !file.exists() {
-            // Deleting a nonexistent file would error.
-            return;
-        }
-        if let Err(e) = fs::remove_file(file.as_std_path()) {
+        if let Err(e) = fs::remove_file(file.as_std_path())
+            && e.kind() != io::ErrorKind::NotFound
+        {
             self.fatal(&format!("failed to delete `{}`: {}", file, e,));
         }
     }

tests/codegen-llvm/intrinsics/rotate_left.rs

@@ -1,4 +1,4 @@
-//@ compile-flags: -C no-prepopulate-passes
+//@ compile-flags: -O
 
 #![crate_type = "lib"]
 #![feature(core_intrinsics)]
@@ -9,7 +9,7 @@ use std::intrinsics::rotate_left;
 #[no_mangle]
 pub unsafe fn rotate_left_u16(x: u16, shift: u32) -> u16 {
     // CHECK: %[[tmp:.*]] = trunc i32 %shift to i16
-    // CHECK: call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %[[tmp]])
+    // CHECK: call noundef i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %[[tmp]])
     rotate_left(x, shift)
 }
 
@@ -18,14 +18,14 @@ pub unsafe fn rotate_left_u16(x: u16, shift: u32) -> u16 {
 pub unsafe fn rotate_left_u32(x: u32, shift: u32) -> u32 {
     // CHECK-NOT: trunc
     // CHECK-NOT: zext
-    // CHECK: call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %shift)
+    // CHECK: call noundef i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %shift)
     rotate_left(x, shift)
 }
 
 // CHECK-LABEL: @rotate_left_u64
 #[no_mangle]
 pub unsafe fn rotate_left_u64(x: u64, shift: u32) -> u64 {
     // CHECK: %[[tmp:.*]] = zext i32 %shift to i64
-    // CHECK: call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %[[tmp]])
+    // CHECK: call noundef i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %[[tmp]])
     rotate_left(x, shift)
 }

tests/ui/autodiff/no_lto_flag.no_lto.stderr

@@ -0,0 +1,4 @@
+error: using the autodiff feature requires setting `lto="fat"` in your Cargo.toml
+
+error: aborting due to 1 previous error
+

tests/ui/autodiff/no_lto_flag.rs

@@ -0,0 +1,31 @@
+//@ needs-enzyme
+//@ no-prefer-dynamic
+//@ revisions: with_lto no_lto
+//@[with_lto] compile-flags: -Zautodiff=Enable -C opt-level=3 -Clto=fat
+//@[no_lto] compile-flags: -Zautodiff=Enable -C opt-level=3 -Clto=thin
+
+#![feature(autodiff)]
+//@[no_lto] build-fail
+//@[with_lto] build-pass
+
+// Autodiff requires users to enable lto=fat (for now).
+// In the past, autodiff did not run if users forget to enable fat-lto, which caused functions to
+// return zero-derivatives. That's obviously wrong and confusing to users. We now added a check
+// which will abort compilation instead.
+
+use std::autodiff::autodiff_reverse;
+//[no_lto]~? ERROR using the autodiff feature requires setting `lto="fat"` in your Cargo.toml
+
+#[autodiff_reverse(d_square, Duplicated, Active)]
+fn square(x: &f64) -> f64 {
+    *x * *x
+}
+
+fn main() {
+    let xf64: f64 = std::hint::black_box(3.0);
+
+    let mut df_dxf64: f64 = std::hint::black_box(0.0);
+
+    let _output_f64 = d_square(&xf64, &mut df_dxf64, 1.0);
+    assert_eq!(6.0, df_dxf64);
+}