diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp index 6bf543cf794b7..f1bacff7fc691 100644 --- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp +++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp @@ -330,6 +330,12 @@ static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType, "zero expects struct, array, vector, or complex type"); } + if (mlir::isa(attrType)) { + if (!mlir::isa(opType)) + return success(); + return op->emitOpError("undef expects non-void type"); + } + if (mlir::isa(attrType)) { if (!mlir::isa(opType)) return op->emitOpError("result type (") diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index cd923a15af132..0c34d87734c3e 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -240,7 +240,7 @@ class CIRAttrToValue { .Case( + cir::UndefAttr, cir::VTableAttr, cir::ZeroAttr>( [&](auto attrT) { return visitCirAttr(attrT); }) .Default([&](auto attrT) { return mlir::Value(); }); } @@ -254,6 +254,7 @@ class CIRAttrToValue { mlir::Value visitCirAttr(cir::ConstVectorAttr attr); mlir::Value visitCirAttr(cir::GlobalViewAttr attr); mlir::Value visitCirAttr(cir::TypeInfoAttr attr); + mlir::Value visitCirAttr(cir::UndefAttr attr); mlir::Value visitCirAttr(cir::VTableAttr attr); mlir::Value visitCirAttr(cir::ZeroAttr attr); @@ -591,6 +592,13 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) { return result; } +/// UndefAttr visitor. +mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) { + mlir::Location loc = parentOp->getLoc(); + return mlir::LLVM::UndefOp::create( + rewriter, loc, converter->convertType(undefAttr.getType())); +} + // VTableAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) { mlir::Type llvmTy = converter->convertType(vtableArr.getType()); @@ -2046,9 +2054,11 @@ CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal( cir::GlobalOp op, mlir::Attribute init, mlir::ConversionPatternRewriter &rewriter) const { // TODO: Generalize this handling when more types are needed here. - assert((isa(init))); + assert( + (isa( + init))); // TODO(cir): once LLVM's dialect has proper equivalent attributes this // should be updated. For now, we use a custom op to initialize globals @@ -2106,8 +2116,8 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite( } else if (mlir::isa( - init.value())) { + cir::TypeInfoAttr, cir::UndefAttr, cir::VTableAttr, + cir::ZeroAttr>(init.value())) { // TODO(cir): once LLVM's dialect has proper equivalent attributes this // should be updated. For now, we use a custom op to initialize globals // to the appropriate value. diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 5590d217e96ff..82ca831f35da2 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1134,6 +1134,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline( CodeGenOpts.SanitizeMinimalRuntime), /*MayReturn=*/ CodeGenOpts.SanitizeRecover.has(SanitizerKind::LocalBounds), + /*HandlerPreserveAllRegs=*/ + static_cast(CodeGenOpts.SanitizeHandlerPreserveAllRegs), }; } FPM.addPass(BoundsCheckingPass(Options)); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index a72cbdb18c694..05f26d0bfc9df 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3819,6 +3819,8 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, bool NeedsAbortSuffix = IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable; bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime; + bool HandlerPreserveAllRegs = + CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs; const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler]; const StringRef CheckName = CheckInfo.Name; std::string FnName = "__ubsan_handle_" + CheckName.str(); @@ -3828,6 +3830,8 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, FnName += "_minimal"; if (NeedsAbortSuffix) FnName += "_abort"; + if (HandlerPreserveAllRegs && !NeedsAbortSuffix) + FnName += "_preserve"; bool MayReturn = !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable; @@ -3848,6 +3852,10 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr()); if (NoMerge) HandlerCall->addFnAttr(llvm::Attribute::NoMerge); + if (HandlerPreserveAllRegs && !NeedsAbortSuffix) { + // N.B. there is also a clang::CallingConv which is not what we want here. + HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll); + } if (!MayReturn) { HandlerCall->setDoesNotReturn(); CGF.Builder.CreateUnreachable(); diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp index dbb7bc99ac638..a49a0c91681fe 100644 --- a/clang/lib/CodeGen/CGPointerAuth.cpp +++ b/clang/lib/CodeGen/CGPointerAuth.cpp @@ -440,9 +440,10 @@ CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key, IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0); } - return llvm::ConstantPtrAuth::get(Pointer, - llvm::ConstantInt::get(Int32Ty, Key), - IntegerDiscriminator, AddressDiscriminator); + return llvm::ConstantPtrAuth::get( + Pointer, llvm::ConstantInt::get(Int32Ty, Key), IntegerDiscriminator, + AddressDiscriminator, + /*DeactivationSymbol=*/llvm::Constant::getNullValue(DefaultPtrTy)); } /// Does a given PointerAuthScheme require us to sign a value diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp index 9902cbbf99436..d5a0b65ab758f 100644 --- a/clang/lib/Driver/SanitizerArgs.cpp +++ b/clang/lib/Driver/SanitizerArgs.cpp @@ -419,6 +419,7 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC, const Driver &D = TC.getDriver(); SanitizerMask TrappingKinds = parseSanitizeTrapArgs(D, Args, DiagnoseErrors); SanitizerMask InvalidTrappingKinds = TrappingKinds & NotAllowedWithTrap; + const llvm::Triple &Triple = TC.getTriple(); MinimalRuntime = Args.hasFlag(options::OPT_fsanitize_minimal_runtime, @@ -426,7 +427,8 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC, HandlerPreserveAllRegs = Args.hasFlag(options::OPT_fsanitize_handler_preserve_all_regs, options::OPT_fno_sanitize_handler_preserve_all_regs, - HandlerPreserveAllRegs); + HandlerPreserveAllRegs) && + MinimalRuntime && (Triple.isAArch64() || Triple.isX86_64()); // The object size sanitizer should not be enabled at -O0. Arg *OptLevel = Args.getLastArg(options::OPT_O_Group); @@ -494,7 +496,6 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC, // -fsanitize=function and -fsanitize=kcfi instrument indirect function // calls to load a type hash before the function label. Therefore, an // execute-only target doesn't support the function and kcfi sanitizers. - const llvm::Triple &Triple = TC.getTriple(); if (isExecuteOnlyTarget(Triple, Args)) { if (SanitizerMask KindsToDiagnose = Add & NotAllowedWithExecuteOnly & ~DiagnosedKinds) { diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp index 8d2758bebe0fd..d28a2268b62da 100644 --- a/clang/lib/Driver/ToolChains/Linux.cpp +++ b/clang/lib/Driver/ToolChains/Linux.cpp @@ -1156,7 +1156,7 @@ SanitizerMask Linux::getSupportedSanitizers() const { if (IsX86_64 || IsMIPS64 || IsAArch64 || IsPowerPC64 || IsSystemZ || IsLoongArch64 || IsRISCV64) Res |= SanitizerKind::Thread; - if (IsX86_64 || IsAArch64 || IsSystemZ) + if (IsX86_64 || IsAArch64) Res |= SanitizerKind::Type; if (IsX86_64 || IsSystemZ || IsPowerPC64) Res |= SanitizerKind::KernelMemory; diff --git a/clang/test/CIR/CodeGen/lambda.cpp b/clang/test/CIR/CodeGen/lambda.cpp index 91380b9bea296..1d06496a85530 100644 --- a/clang/test/CIR/CodeGen/lambda.cpp +++ b/clang/test/CIR/CodeGen/lambda.cpp @@ -8,6 +8,39 @@ // We declare anonymous record types to represent lambdas. Rather than trying to // to match the declarations, we establish variables for these when they are used. +auto global_lambda = [](){}; +void use_global_lambda() { + global_lambda(); +} + +// CIR: cir.global "private" internal dso_local @global_lambda = #cir.undef : ![[REC_LAM_GLOBAL_LAMBDA:.*]] {alignment = 1 : i64} +// CIR: cir.func lambda internal private dso_local @_ZNK3$_0clEv(%[[THIS_ARG:.*]]: !cir.ptr {{.*}}) +// CIR: %[[THIS:.*]] = cir.alloca !cir.ptr, !cir.ptr>, ["this", init] +// CIR: cir.store %[[THIS_ARG]], %[[THIS]] +// CIR: cir.load %[[THIS]] +// +// CIR: cir.func {{.*}} @_Z17use_global_lambdav() +// CIR: %[[LAMBDA:.*]] = cir.get_global @global_lambda : !cir.ptr +// CIR: cir.call @_ZNK3$_0clEv(%[[LAMBDA]]) : (!cir.ptr) -> () + +// LLVM: @global_lambda = internal global %[[REC_LAM_GLOBAL_LAMBDA:.*]] undef, align 1 +// LLVM: define internal void @"_ZNK3$_0clEv"(ptr %[[THIS_ARG:.*]]) +// LLVM: %[[THIS_ADDR:.*]] = alloca ptr +// LLVM: store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]] +// LLVM: %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]] +// +// LLVM: define dso_local void @_Z17use_global_lambdav() +// LLVM: call void @"_ZNK3$_0clEv"(ptr @global_lambda) + +// OGCG: @global_lambda = internal global %[[REC_LAM_GLOBAL_LAMBDA:.*]] undef, align 1 +// OGCG: define dso_local void @_Z17use_global_lambdav() +// OGCG: call void @"_ZNK3$_0clEv"(ptr noundef nonnull align 1 dereferenceable(1) @global_lambda) +// +// OGCG: define internal void @"_ZNK3$_0clEv"(ptr {{.*}} %[[THIS_ARG:.*]]) +// OGCG: %[[THIS_ADDR:.*]] = alloca ptr +// OGCG: store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]] +// OGCG: %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]] + void fn() { auto a = [](){}; a(); diff --git a/clang/test/CIR/CodeGen/X86/avx-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx10_2_512bf16-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx10_2_512bf16-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx10_2_512bf16-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx10_2_512bf16-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx10_2bf16-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx10_2bf16-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx10_2bf16-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx10_2bf16-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx512bw-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx512bw-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx512f-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx512f-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/avx512fp16-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512fp16-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/avx512fp16-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/avx512fp16-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/bmi-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/bmi-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/bmi-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/bmi-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/lzcnt-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/lzcnt-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/lzcnt-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/lzcnt-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/sse-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/sse-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c diff --git a/clang/test/CIR/CodeGen/X86/sse2-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c similarity index 100% rename from clang/test/CIR/CodeGen/X86/sse2-builtins.c rename to clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c diff --git a/clang/test/CIR/CodeGen/builtin-fcmp-sse.c b/clang/test/CIR/CodeGenBuiltins/builtin-fcmp-sse.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin-fcmp-sse.c rename to clang/test/CIR/CodeGenBuiltins/builtin-fcmp-sse.c diff --git a/clang/test/CIR/CodeGen/builtin-isfpclass.c b/clang/test/CIR/CodeGenBuiltins/builtin-isfpclass.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin-isfpclass.c rename to clang/test/CIR/CodeGenBuiltins/builtin-isfpclass.c diff --git a/clang/test/CIR/CodeGen/builtin_bit.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_bit.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_bit.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_bit.cpp diff --git a/clang/test/CIR/CodeGen/builtin_call.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_call.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_call.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_call.cpp diff --git a/clang/test/CIR/CodeGen/builtin_inline.c b/clang/test/CIR/CodeGenBuiltins/builtin_inline.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin_inline.c rename to clang/test/CIR/CodeGenBuiltins/builtin_inline.c diff --git a/clang/test/CIR/CodeGen/builtin_new_delete.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_new_delete.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_new_delete.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_new_delete.cpp diff --git a/clang/test/CIR/CodeGen/builtin_prefetch.c b/clang/test/CIR/CodeGenBuiltins/builtin_prefetch.c similarity index 100% rename from clang/test/CIR/CodeGen/builtin_prefetch.c rename to clang/test/CIR/CodeGenBuiltins/builtin_prefetch.c diff --git a/clang/test/CIR/CodeGen/builtin_printf.cpp b/clang/test/CIR/CodeGenBuiltins/builtin_printf.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtin_printf.cpp rename to clang/test/CIR/CodeGenBuiltins/builtin_printf.cpp diff --git a/clang/test/CIR/CodeGen/builtins-elementwise.c b/clang/test/CIR/CodeGenBuiltins/builtins-elementwise.c similarity index 100% rename from clang/test/CIR/CodeGen/builtins-elementwise.c rename to clang/test/CIR/CodeGenBuiltins/builtins-elementwise.c diff --git a/clang/test/CIR/CodeGen/builtins-floating-point.c b/clang/test/CIR/CodeGenBuiltins/builtins-floating-point.c similarity index 100% rename from clang/test/CIR/CodeGen/builtins-floating-point.c rename to clang/test/CIR/CodeGenBuiltins/builtins-floating-point.c diff --git a/clang/test/CIR/CodeGen/builtins-overflow.cpp b/clang/test/CIR/CodeGenBuiltins/builtins-overflow.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtins-overflow.cpp rename to clang/test/CIR/CodeGenBuiltins/builtins-overflow.cpp diff --git a/clang/test/CIR/CodeGen/builtins.cpp b/clang/test/CIR/CodeGenBuiltins/builtins.cpp similarity index 100% rename from clang/test/CIR/CodeGen/builtins.cpp rename to clang/test/CIR/CodeGenBuiltins/builtins.cpp diff --git a/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c b/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c index 9fbb1221ab39a..2c44842f9d28e 100644 --- a/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c +++ b/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c @@ -171,7 +171,7 @@ void xf(); // PRESERVE_MIN-NEXT: [[TMP3:%.*]] = call i1 @llvm.type.test(ptr [[TMP2]], metadata !"_ZTSFvE"), !nosanitize [[META10:![0-9]+]] // PRESERVE_MIN-NEXT: br i1 [[TMP3]], label %[[CONT:.*]], label %[[HANDLER_CFI_CHECK_FAIL:.*]], !prof [[PROF11:![0-9]+]], !nosanitize [[META10]] // PRESERVE_MIN: [[HANDLER_CFI_CHECK_FAIL]]: -// PRESERVE_MIN-NEXT: call void @__ubsan_handle_cfi_check_fail_minimal() #[[ATTR4:[0-9]+]], !nosanitize [[META10]] +// PRESERVE_MIN-NEXT: call preserve_allcc void @__ubsan_handle_cfi_check_fail_minimal_preserve() #[[ATTR4:[0-9]+]], !nosanitize [[META10]] // PRESERVE_MIN-NEXT: br label %[[CONT]], !nosanitize [[META10]] // PRESERVE_MIN: [[CONT]]: // PRESERVE_MIN-NEXT: call void (...) [[TMP2]]() diff --git a/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp b/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp index 0130d9e33cd9d..2451d31e9a489 100644 --- a/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp +++ b/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp @@ -127,7 +127,7 @@ struct S1 { // PRESERVE_MIN-NEXT: [[TMP2:%.*]] = call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"all-vtables"), !nosanitize [[META5]] // PRESERVE_MIN-NEXT: br i1 [[TMP1]], label %[[CONT:.*]], label %[[HANDLER_CFI_CHECK_FAIL:.*]], !prof [[PROF6:![0-9]+]], !nosanitize [[META5]] // PRESERVE_MIN: [[HANDLER_CFI_CHECK_FAIL]]: -// PRESERVE_MIN-NEXT: call void @__ubsan_handle_cfi_check_fail_minimal() #[[ATTR3:[0-9]+]], !nosanitize [[META5]] +// PRESERVE_MIN-NEXT: call preserve_allcc void @__ubsan_handle_cfi_check_fail_minimal_preserve() #[[ATTR3:[0-9]+]], !nosanitize [[META5]] // PRESERVE_MIN-NEXT: br label %[[CONT]], !nosanitize [[META5]] // PRESERVE_MIN: [[CONT]]: // PRESERVE_MIN-NEXT: [[VFN:%.*]] = getelementptr inbounds ptr, ptr [[VTABLE]], i64 0 diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c index f2a4d8c50ec23..c02b8828062f2 100644 --- a/clang/test/Driver/fsanitize.c +++ b/clang/test/Driver/fsanitize.c @@ -984,10 +984,20 @@ // CHECK-UBSAN-MINIMAL: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} // CHECK-UBSAN-MINIMAL: "-fsanitize-minimal-runtime" -// RUN: %clang --target=x86_64-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE -// CHECK-UBSAN-MINIMAL-PRESERVE: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} -// CHECK-UBSAN-MINIMAL-PRESERVE: "-fsanitize-minimal-runtime" -// CHECK-UBSAN-MINIMAL-PRESERVE: "-fsanitize-handler-preserve-all-regs +// RUN: %clang --target=x86_64-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE-X86-64 +// CHECK-UBSAN-MINIMAL-PRESERVE-X86-64: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} +// CHECK-UBSAN-MINIMAL-PRESERVE-X86-64: "-fsanitize-minimal-runtime" +// CHECK-UBSAN-MINIMAL-PRESERVE-X86-64: "-fsanitize-handler-preserve-all-regs + +// RUN: %clang --target=aarch64-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64 +// CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} +// CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64: "-fsanitize-minimal-runtime" +// CHECK-UBSAN-MINIMAL-PRESERVE-AARCH64: "-fsanitize-handler-preserve-all-regs + +// RUN: %clang --target=i386-linux-gnu -fsanitize=undefined -fsanitize-minimal-runtime -fsanitize-handler-preserve-all-regs %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-MINIMAL-PRESERVE-I386 +// CHECK-UBSAN-MINIMAL-PRESERVE-I386: "-fsanitize={{((signed-integer-overflow|integer-divide-by-zero|shift-base|shift-exponent|unreachable|return|vla-bound|alignment|null|pointer-overflow|float-cast-overflow|array-bounds|enum|bool|builtin|returns-nonnull-attribute|nonnull-attribute|function),?){18}"}} +// CHECK-UBSAN-MINIMAL-PRESERVE-I386: "-fsanitize-minimal-runtime" +// CHECK-UBSAN-MINIMAL-PRESERVE-I386-NOT: "-fsanitize-handler-preserve-all-regs // RUN: %clang --target=x86_64-linux-gnu -fsanitize=integer -fsanitize-trap=integer %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-INTSAN-TRAP // CHECK-INTSAN-TRAP: "-fsanitize-trap=integer-divide-by-zero,shift-base,shift-exponent,signed-integer-overflow,unsigned-integer-overflow,unsigned-shift-base,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change" diff --git a/clang/test/Driver/hip-spirv-backend-opt.c b/clang/test/Driver/hip-spirv-backend-opt.c index 88c4a848d5760..10d9a0b01caf3 100644 --- a/clang/test/Driver/hip-spirv-backend-opt.c +++ b/clang/test/Driver/hip-spirv-backend-opt.c @@ -4,58 +4,58 @@ // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only -S \ +// RUN: -use-spirv-backend --offload-device-only -S -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-TEXTUAL // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only \ +// RUN: -use-spirv-backend --offload-device-only -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BINARY // The new driver's behavior is to emit LLVM IR for --offload-device-only and -fgpu-rdc (independently of SPIR-V). // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -### -nogpuinc -nogpulib -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only -S -fgpu-rdc \ +// RUN: -use-spirv-backend --offload-device-only -S -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-LL,CHECK-FGPU-RDC // The new driver's behavior is to emit LLVM IR for --offload-device-only and -fgpu-rdc (independently of SPIR-V). // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend --offload-device-only -fgpu-rdc \ +// RUN: -use-spirv-backend --offload-device-only -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-FGPU-RDC // --offload-device-only is always unset --- testing interactions with -S and -fgpu-rdc // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend -S -fgpu-rdc \ +// RUN: -use-spirv-backend -S -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-FGPU-RDC // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend -S \ +// RUN: -use-spirv-backend -S -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend -fgpu-rdc \ +// RUN: -use-spirv-backend -fgpu-rdc -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-CLANG-LINKER-WRAPPER // RUN: %clang --offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend \ +// RUN: -use-spirv-backend -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-CLANG-LINKER-WRAPPER // RUN: %clang --no-offload-new-driver --target=x86_64-unknown-linux-gnu --offload-arch=amdgcnspirv \ // RUN: -nogpuinc -nogpulib -### -x hip %s -save-temps \ -// RUN: -use-spirv-backend \ +// RUN: -use-spirv-backend -no-canonical-prefixes \ // RUN: 2>&1 | FileCheck %s --check-prefixes=CHECK-SPIRV-TRANSLATOR,CHECK-SPIRV-BACKEND-BC,CHECK-SPIRV-BACKEND-BINARY-EQ-TRIPLE // CHECK-SPIRV-TRANSLATOR-NOT: "{{.*llvm-spirv.*}}" -// CHECK-SPIRV-BACKEND-TEXTUAL: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-S" -// CHECK-SPIRV-BACKEND-BINARY: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-obj" -// CHECK-SPIRV-BACKEND-BC: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm-bc" -// CHECK-SPIRV-BACKEND-LL: "{{.*}}clang{{.*}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm" -// CHECK-SPIRV-BACKEND-BINARY-EQ-TRIPLE: "{{.*}}clang{{.*}}" "-cc1" {{.*}}"-triple=spirv64-amd-amdhsa" {{.*}}"-emit-obj" +// CHECK-SPIRV-BACKEND-TEXTUAL: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-S" +// CHECK-SPIRV-BACKEND-BINARY: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-obj" +// CHECK-SPIRV-BACKEND-BC: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm-bc" +// CHECK-SPIRV-BACKEND-LL: "{{.*clang(\.exe)?}}" "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-emit-llvm" +// CHECK-SPIRV-BACKEND-BINARY-EQ-TRIPLE: "{{.*clang(\.exe)?}}" "-cc1" {{.*}}"-triple=spirv64-amd-amdhsa" {{.*}}"-emit-obj" // CHECK-FGPU-RDC-SAME: {{.*}} "-fgpu-rdc" // CHECK-CLANG-LINKER-WRAPPER: "{{.*}}clang-linker-wrapper" "--should-extract=amdgcnspirv" {{.*}} "--device-compiler=spirv64-amd-amdhsa=-use-spirv-backend" diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake index f2317de8916e9..c10367715396e 100644 --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -89,7 +89,7 @@ else() set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X} ${LOONGARCH64} ${RISCV64}) endif() -set(ALL_TYSAN_SUPPORTED_ARCH ${X86_64} ${ARM64} ${S390X}) +set(ALL_TYSAN_SUPPORTED_ARCH ${X86_64} ${ARM64}) set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64} ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON} ${LOONGARCH64}) diff --git a/compiler-rt/lib/tysan/tysan_platform.h b/compiler-rt/lib/tysan/tysan_platform.h index 7d84ef228216d..19f77f0cace6b 100644 --- a/compiler-rt/lib/tysan/tysan_platform.h +++ b/compiler-rt/lib/tysan/tysan_platform.h @@ -45,12 +45,6 @@ struct Mapping48 { static const uptr kPtrShift = 3; }; #define TYSAN_RUNTIME_VMA 1 -#elif defined(__s390x__) -struct Mapping { - static const uptr kShadowAddr = 0x080000000000ULL; - static const uptr kAppAddr = 0x460000000000ULL; - static const uptr kAppMemMsk = ~0xC00000000000ULL; -}; #else #error "TySan not supported for this platform!" #endif diff --git a/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c b/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c index aaed134b3ae81..8c04a0091cb11 100644 --- a/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c +++ b/compiler-rt/test/ubsan_minimal/TestCases/override-callback.c @@ -1,6 +1,7 @@ -// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change %s -o %t && %run %t 2>&1 | FileCheck %s -// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all %s -o %t && not --crash %run %t 2>&1 | FileCheck %s -// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all -DOVERRIDE=1 %s -o %t && not --crash %run %t 2>&1 | FileCheck %s --check-prefixes=FATAL +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change %s -o %t && %run %t 2>&1 | FileCheck %s +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fsanitize-handler-preserve-all-regs -DPRESERVE %s -o %t && %run %t 2>&1 | FileCheck %s --check-prefixes=PRESERVE +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all %s -o %t && not --crash %run %t 2>&1 | FileCheck %s +// RUN: %clang_min_runtime -fsanitize=implicit-integer-sign-change -fno-sanitize-recover=all -DOVERRIDE=1 %s -o %t && not --crash %run %t 2>&1 | FileCheck %s --check-prefixes=FATAL #include #include @@ -9,8 +10,21 @@ static int Result; void __ubsan_report_error(const char *kind, uintptr_t caller) { +// -fsanitize-handler-preserve-all-regs is ignored on other architectures. +// Prented we called to other handler on those. +#if defined(PRESERVE) && !defined(__aarch64__) && !defined(__x86_64__) + fprintf(stderr, "CUSTOM_CALLBACK_PRESERVE: %s\n", kind); +#else fprintf(stderr, "CUSTOM_CALLBACK: %s\n", kind); +#endif +} + +#if defined(__aarch64__) || defined(__x86_64__) +[[clang::preserve_all]] void __ubsan_report_error_preserve(const char *kind, + uintptr_t caller) { + fprintf(stderr, "CUSTOM_CALLBACK_PRESERVE: %s\n", kind); } +#endif #if OVERRIDE void __ubsan_report_error_fatal(const char *kind, uintptr_t caller) { @@ -21,5 +35,6 @@ void __ubsan_report_error_fatal(const char *kind, uintptr_t caller) { int main(int argc, const char **argv) { int32_t t0 = (~((uint32_t)0)); // CHECK: CUSTOM_CALLBACK: implicit-conversion + // PRESERVE: CUSTOM_CALLBACK_PRESERVE: implicit-conversion // FATAL: FATAL_CALLBACK: implicit-conversion } diff --git a/flang-rt/lib/cuda/allocator.cpp b/flang-rt/lib/cuda/allocator.cpp index 5436051002265..d2aa832883e65 100644 --- a/flang-rt/lib/cuda/allocator.cpp +++ b/flang-rt/lib/cuda/allocator.cpp @@ -19,8 +19,6 @@ #include "flang/Runtime/CUDA/common.h" #include "flang/Support/Fortran.h" -#include "cuda_runtime.h" - namespace Fortran::runtime::cuda { struct DeviceAllocation { @@ -133,6 +131,15 @@ void RTDEF(CUFRegisterAllocator)() { allocatorRegistry.Register( kUnifiedAllocatorPos, {&CUFAllocUnified, CUFFreeUnified}); } + +cudaStream_t RTDECL(CUFAssociatedGetStream)(void *p) { + int pos = findAllocation(p); + if (pos >= 0) { + cudaStream_t stream = deviceAllocations[pos].stream; + return stream; + } + return nullptr; +} } void *CUFAllocPinned( diff --git a/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp b/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp index 9935ae0eaac2f..4e65326b31a62 100644 --- a/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp +++ b/flang-rt/unittests/Runtime/CUDA/Allocatable.cpp @@ -121,3 +121,54 @@ TEST(AllocatableCUFTest, StreamDeviceAllocatable) { cudaDeviceSynchronize(); EXPECT_EQ(cudaSuccess, cudaGetLastError()); } + +TEST(AllocatableAsyncTest, StreamDeviceAllocatable) { + using Fortran::common::TypeCategory; + RTNAME(CUFRegisterAllocator)(); + // REAL(4), DEVICE, ALLOCATABLE :: a(:) + auto a{createAllocatable(TypeCategory::Real, 4)}; + a->SetAllocIdx(kDeviceAllocatorPos); + EXPECT_EQ((int)kDeviceAllocatorPos, a->GetAllocIdx()); + EXPECT_FALSE(a->HasAddendum()); + RTNAME(AllocatableSetBounds)(*a, 0, 1, 10); + + cudaStream_t stream; + cudaStreamCreate(&stream); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + + RTNAME(AllocatableAllocate) + (*a, /*asyncObject=*/(int64_t *)&stream, /*hasStat=*/false, + /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_TRUE(a->IsAllocated()); + cudaDeviceSynchronize(); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + cudaStream_t s = RTDECL(CUFAssociatedGetStream)(a->raw().base_addr); + EXPECT_EQ(s, stream); + RTNAME(AllocatableDeallocate) + (*a, /*hasStat=*/false, /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_FALSE(a->IsAllocated()); + cudaDeviceSynchronize(); + + cudaStream_t defaultStream = 0; + RTNAME(AllocatableAllocate) + (*a, /*asyncObject=*/(int64_t *)&defaultStream, /*hasStat=*/false, + /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_TRUE(a->IsAllocated()); + cudaDeviceSynchronize(); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + cudaStream_t d = RTDECL(CUFAssociatedGetStream)(a->raw().base_addr); + EXPECT_EQ(d, defaultStream); + RTNAME(AllocatableDeallocate) + (*a, /*hasStat=*/false, /*errMsg=*/nullptr, __FILE__, __LINE__); + EXPECT_FALSE(a->IsAllocated()); + cudaDeviceSynchronize(); + + RTNAME(AllocatableAllocate) + (*a, /*asyncObject=*/nullptr, /*hasStat=*/false, /*errMsg=*/nullptr, __FILE__, + __LINE__); + EXPECT_TRUE(a->IsAllocated()); + cudaDeviceSynchronize(); + EXPECT_EQ(cudaSuccess, cudaGetLastError()); + cudaStream_t empty = RTDECL(CUFAssociatedGetStream)(a->raw().base_addr); + EXPECT_EQ(empty, nullptr); +} diff --git a/flang/include/flang/Runtime/CUDA/allocator.h b/flang/include/flang/Runtime/CUDA/allocator.h index 59fdb22b6e663..4e38482a7de30 100644 --- a/flang/include/flang/Runtime/CUDA/allocator.h +++ b/flang/include/flang/Runtime/CUDA/allocator.h @@ -13,11 +13,14 @@ #include "flang/Runtime/descriptor-consts.h" #include "flang/Runtime/entry-names.h" +#include "cuda_runtime.h" + namespace Fortran::runtime::cuda { extern "C" { void RTDECL(CUFRegisterAllocator)(); +cudaStream_t RTDECL(CUFAssociatedGetStream)(void *); } void *CUFAllocPinned(std::size_t, std::int64_t *); diff --git a/libcxx/include/__condition_variable/condition_variable.h b/libcxx/include/__condition_variable/condition_variable.h index 1e8edd5dcb009..b7151930e9226 100644 --- a/libcxx/include/__condition_variable/condition_variable.h +++ b/libcxx/include/__condition_variable/condition_variable.h @@ -170,7 +170,7 @@ class _LIBCPP_EXPORTED_FROM_ABI condition_variable { wait_for(unique_lock& __lk, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred); typedef __libcpp_condvar_t* native_handle_type; - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; } private: void diff --git a/libcxx/include/__mutex/mutex.h b/libcxx/include/__mutex/mutex.h index 68c8842b35eda..e9cedf8db1cca 100644 --- a/libcxx/include/__mutex/mutex.h +++ b/libcxx/include/__mutex/mutex.h @@ -37,11 +37,11 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("mutex") mutex { # endif _LIBCPP_ACQUIRE_CAPABILITY() void lock(); - _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT; + [[__nodiscard__]] _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT; _LIBCPP_RELEASE_CAPABILITY void unlock() _NOEXCEPT; typedef __libcpp_mutex_t* native_handle_type; - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } }; static_assert(is_nothrow_default_constructible::value, "the default constructor for std::mutex must be nothrow"); diff --git a/libcxx/include/__thread/thread.h b/libcxx/include/__thread/thread.h index a3b672bc0f0e7..561f092ddb7c0 100644 --- a/libcxx/include/__thread/thread.h +++ b/libcxx/include/__thread/thread.h @@ -242,13 +242,13 @@ class _LIBCPP_EXPORTED_FROM_ABI thread { _LIBCPP_HIDE_FROM_ABI void swap(thread& __t) _NOEXCEPT { std::swap(__t_, __t.__t_); } - _LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); } void join(); void detach(); - _LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); } - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; } - static unsigned hardware_concurrency() _NOEXCEPT; + [[__nodiscard__]] static unsigned hardware_concurrency() _NOEXCEPT; }; inline _LIBCPP_HIDE_FROM_ABI void swap(thread& __x, thread& __y) _NOEXCEPT { __x.swap(__y); } diff --git a/libcxx/include/barrier b/libcxx/include/barrier index 41fbfb3e8fb7b..5f9b471f01741 100644 --- a/libcxx/include/barrier +++ b/libcxx/include/barrier @@ -158,7 +158,9 @@ class barrier { public: using arrival_token = typename __barrier_base<_CompletionF>::arrival_token; - static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return __barrier_base<_CompletionF>::max(); } + [[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { + return __barrier_base<_CompletionF>::max(); + } _LIBCPP_HIDE_FROM_ABI explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF()) : __b_(__count, std::move(__completion)) { diff --git a/libcxx/include/latch b/libcxx/include/latch index c3b8f62e9b50e..33268d9655f25 100644 --- a/libcxx/include/latch +++ b/libcxx/include/latch @@ -70,7 +70,9 @@ class latch { atomic __a_; public: - static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits::max(); } + [[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { + return numeric_limits::max(); + } inline _LIBCPP_HIDE_FROM_ABI constexpr explicit latch(ptrdiff_t __expected) : __a_(__expected) { _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( @@ -97,7 +99,7 @@ public: if (__old == __update) __a_.notify_all(); } - inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept { + [[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept { auto __value = __a_.load(memory_order_acquire); return try_wait_impl(__value); } diff --git a/libcxx/include/mutex b/libcxx/include/mutex index 0b81f1bb1c8a6..bec0185ede21a 100644 --- a/libcxx/include/mutex +++ b/libcxx/include/mutex @@ -229,12 +229,12 @@ public: recursive_mutex& operator=(const recursive_mutex&) = delete; void lock(); - bool try_lock() _NOEXCEPT; + [[__nodiscard__]] bool try_lock() _NOEXCEPT; void unlock() _NOEXCEPT; typedef __libcpp_recursive_mutex_t* native_handle_type; - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; } }; class _LIBCPP_EXPORTED_FROM_ABI timed_mutex { @@ -251,14 +251,14 @@ public: public: void lock(); - bool try_lock() _NOEXCEPT; + [[__nodiscard__]] bool try_lock() _NOEXCEPT; template - _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { return try_lock_until(chrono::steady_clock::now() + __d); } template - _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { using namespace chrono; unique_lock __lk(__m_); bool __no_timeout = _Clock::now() < __t; @@ -288,14 +288,14 @@ public: recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete; void lock(); - bool try_lock() _NOEXCEPT; + [[__nodiscard__]] bool try_lock() _NOEXCEPT; template - _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) { return try_lock_until(chrono::steady_clock::now() + __d); } template - _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { + [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) { using namespace chrono; __thread_id __id = this_thread::get_id(); unique_lock __lk(__m_); @@ -320,7 +320,7 @@ public: }; template -_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) { +[[__nodiscard__]] _LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) { unique_lock<_L0> __u0(__l0, try_to_lock_t()); if (__u0.owns_lock()) { if (__l1.try_lock()) { @@ -335,7 +335,8 @@ _LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, # ifndef _LIBCPP_CXX03_LANG template -_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) { +[[__nodiscard__]] _LIBCPP_NO_THREAD_SAFETY_ANALYSIS + _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) { int __r = 0; unique_lock<_L0> __u0(__l0, try_to_lock); if (__u0.owns_lock()) { diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index 99c4ad24b35ec..1f19d50e32af7 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -133,7 +133,7 @@ class counting_semaphore { public: static_assert(__least_max_value >= 0, "The least maximum value must be a positive number"); - static constexpr ptrdiff_t max() noexcept { return __least_max_value; } + [[nodiscard]] static constexpr ptrdiff_t max() noexcept { return __least_max_value; } _LIBCPP_HIDE_FROM_ABI constexpr explicit counting_semaphore(ptrdiff_t __count) : __semaphore_(__count) { _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN( @@ -156,12 +156,12 @@ public: } _LIBCPP_HIDE_FROM_ABI void acquire() { __semaphore_.acquire(); } template - _LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) { + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) { return __semaphore_.try_acquire_for(chrono::duration_cast(__rel_time)); } - _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); } + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); } template - _LIBCPP_HIDE_FROM_ABI bool try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) { + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) { auto const __current = _Clock::now(); if (__current >= __abs_time) return try_acquire(); diff --git a/libcxx/test/libcxx/thread/nodiscard.verify.cpp b/libcxx/test/libcxx/thread/nodiscard.verify.cpp new file mode 100644 index 0000000000000..19e43f88db700 --- /dev/null +++ b/libcxx/test/libcxx/thread/nodiscard.verify.cpp @@ -0,0 +1,144 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03 +// UNSUPPORTED: no-threads + +// Check that functions are marked [[nodiscard]] + +#include +#include +#include +#include +#include +#include + +#include "test_macros.h" + +const auto timePoint = std::chrono::steady_clock::now(); + +void test() { + // Threads + { + std::thread th; + + th.joinable(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + th.get_id(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + th.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + th.hardware_concurrency(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } +#if TEST_STD_VER >= 20 + { + std::jthread jt; + + jt.joinable(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.get_id(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.get_stop_source(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.get_stop_token(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + jt.hardware_concurrency(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } +#endif + + // Mutual exclusion + + { // + std::mutex m; + + m.try_lock(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + m.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + { + std::recursive_mutex m; + + m.try_lock(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + m.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + { + std::timed_mutex m; + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_until(timePoint); + } + { + std::recursive_timed_mutex m; + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + m.try_lock_until(timePoint); + } + { + std::mutex m1; + std::mutex m2; + std::mutex m3; + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + std::try_lock(m1, m2); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + std::try_lock(m1, m2, m3); + } + + // Condition variables + + { // + std::condition_variable cv; + + cv.native_handle(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + +#if TEST_STD_VER >= 20 + + // Semaphores + + { // + std::counting_semaphore<> cs{0}; + + cs.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + cs.try_acquire_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + cs.try_acquire(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + cs.try_acquire_until(timePoint); + + std::binary_semaphore bs{0}; + + bs.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + bs.try_acquire_for(std::chrono::nanoseconds{82}); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + bs.try_acquire(); + // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} + bs.try_acquire_until(timePoint); + } + + // Latches and barriers + + { // + std::barrier<> b{94}; + + b.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + { // + std::latch l{94}; + + l.max(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + l.try_wait(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} + } + +#endif +} diff --git a/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp b/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp deleted file mode 100644 index 2ef5cf874da90..0000000000000 --- a/libcxx/test/std/thread/thread.jthread/nodiscard.verify.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// UNSUPPORTED: no-threads -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// [[nodiscard]] bool joinable() const noexcept; -// [[nodiscard]] id get_id() const noexcept; -// [[nodiscard]] native_handle_type native_handle(); -// [[nodiscard]] stop_source get_stop_source() noexcept; -// [[nodiscard]] stop_token get_stop_token() const noexcept; -// [[nodiscard]] static unsigned int hardware_concurrency() noexcept; - -#include - -void test() { - std::jthread jt; - jt.joinable(); // expected-warning {{ignoring return value of function}} - jt.get_id(); // expected-warning {{ignoring return value of function}} - jt.native_handle(); // expected-warning {{ignoring return value of function}} - jt.get_stop_source(); // expected-warning {{ignoring return value of function}} - jt.get_stop_token(); // expected-warning {{ignoring return value of function}} - jt.hardware_concurrency(); // expected-warning {{ignoring return value of function}} -} diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 32b20993af67c..28c817c54c85d 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -41,6 +41,7 @@ #include "llvm/Object/Archive.h" #include "llvm/Option/ArgList.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Parallel.h" #include "llvm/Support/Path.h" @@ -53,6 +54,10 @@ #include "llvm/TextAPI/Architecture.h" #include "llvm/TextAPI/PackedVersion.h" +#if !_WIN32 +#include +#endif + using namespace llvm; using namespace llvm::MachO; using namespace llvm::object; @@ -292,12 +297,13 @@ struct DeferredFile { using DeferredFiles = std::vector; #if LLVM_ENABLE_THREADS -class SerialBackgroundQueue { +class SerialBackgroundWorkQueue { std::deque> queue; std::thread *running; std::mutex mutex; public: + std::atomic_bool stopAllWork = false; void queueWork(std::function work) { mutex.lock(); if (running && queue.empty()) { @@ -312,7 +318,7 @@ class SerialBackgroundQueue { queue.emplace_back(std::move(work)); if (!running) running = new std::thread([&]() { - while (true) { + while (!stopAllWork) { mutex.lock(); if (queue.empty()) { mutex.unlock(); @@ -331,6 +337,8 @@ class SerialBackgroundQueue { } }; +static SerialBackgroundWorkQueue pageInQueue; + // Most input files have been mapped but not yet paged in. // This code forces the page-ins on multiple threads so // the process is not stalled waiting on disk buffer i/o. @@ -339,8 +347,8 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) { static const size_t largeArchive = 10 * 1024 * 1024; #ifndef NDEBUG using namespace std::chrono; - std::atomic_int numDeferedFilesTouched = 0; static std::atomic_uint64_t totalBytes = 0; + std::atomic_int numDeferedFilesAdvised = 0; auto t0 = high_resolution_clock::now(); #endif @@ -348,24 +356,34 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) { const StringRef &buff = deferredFile.buffer.getBuffer(); if (buff.size() > largeArchive) return; + #ifndef NDEBUG totalBytes += buff.size(); - numDeferedFilesTouched += 1; + numDeferedFilesAdvised += 1; #endif - +#if _WIN32 // Reference all file's mmap'd pages to load them into memory. - for (const char *page = buff.data(), *end = page + buff.size(); page < end; - page += pageSize) { + for (const char *page = buff.data(), *end = page + buff.size(); + page < end && !pageInQueue.stopAllWork; page += pageSize) { [[maybe_unused]] volatile char t = *page; (void)t; } +#else +#define DEBUG_TYPE "lld-madvise" + auto aligned = + llvm::alignDown(reinterpret_cast(buff.data()), pageSize); + if (madvise((void *)aligned, buff.size(), MADV_WILLNEED) < 0) + LLVM_DEBUG(llvm::dbgs() << "madvise error: " << strerror(errno) << "\n"); +#undef DEBUG_TYPE +#endif }; + { // Create scope for waiting for the taskGroup std::atomic_size_t index = 0; llvm::parallel::TaskGroup taskGroup; for (int w = 0; w < config->readWorkers; w++) taskGroup.spawn([&index, &preloadDeferredFile, &deferred]() { - while (true) { + while (!pageInQueue.stopAllWork) { size_t localIndex = index.fetch_add(1); if (localIndex >= deferred.size()) break; @@ -373,17 +391,17 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) { } }); } + #ifndef NDEBUG auto dt = high_resolution_clock::now() - t0; if (Process::GetEnv("LLD_MULTI_THREAD_PAGE")) llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/" - << numDeferedFilesTouched << "/" << deferred.size() << "/" + << numDeferedFilesAdvised << "/" << deferred.size() << "/" << duration_cast(dt).count() / 1000. << "\n"; #endif } static void multiThreadedPageIn(const DeferredFiles &deferred) { - static SerialBackgroundQueue pageInQueue; pageInQueue.queueWork([=]() { DeferredFiles files = deferred; multiThreadedPageInBackground(files); @@ -489,7 +507,7 @@ static InputFile *processFile(std::optional buffer, continue; } - if (archiveContents) + if (config->readWorkers && archiveContents) archiveContents->push_back({path, isLazy, *mb}); if (!hasObjCSection(*mb)) continue; @@ -1447,6 +1465,8 @@ static void createFiles(const InputArgList &args) { multiThreadedPageIn(archiveContents); for (auto *archive : archives) archive->addLazySymbols(); + + pageInQueue.stopAllWork = true; } #endif } @@ -1845,8 +1865,8 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, "'"); config->readWorkers = workers; #else - error(arg->getSpelling() + - ": option unavailable because lld was not built with thread support"); + warn(arg->getSpelling() + + ": option unavailable because lld was not built with thread support"); #endif } if (auto *arg = args.getLastArg(OPT_threads_eq)) { diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index efcffc9c53383..81caef5f15ae1 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -217,7 +217,8 @@ std::optional macho::readFile(StringRef path) { if (entry != cachedReads.end()) return entry->second; - ErrorOr> mbOrErr = MemoryBuffer::getFile(path); + ErrorOr> mbOrErr = + MemoryBuffer::getFile(path, false, /*RequiresNullTerminator=*/false); if (std::error_code ec = mbOrErr.getError()) { error("cannot open " + path + ": " + ec.message()); return std::nullopt; diff --git a/lld/test/MachO/read-workers.s b/lld/test/MachO/read-workers.s index 294106ba0b084..4d2f88c2a757c 100644 --- a/lld/test/MachO/read-workers.s +++ b/lld/test/MachO/read-workers.s @@ -1,7 +1,4 @@ # REQUIRES: x86 && thread_support -## Sometimes fails, particularly in an ASAN build, do not run until -## https://github.com/llvm/llvm-project/pull/157917 addresses the cause. -# UNSUPPORTED: target={{.*}} # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o ## A non-negative integer is allowed. diff --git a/lldb/source/Utility/RegisterValue.cpp b/lldb/source/Utility/RegisterValue.cpp index c28c9e2d4d106..4d762dc80e7f5 100644 --- a/lldb/source/Utility/RegisterValue.cpp +++ b/lldb/source/Utility/RegisterValue.cpp @@ -196,9 +196,7 @@ Status RegisterValue::SetValueFromData(const RegisterInfo ®_info, SetUInt64(src.GetMaxU64(&src_offset, src_len)); else { std::vector native_endian_src(src_len, 0); - src.ExtractBytes(src_offset, src_len, - llvm::sys::IsLittleEndianHost ? eByteOrderLittle - : eByteOrderBig, + src.ExtractBytes(src_offset, src_len, endian::InlHostByteOrder(), native_endian_src.data()); llvm::APInt uint = llvm::APInt::getZero(src_len * 8); llvm::LoadIntFromMemory(uint, native_endian_src.data(), src_len); diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index f1787083530ee..8db3268e403ff 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -3234,6 +3234,24 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation. When present, the operand bundle must contain exactly one value of token type. See the :doc:`ConvergentOperations` document for details. +.. _deactivationsymbol: + +Deactivation Symbol Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A ``"deactivation-symbol"`` operand bundle is valid on the following +instructions (AArch64 only): + +- Call to a normal function with ``notail`` attribute and a first argument and + return value of type ``ptr``. +- Call to ``llvm.ptrauth.sign`` or ``llvm.ptrauth.auth`` intrinsics. + +This operand bundle specifies that if the deactivation symbol is defined +to a valid value for the target, the marked instruction will return the +value of its first argument instead of calling the specified function +or intrinsic. This is achieved with ``PATCHINST`` relocations on the +target instructions (see the AArch64 psABI for details). + .. _moduleasm: Module-Level Inline Assembly @@ -5284,7 +5302,7 @@ need to refer to the actual function body. Pointer Authentication Constants -------------------------------- -``ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?)`` +``ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?)`` A '``ptrauth``' constant represents a pointer with a cryptographic authentication signature embedded into some bits, as described in the @@ -5313,6 +5331,11 @@ Otherwise, the expression is equivalent to: %tmp2 = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr CST to i64), i32 KEY, i64 %tmp1) %val = inttoptr i64 %tmp2 to ptr +If the deactivation symbol operand ``DS`` has a non-null value, +the semantics are as if a :ref:`deactivation-symbol operand bundle +` were added to the ``llvm.ptrauth.sign`` intrinsic +calls above, with ``DS`` as the only operand. + .. _constantexprs: Constant Expressions diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index ec208862d827a..08b478afbb1b5 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -440,6 +440,8 @@ enum ConstantsCodes { CST_CODE_CE_GEP_WITH_INRANGE = 31, // [opty, flags, range, n x operands] CST_CODE_CE_GEP = 32, // [opty, flags, n x operands] CST_CODE_PTRAUTH = 33, // [ptr, key, disc, addrdisc] + CST_CODE_PTRAUTH2 = 34, // [ptr, key, disc, addrdisc, + // deactivation_symbol] }; /// CastOpcodes - These are values used in the bitcode files to encode which diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index a8bde824527a5..fea900f37ec74 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -159,6 +159,8 @@ class LLVM_ABI CallLowering { /// True if this call results in convergent operations. bool IsConvergent = true; + + GlobalValue *DeactivationSymbol = nullptr; }; /// Argument handling is mostly uniform between the four places that diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 40c7792f7e8a2..5f3f1d386569c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -56,6 +56,7 @@ struct MachineIRBuilderState { MDNode *PCSections = nullptr; /// MMRA Metadata to be set on any instruction we create. MDNode *MMRA = nullptr; + Value *DS = nullptr; /// \name Fields describing the insertion point. /// @{ @@ -369,6 +370,7 @@ class LLVM_ABI MachineIRBuilder { State.II = MI.getIterator(); setPCSections(MI.getPCSections()); setMMRAMetadata(MI.getMMRAMetadata()); + setDeactivationSymbol(MI.getDeactivationSymbol()); } /// @} @@ -405,6 +407,9 @@ class LLVM_ABI MachineIRBuilder { /// Set the PC sections metadata to \p MD for all the next build instructions. void setMMRAMetadata(MDNode *MMRA) { State.MMRA = MMRA; } + Value *getDeactivationSymbol() { return State.DS; } + void setDeactivationSymbol(Value *DS) { State.DS = DS; } + /// Get the current instruction's MMRA metadata. MDNode *getMMRAMetadata() { return State.MMRA; } diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index cdaa916548c25..b32f3dacbb3a4 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1579,6 +1579,10 @@ enum NodeType { // Outputs: Output Chain CLEAR_CACHE, + // Untyped node storing deactivation symbol reference + // (DeactivationSymbolSDNode). + DEACTIVATION_SYMBOL, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index 57befa3006d46..8592a51f5866e 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -1209,7 +1209,7 @@ class LLVM_ABI MachineFunction { ArrayRef MMOs, MCSymbol *PreInstrSymbol = nullptr, MCSymbol *PostInstrSymbol = nullptr, MDNode *HeapAllocMarker = nullptr, MDNode *PCSections = nullptr, uint32_t CFIType = 0, - MDNode *MMRAs = nullptr); + MDNode *MMRAs = nullptr, Value *DS = nullptr); /// Allocate a string and populate it with the given external symbol name. const char *createExternalSymbolName(StringRef Name); diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index ca984459c365a..077e39b49df6f 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -160,8 +160,9 @@ class MachineInstr /// /// This has to be defined eagerly due to the implementation constraints of /// `PointerSumType` where it is used. - class ExtraInfo final : TrailingObjects { + class ExtraInfo final + : TrailingObjects { public: static ExtraInfo *create(BumpPtrAllocator &Allocator, ArrayRef MMOs, @@ -169,20 +170,23 @@ class MachineInstr MCSymbol *PostInstrSymbol = nullptr, MDNode *HeapAllocMarker = nullptr, MDNode *PCSections = nullptr, uint32_t CFIType = 0, - MDNode *MMRAs = nullptr) { + MDNode *MMRAs = nullptr, Value *DS = nullptr) { bool HasPreInstrSymbol = PreInstrSymbol != nullptr; bool HasPostInstrSymbol = PostInstrSymbol != nullptr; bool HasHeapAllocMarker = HeapAllocMarker != nullptr; bool HasMMRAs = MMRAs != nullptr; bool HasCFIType = CFIType != 0; bool HasPCSections = PCSections != nullptr; + bool HasDS = DS != nullptr; auto *Result = new (Allocator.Allocate( - totalSizeToAlloc( + totalSizeToAlloc( MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, - HasHeapAllocMarker + HasPCSections + HasMMRAs, HasCFIType), + HasHeapAllocMarker + HasPCSections + HasMMRAs, HasCFIType, HasDS), alignof(ExtraInfo))) ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, - HasHeapAllocMarker, HasPCSections, HasCFIType, HasMMRAs); + HasHeapAllocMarker, HasPCSections, HasCFIType, HasMMRAs, + HasDS); // Copy the actual data into the trailing objects. llvm::copy(MMOs, Result->getTrailingObjects()); @@ -202,6 +206,8 @@ class MachineInstr Result->getTrailingObjects()[0] = CFIType; if (HasMMRAs) Result->getTrailingObjects()[MDNodeIdx++] = MMRAs; + if (HasDS) + Result->getTrailingObjects()[0] = DS; return Result; } @@ -240,6 +246,10 @@ class MachineInstr : nullptr; } + Value *getDeactivationSymbol() const { + return HasDS ? getTrailingObjects()[0] : 0; + } + private: friend TrailingObjects; @@ -255,6 +265,7 @@ class MachineInstr const bool HasPCSections; const bool HasCFIType; const bool HasMMRAs; + const bool HasDS; // Implement the `TrailingObjects` internal API. size_t numTrailingObjects(OverloadToken) const { @@ -269,16 +280,17 @@ class MachineInstr size_t numTrailingObjects(OverloadToken) const { return HasCFIType; } + size_t numTrailingObjects(OverloadToken) const { return HasDS; } // Just a boring constructor to allow us to initialize the sizes. Always use // the `create` routine above. ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, bool HasHeapAllocMarker, bool HasPCSections, bool HasCFIType, - bool HasMMRAs) + bool HasMMRAs, bool HasDS) : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), HasPostInstrSymbol(HasPostInstrSymbol), HasHeapAllocMarker(HasHeapAllocMarker), HasPCSections(HasPCSections), - HasCFIType(HasCFIType), HasMMRAs(HasMMRAs) {} + HasCFIType(HasCFIType), HasMMRAs(HasMMRAs), HasDS(HasDS) {} }; /// Enumeration of the kinds of inline extra info available. It is important @@ -867,6 +879,14 @@ class MachineInstr return nullptr; } + Value *getDeactivationSymbol() const { + if (!Info) + return nullptr; + if (ExtraInfo *EI = Info.get()) + return EI->getDeactivationSymbol(); + return nullptr; + } + /// Helper to extract a CFI type hash if one has been added. uint32_t getCFIType() const { if (!Info) @@ -1969,6 +1989,8 @@ class MachineInstr /// Set the CFI type for the instruction. LLVM_ABI void setCFIType(MachineFunction &MF, uint32_t Type); + LLVM_ABI void setDeactivationSymbol(MachineFunction &MF, Value *DS); + /// Return the MIFlags which represent both MachineInstrs. This /// should be used when merging two MachineInstrs into one. This routine does /// not modify the MIFlags of this MachineInstr. @@ -2088,7 +2110,7 @@ class MachineInstr void setExtraInfo(MachineFunction &MF, ArrayRef MMOs, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker, MDNode *PCSections, - uint32_t CFIType, MDNode *MMRAs); + uint32_t CFIType, MDNode *MMRAs, Value *DS); }; /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index e705d7d99544c..caeb430d6fd1c 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -70,29 +70,44 @@ enum { } // end namespace RegState /// Set of metadata that should be preserved when using BuildMI(). This provides -/// a more convenient way of preserving DebugLoc, PCSections and MMRA. +/// a more convenient way of preserving certain data from the original +/// instruction. class MIMetadata { public: MIMetadata() = default; - MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) - : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA) {} + MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr, + Value *DeactivationSymbol = nullptr) + : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA), + DeactivationSymbol(DeactivationSymbol) {} MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) : DL(DI), PCSections(PCSections), MMRA(MMRA) {} explicit MIMetadata(const Instruction &From) : DL(From.getDebugLoc()), - PCSections(From.getMetadata(LLVMContext::MD_pcsections)) {} + PCSections(From.getMetadata(LLVMContext::MD_pcsections)), + DeactivationSymbol(getDeactivationSymbol(&From)) {} explicit MIMetadata(const MachineInstr &From) - : DL(From.getDebugLoc()), PCSections(From.getPCSections()) {} + : DL(From.getDebugLoc()), PCSections(From.getPCSections()), + DeactivationSymbol(From.getDeactivationSymbol()) {} const DebugLoc &getDL() const { return DL; } MDNode *getPCSections() const { return PCSections; } MDNode *getMMRAMetadata() const { return MMRA; } + Value *getDeactivationSymbol() const { return DeactivationSymbol; } private: DebugLoc DL; MDNode *PCSections = nullptr; MDNode *MMRA = nullptr; + Value *DeactivationSymbol = nullptr; + + static inline Value *getDeactivationSymbol(const Instruction *I) { + if (auto *CB = dyn_cast(I)) + if (auto Bundle = + CB->getOperandBundle(llvm::LLVMContext::OB_deactivation_symbol)) + return Bundle->Inputs[0].get(); + return nullptr; + } }; class MachineInstrBuilder { @@ -348,6 +363,8 @@ class MachineInstrBuilder { MI->setPCSections(*MF, MIMD.getPCSections()); if (MIMD.getMMRAMetadata()) MI->setMMRAMetadata(*MF, MIMD.getMMRAMetadata()); + if (MIMD.getDeactivationSymbol()) + MI->setDeactivationSymbol(*MF, MIMD.getDeactivationSymbol()); return *this; } diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index b024e8a68bd6e..501cbc947132e 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -759,6 +759,7 @@ class SelectionDAG { int64_t offset = 0, unsigned TargetFlags = 0) { return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); } + LLVM_ABI SDValue getDeactivationSymbol(const GlobalValue *GV); LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); SDValue getTargetFrameIndex(int FI, EVT VT) { return getFrameIndex(FI, VT, true); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index c5cdf76f4777e..7add717227963 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -151,6 +151,7 @@ class SelectionDAGISel { OPC_RecordChild7, OPC_RecordMemRef, OPC_CaptureGlueInput, + OPC_CaptureDeactivationSymbol, OPC_MoveChild, OPC_MoveChild0, OPC_MoveChild1, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index cfc8a4243e894..aa72e81b2ab54 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -2005,6 +2005,22 @@ class GlobalAddressSDNode : public SDNode { } }; +class DeactivationSymbolSDNode : public SDNode { + friend class SelectionDAG; + + const GlobalValue *TheGlobal; + + DeactivationSymbolSDNode(const GlobalValue *GV, SDVTList VTs) + : SDNode(ISD::DEACTIVATION_SYMBOL, 0, DebugLoc(), VTs), TheGlobal(GV) {} + +public: + const GlobalValue *getGlobal() const { return TheGlobal; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::DEACTIVATION_SYMBOL; + } +}; + class FrameIndexSDNode : public SDNode { friend class SelectionDAG; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 7df5d8a09f0f6..b2697c81fd825 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4765,6 +4765,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { SmallVector InVals; const ConstantInt *CFIType = nullptr; SDValue ConvergenceControlToken; + GlobalValue *DeactivationSymbol = nullptr; std::optional PAI; @@ -4918,6 +4919,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { return *this; } + CallLoweringInfo &setDeactivationSymbol(GlobalValue *Sym) { + DeactivationSymbol = Sym; + return *this; + } + ArgListTy &getArgs() { return Args; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h index 0b46c7fb1f445..93412d9d22f8c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h @@ -338,9 +338,9 @@ template class WaitingOnGraph { // incorporate NewSNs. std::vector> ReadyNodes, FailedNodes; processReadyOrFailed(ModifiedPendingSNs, ReadyNodes, FailedNodes, - SuperNodeDeps, ElemToPendingSN, FailedSNs); + SuperNodeDeps, FailedSNs, &ElemToPendingSN); processReadyOrFailed(NewSNs, ReadyNodes, FailedNodes, SuperNodeDeps, - ElemToNewSN, FailedSNs); + FailedSNs, nullptr); CoalesceToPendingSNs.coalesce(ModifiedPendingSNs, ElemToPendingSN); CoalesceToPendingSNs.coalesce(NewSNs, ElemToPendingSN); @@ -591,8 +591,11 @@ template class WaitingOnGraph { std::vector> &Ready, std::vector> &Failed, SuperNodeDepsMap &SuperNodeDeps, - ElemToSuperNodeMap &ElemToSNs, - const std::vector &FailedSNs) { + const std::vector &FailedSNs, + ElemToSuperNodeMap *ElemToSNs) { + + SmallVector ToRemoveFromElemToSNs; + for (size_t I = 0; I != SNs.size();) { auto &SN = SNs[I]; @@ -609,6 +612,8 @@ template class WaitingOnGraph { bool SNReady = SN->Deps.empty(); if (SNReady || SNFailed) { + if (ElemToSNs) + ToRemoveFromElemToSNs.push_back(SN.get()); auto &NodeList = SNFailed ? Failed : Ready; NodeList.push_back(std::move(SN)); std::swap(SN, SNs.back()); @@ -616,6 +621,15 @@ template class WaitingOnGraph { } else ++I; } + + // Update ElemToSNs (if passed) to remove elements pointing at SN. + for (auto *SN : ToRemoveFromElemToSNs) { + for (auto &[Container, Elems] : SN->defs()) { + auto &Row = (*ElemToSNs)[Container]; + for (auto &Elem : Elems) + Row.erase(Elem); + } + } } std::vector> PendingSNs; diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index e06e6adbc3130..e3f2eb9fa44b8 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -1033,10 +1033,10 @@ class ConstantPtrAuth final : public Constant { friend struct ConstantPtrAuthKeyType; friend class Constant; - constexpr static IntrusiveOperandsAllocMarker AllocMarker{4}; + constexpr static IntrusiveOperandsAllocMarker AllocMarker{5}; ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, - Constant *AddrDisc); + Constant *AddrDisc, Constant *DeactivationSymbol); void *operator new(size_t s) { return User::operator new(s, AllocMarker); } @@ -1046,7 +1046,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. LLVM_ABI static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// Produce a new ptrauth expression signing the given value using /// the same schema as is stored in one. @@ -1078,6 +1079,10 @@ class ConstantPtrAuth final : public Constant { return !getAddrDiscriminator()->isNullValue(); } + Constant *getDeactivationSymbol() const { + return cast(Op<4>().get()); + } + /// A constant value for the address discriminator which has special /// significance to ctors/dtors lowering. Regular address discrimination can't /// be applied for them since uses of llvm.global_{c|d}tors are disallowed @@ -1106,7 +1111,7 @@ class ConstantPtrAuth final : public Constant { template <> struct OperandTraits - : public FixedNumOperandTraits {}; + : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPtrAuth, Constant) diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index 5972dcb637dfa..d938f4609742b 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -98,7 +98,8 @@ class LLVMContext { OB_kcfi = 8, // "kcfi" OB_convergencectrl = 9, // "convergencectrl" OB_align = 10, // "align" - OB_LastBundleID = OB_align // Marker for last bundle ID + OB_deactivation_symbol = 11, // "deactivation-symbol" + OB_LastBundleID = OB_deactivation_symbol }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h index 6f682a7059d10..2fe923f6c3866 100644 --- a/llvm/include/llvm/SandboxIR/Constant.h +++ b/llvm/include/llvm/SandboxIR/Constant.h @@ -1363,7 +1363,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. LLVM_ABI static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// The pointer that is signed in this ptrauth signed pointer. LLVM_ABI Constant *getPointer() const; @@ -1378,6 +1379,8 @@ class ConstantPtrAuth final : public Constant { /// the only global-initializer user of the ptrauth signed pointer. LLVM_ABI Constant *getAddrDiscriminator() const; + Constant *getDeactivationSymbol() const; + /// Whether there is any non-null address discriminator. bool hasAddressDiscriminator() const { return cast(Val)->hasAddressDiscriminator(); diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 96a7d7c2091d2..54162dc6bb30f 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -694,6 +694,7 @@ class Instruction : InstructionEncoding { // If so, make sure to override // TargetInstrInfo::getInsertSubregLikeInputs. bit variadicOpsAreDefs = false; // Are variadic operands definitions? + bit supportsDeactivationSymbol = false; // Does the instruction have side effects that are not captured by any // operands of the instruction or other flags? diff --git a/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h index 8e7df5e6b10f0..e4bfcd395c2d6 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h +++ b/llvm/include/llvm/Transforms/Instrumentation/BoundsChecking.h @@ -11,6 +11,7 @@ #include "llvm/IR/PassManager.h" #include "llvm/Support/Compiler.h" +#include "llvm/TargetParser/Triple.h" #include namespace llvm { @@ -23,10 +24,12 @@ class BoundsCheckingPass : public PassInfoMixin { public: struct Options { struct Runtime { - Runtime(bool MinRuntime, bool MayReturn) - : MinRuntime(MinRuntime), MayReturn(MayReturn) {} + Runtime(bool MinRuntime, bool MayReturn, bool HandlerPreserveAllRegs) + : MinRuntime(MinRuntime), MayReturn(MayReturn), + HandlerPreserveAllRegs(HandlerPreserveAllRegs) {} bool MinRuntime; bool MayReturn; + bool HandlerPreserveAllRegs; }; std::optional Rt; // Trap if empty. bool Merge = false; diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 5528b6f5b3b13..29b41b78fd06a 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -4260,11 +4260,13 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { } case lltok::kw_ptrauth: { // ValID ::= 'ptrauth' '(' ptr @foo ',' i32 - // (',' i64 (',' ptr addrdisc)? )? ')' + // (',' i64 (',' ptr addrdisc (',' ptr ds)? + // )? )? ')' Lex.Lex(); Constant *Ptr, *Key; - Constant *Disc = nullptr, *AddrDisc = nullptr; + Constant *Disc = nullptr, *AddrDisc = nullptr, + *DeactivationSymbol = nullptr; if (parseToken(lltok::lparen, "expected '(' in constant ptrauth expression") || @@ -4273,11 +4275,14 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { "expected comma in constant ptrauth expression") || parseGlobalTypeAndValue(Key)) return true; - // If present, parse the optional disc/addrdisc. - if (EatIfPresent(lltok::comma)) - if (parseGlobalTypeAndValue(Disc) || - (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc))) - return true; + // If present, parse the optional disc/addrdisc/ds. + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(Disc)) + return true; + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc)) + return true; + if (EatIfPresent(lltok::comma) && + parseGlobalTypeAndValue(DeactivationSymbol)) + return true; if (parseToken(lltok::rparen, "expected ')' in constant ptrauth expression")) return true; @@ -4308,7 +4313,15 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { AddrDisc = ConstantPointerNull::get(PointerType::get(Context, 0)); } - ID.ConstantVal = ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc); + if (!DeactivationSymbol) + DeactivationSymbol = + ConstantPointerNull::get(PointerType::get(Context, 0)); + if (!DeactivationSymbol->getType()->isPointerTy()) + return error(ID.Loc, + "constant ptrauth deactivation symbol must be a pointer"); + + ID.ConstantVal = + ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc, DeactivationSymbol); ID.Kind = ValID::t_Constant; return false; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 034d973f307f8..3d3f70053e473 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1609,7 +1609,16 @@ Expected BitcodeReader::materializeValue(unsigned StartValID, if (!Disc) return error("ptrauth disc operand must be ConstantInt"); - C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3]); + Constant *DeactivationSymbol = + ConstOps.size() > 4 ? ConstOps[4] + : ConstantPointerNull::get(cast( + ConstOps[3]->getType())); + if (!DeactivationSymbol->getType()->isPointerTy()) + return error( + "ptrauth deactivation symbol operand must be a pointer"); + + C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3], + DeactivationSymbol); break; } case BitcodeConstant::NoCFIOpcode: { @@ -3815,6 +3824,16 @@ Error BitcodeReader::parseConstants() { (unsigned)Record[2], (unsigned)Record[3]}); break; } + case bitc::CST_CODE_PTRAUTH2: { + if (Record.size() < 5) + return error("Invalid ptrauth record"); + // Ptr, Key, Disc, AddrDisc, DeactivationSymbol + V = BitcodeConstant::create( + Alloc, CurTy, BitcodeConstant::ConstantPtrAuthOpcode, + {(unsigned)Record[0], (unsigned)Record[1], (unsigned)Record[2], + (unsigned)Record[3], (unsigned)Record[4]}); + break; + } } assert(V->getType() == getTypeByID(CurTyID) && "Incorrect result type ID"); diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 72218bfa95834..0992e1f144a6a 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3114,11 +3114,12 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, Record.push_back(VE.getTypeID(NC->getGlobalValue()->getType())); Record.push_back(VE.getValueID(NC->getGlobalValue())); } else if (const auto *CPA = dyn_cast(C)) { - Code = bitc::CST_CODE_PTRAUTH; + Code = bitc::CST_CODE_PTRAUTH2; Record.push_back(VE.getValueID(CPA->getPointer())); Record.push_back(VE.getValueID(CPA->getKey())); Record.push_back(VE.getValueID(CPA->getDiscriminator())); Record.push_back(VE.getValueID(CPA->getAddrDiscriminator())); + Record.push_back(VE.getValueID(CPA->getDeactivationSymbol())); } else { #ifndef NDEBUG C->dump(); diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 7be7468300569..e2ed45eec0ecd 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -196,6 +196,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); } + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + Info.DeactivationSymbol = cast(Bundle->Inputs[0]); + } + Info.CB = &CB; Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); Info.CallConv = CallConv; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index f3fb0c6bb569d..88e554244c663 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2920,6 +2920,9 @@ bool IRTranslator::translateIntrinsic( } } + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) + MIB->setDeactivationSymbol(*MF, Bundle->Inputs[0].get()); + return true; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 637acd61c8a5f..3906b311addf0 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -38,8 +38,10 @@ void MachineIRBuilder::setMF(MachineFunction &MF) { //------------------------------------------------------------------------------ MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) { - return BuildMI(getMF(), {getDL(), getPCSections(), getMMRAMetadata()}, - getTII().get(Opcode)); + return BuildMI( + getMF(), + {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()}, + getTII().get(Opcode)); } MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) { diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp index 8ed590669a3b0..0deee84d374a6 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -286,6 +286,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) { .Case("heap-alloc-marker", MIToken::kw_heap_alloc_marker) .Case("pcsections", MIToken::kw_pcsections) .Case("cfi-type", MIToken::kw_cfi_type) + .Case("deactivation-symbol", MIToken::kw_deactivation_symbol) .Case("bbsections", MIToken::kw_bbsections) .Case("bb_id", MIToken::kw_bb_id) .Case("unknown-size", MIToken::kw_unknown_size) diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h index abac1880f94e0..483eadddc1cf4 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.h +++ b/llvm/lib/CodeGen/MIRParser/MILexer.h @@ -140,6 +140,7 @@ struct MIToken { kw_heap_alloc_marker, kw_pcsections, kw_cfi_type, + kw_deactivation_symbol, kw_bbsections, kw_bb_id, kw_unknown_size, diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp index 444075cf01908..369bbdae8907a 100644 --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -1073,6 +1073,7 @@ bool MIParser::parse(MachineInstr *&MI) { Token.isNot(MIToken::kw_heap_alloc_marker) && Token.isNot(MIToken::kw_pcsections) && Token.isNot(MIToken::kw_cfi_type) && + Token.isNot(MIToken::kw_deactivation_symbol) && Token.isNot(MIToken::kw_debug_location) && Token.isNot(MIToken::kw_debug_instr_number) && Token.isNot(MIToken::coloncolon) && Token.isNot(MIToken::lbrace)) { @@ -1121,6 +1122,14 @@ bool MIParser::parse(MachineInstr *&MI) { lex(); } + GlobalValue *DS = nullptr; + if (Token.is(MIToken::kw_deactivation_symbol)) { + lex(); + if (parseGlobalValue(DS)) + return true; + lex(); + } + unsigned InstrNum = 0; if (Token.is(MIToken::kw_debug_instr_number)) { lex(); @@ -1197,6 +1206,8 @@ bool MIParser::parse(MachineInstr *&MI) { MI->setPCSections(MF, PCSections); if (CFIType) MI->setCFIType(MF, CFIType); + if (DS) + MI->setDeactivationSymbol(MF, DS); if (!MemOperands.empty()) MI->setMemRefs(MF, MemOperands); if (InstrNum) diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 1d54d72336860..c0554497653f8 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MIRFormatter.h" #include "llvm/CodeGen/MIRYamlMapping.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" @@ -895,6 +896,10 @@ static void printMI(raw_ostream &OS, MFPrintState &State, } if (uint32_t CFIType = MI.getCFIType()) OS << LS << "cfi-type " << CFIType; + if (Value *DS = MI.getDeactivationSymbol()) { + OS << LS << "deactivation-symbol "; + MIRFormatter::printIRValue(OS, *DS, State.MST); + } if (auto Num = MI.peekDebugInstrNum()) OS << LS << "debug-instr-number " << Num; diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index 92aa094da26d9..4c2546cc88c76 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -620,10 +620,10 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfo( ArrayRef MMOs, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker, MDNode *PCSections, - uint32_t CFIType, MDNode *MMRAs) { + uint32_t CFIType, MDNode *MMRAs, Value *DS) { return MachineInstr::ExtraInfo::create(Allocator, MMOs, PreInstrSymbol, PostInstrSymbol, HeapAllocMarker, - PCSections, CFIType, MMRAs); + PCSections, CFIType, MMRAs, DS); } const char *MachineFunction::createExternalSymbolName(StringRef Name) { diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 5014ac7cdd58f..ff01763fce60d 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -322,15 +322,17 @@ void MachineInstr::setExtraInfo(MachineFunction &MF, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker, MDNode *PCSections, - uint32_t CFIType, MDNode *MMRAs) { + uint32_t CFIType, MDNode *MMRAs, Value *DS) { bool HasPreInstrSymbol = PreInstrSymbol != nullptr; bool HasPostInstrSymbol = PostInstrSymbol != nullptr; bool HasHeapAllocMarker = HeapAllocMarker != nullptr; bool HasPCSections = PCSections != nullptr; bool HasCFIType = CFIType != 0; bool HasMMRAs = MMRAs != nullptr; + bool HasDS = DS != nullptr; int NumPointers = MMOs.size() + HasPreInstrSymbol + HasPostInstrSymbol + - HasHeapAllocMarker + HasPCSections + HasCFIType + HasMMRAs; + HasHeapAllocMarker + HasPCSections + HasCFIType + HasMMRAs + + HasDS; // Drop all extra info if there is none. if (NumPointers <= 0) { @@ -343,10 +345,10 @@ void MachineInstr::setExtraInfo(MachineFunction &MF, // 32-bit pointers. // FIXME: Maybe we should make the symbols in the extra info mutable? else if (NumPointers > 1 || HasMMRAs || HasHeapAllocMarker || HasPCSections || - HasCFIType) { + HasCFIType || HasDS) { Info.set( MF.createMIExtraInfo(MMOs, PreInstrSymbol, PostInstrSymbol, - HeapAllocMarker, PCSections, CFIType, MMRAs)); + HeapAllocMarker, PCSections, CFIType, MMRAs, DS)); return; } @@ -365,7 +367,7 @@ void MachineInstr::dropMemRefs(MachineFunction &MF) { setExtraInfo(MF, {}, getPreInstrSymbol(), getPostInstrSymbol(), getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setMemRefs(MachineFunction &MF, @@ -377,7 +379,7 @@ void MachineInstr::setMemRefs(MachineFunction &MF, setExtraInfo(MF, MMOs, getPreInstrSymbol(), getPostInstrSymbol(), getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::addMemOperand(MachineFunction &MF, @@ -488,7 +490,7 @@ void MachineInstr::setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { setExtraInfo(MF, memoperands(), Symbol, getPostInstrSymbol(), getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { @@ -504,7 +506,7 @@ void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { setExtraInfo(MF, memoperands(), getPreInstrSymbol(), Symbol, getHeapAllocMarker(), getPCSections(), getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setHeapAllocMarker(MachineFunction &MF, MDNode *Marker) { @@ -513,7 +515,8 @@ void MachineInstr::setHeapAllocMarker(MachineFunction &MF, MDNode *Marker) { return; setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), - Marker, getPCSections(), getCFIType(), getMMRAMetadata()); + Marker, getPCSections(), getCFIType(), getMMRAMetadata(), + getDeactivationSymbol()); } void MachineInstr::setPCSections(MachineFunction &MF, MDNode *PCSections) { @@ -523,7 +526,7 @@ void MachineInstr::setPCSections(MachineFunction &MF, MDNode *PCSections) { setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), getHeapAllocMarker(), PCSections, getCFIType(), - getMMRAMetadata()); + getMMRAMetadata(), getDeactivationSymbol()); } void MachineInstr::setCFIType(MachineFunction &MF, uint32_t Type) { @@ -532,7 +535,8 @@ void MachineInstr::setCFIType(MachineFunction &MF, uint32_t Type) { return; setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), - getHeapAllocMarker(), getPCSections(), Type, getMMRAMetadata()); + getHeapAllocMarker(), getPCSections(), Type, getMMRAMetadata(), + getDeactivationSymbol()); } void MachineInstr::setMMRAMetadata(MachineFunction &MF, MDNode *MMRAs) { @@ -541,7 +545,18 @@ void MachineInstr::setMMRAMetadata(MachineFunction &MF, MDNode *MMRAs) { return; setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), - getHeapAllocMarker(), getPCSections(), getCFIType(), MMRAs); + getHeapAllocMarker(), getPCSections(), getCFIType(), MMRAs, + getDeactivationSymbol()); +} + +void MachineInstr::setDeactivationSymbol(MachineFunction &MF, Value *DS) { + // Do nothing if old and new symbols are the same. + if (DS == getDeactivationSymbol()) + return; + + setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), + getHeapAllocMarker(), getPCSections(), getCFIType(), + getMMRAMetadata(), DS); } void MachineInstr::cloneInstrSymbols(MachineFunction &MF, @@ -730,6 +745,8 @@ bool MachineInstr::isIdenticalTo(const MachineInstr &Other, // Call instructions with different CFI types are not identical. if (isCall() && getCFIType() != Other.getCFIType()) return false; + if (getDeactivationSymbol() != Other.getDeactivationSymbol()) + return false; return true; } @@ -2041,6 +2058,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << ','; OS << " cfi-type " << CFIType; } + if (getDeactivationSymbol()) + OS << ", deactivation-symbol " << getDeactivationSymbol()->getName(); if (DebugInstrNum) { if (!FirstOp) diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 62fd5bc260397..bfcbeeaee6e87 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -15,10 +15,12 @@ #include "InstrEmitter.h" #include "SDNodeDbgValue.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -61,6 +63,8 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses, unsigned N = Node->getNumOperands(); while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) --N; + if (N && Node->getOperand(N - 1).getOpcode() == ISD::DEACTIVATION_SYMBOL) + --N; // Ignore deactivation symbol if it exists. if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) --N; // Ignore chain if it exists. @@ -1216,15 +1220,23 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, } } - if (SDNode *GluedNode = Node->getGluedNode()) { - // FIXME: Possibly iterate over multiple glue nodes? - if (GluedNode->getOpcode() == - ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { - Register VReg = getVR(GluedNode->getOperand(0), VRBaseMap); - MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, - /*isImp=*/true); - MIB->addOperand(MO); - } + unsigned Op = Node->getNumOperands(); + if (Op != 0 && Node->getOperand(Op - 1)->getOpcode() == + ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { + Register VReg = getVR(Node->getOperand(Op - 1)->getOperand(0), VRBaseMap); + MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, + /*isImp=*/true); + MIB->addOperand(MO); + Op--; + } + + if (Op != 0 && + Node->getOperand(Op - 1)->getOpcode() == ISD::DEACTIVATION_SYMBOL) { + MI->setDeactivationSymbol( + *MF, const_cast( + cast(Node->getOperand(Op - 1)) + ->getGlobal())); + Op--; } // Run post-isel target hook to adjust this instruction if needed. @@ -1245,7 +1257,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, llvm_unreachable("This target-independent node should have been selected!"); case ISD::EntryToken: case ISD::MERGE_VALUES: - case ISD::TokenFactor: // fall thru + case ISD::TokenFactor: + case ISD::DEACTIVATION_SYMBOL: break; case ISD::CopyToReg: { Register DestReg = cast(Node->getOperand(1))->getReg(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 56909bb8631c1..42786db653fa5 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1916,6 +1916,21 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, return SDValue(N, 0); } +SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) { + SDVTList VTs = getVTList(MVT::Untyped); + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::DEACTIVATION_SYMBOL, VTs, {}); + ID.AddPointer(GV); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) + return SDValue(E, 0); + + auto *N = newSDNode(GV, VTs); + CSEMap.InsertNode(N, IP); + InsertNode(N); + return SDValue(N, 0); +} + SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; SDVTList VTs = getVTList(VT); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 86852f2e44633..3b7db2c54bae0 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -45,6 +45,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" @@ -5376,6 +5377,14 @@ SmallVector SelectionDAGBuilder::getTargetIntrinsicOperands( } } + if (std::optional Bundle = + I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + auto *Sym = Bundle->Inputs[0].get(); + SDValue SDSym = getValue(Sym); + SDSym = DAG.getDeactivationSymbol(cast(Sym)); + Ops.push_back(SDSym); + } + if (std::optional Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { Value *Token = Bundle->Inputs[0].get(); @@ -9109,6 +9118,11 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, ConvControlToken = getValue(Token); } + GlobalValue *DeactivationSymbol = nullptr; + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + DeactivationSymbol = cast(Bundle->Inputs[0].get()); + } + TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) @@ -9118,7 +9132,8 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, .setIsPreallocated( CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) .setCFIType(CFIType) - .setConvergenceControlToken(ConvControlToken); + .setConvergenceControlToken(ConvControlToken) + .setDeactivationSymbol(DeactivationSymbol); // Set the pointer authentication info if we have it. if (PAI) { @@ -9738,7 +9753,7 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}); + LLVMContext::OB_convergencectrl, LLVMContext::OB_deactivation_symbol}); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 0fad4722b1871..dd8f18d3b8a6a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -3308,6 +3308,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::LIFETIME_START: case ISD::LIFETIME_END: case ISD::PSEUDO_PROBE: + case ISD::DEACTIVATION_SYMBOL: NodeToMatch->setNodeId(-1); // Mark selected. return; case ISD::AssertSext: @@ -3389,7 +3390,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // These are the current input chain and glue for use when generating nodes. // Various Emit operations change these. For example, emitting a copytoreg // uses and updates these. - SDValue InputChain, InputGlue; + SDValue InputChain, InputGlue, DeactivationSymbol; // ChainNodesMatched - If a pattern matches nodes that have input/output // chains, the OPC_EmitMergeInputChains operation is emitted which indicates @@ -3542,6 +3543,15 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, InputGlue = N->getOperand(N->getNumOperands()-1); continue; + case OPC_CaptureDeactivationSymbol: + // If the current node has a deactivation symbol, capture it in + // DeactivationSymbol. + if (N->getNumOperands() != 0 && + N->getOperand(N->getNumOperands() - 1).getOpcode() == + ISD::DEACTIVATION_SYMBOL) + DeactivationSymbol = N->getOperand(N->getNumOperands() - 1); + continue; + case OPC_MoveChild: { unsigned ChildNo = MatcherTable[MatcherIndex++]; if (ChildNo >= N.getNumOperands()) @@ -4223,6 +4233,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // If this has chain/glue inputs, add them. if (EmitNodeInfo & OPFL_Chain) Ops.push_back(InputChain); + if (DeactivationSymbol.getNode() != nullptr) + Ops.push_back(DeactivationSymbol); if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr) Ops.push_back(InputGlue); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 97f211ee5ba82..246692e2cdf77 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1674,12 +1674,14 @@ static void writeConstantInternal(raw_ostream &Out, const Constant *CV, if (const auto *CPA = dyn_cast(CV)) { Out << "ptrauth ("; - // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?) + // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?) unsigned NumOpsToWrite = 2; if (!CPA->getOperand(2)->isNullValue()) NumOpsToWrite = 3; if (!CPA->getOperand(3)->isNullValue()) NumOpsToWrite = 4; + if (!CPA->getOperand(4)->isNullValue()) + NumOpsToWrite = 5; ListSeparator LS; for (unsigned i = 0, e = NumOpsToWrite; i != e; ++i) { diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index a3aa5e9571657..6b82da140256f 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -2081,28 +2081,33 @@ Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) { // ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { - Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc}; + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { + Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc, DeactivationSymbol}; ConstantPtrAuthKeyType MapKey(ArgVec); LLVMContextImpl *pImpl = Ptr->getContext().pImpl; return pImpl->ConstantPtrAuths.getOrCreate(Ptr->getType(), MapKey); } ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { - return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator()); + return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator(), + getDeactivationSymbol()); } ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) : Constant(Ptr->getType(), Value::ConstantPtrAuthVal, AllocMarker) { assert(Ptr->getType()->isPointerTy()); assert(Key->getBitWidth() == 32); assert(Disc->getBitWidth() == 64); assert(AddrDisc->getType()->isPointerTy()); + assert(DeactivationSymbol->getType()->isPointerTy()); setOperand(0, Ptr); setOperand(1, Key); setOperand(2, Disc); setOperand(3, AddrDisc); + setOperand(4, DeactivationSymbol); } /// Remove the constant from the constant table. @@ -2150,6 +2155,11 @@ bool ConstantPtrAuth::hasSpecialAddressDiscriminator(uint64_t Value) const { bool ConstantPtrAuth::isKnownCompatibleWith(const Value *Key, const Value *Discriminator, const DataLayout &DL) const { + // This function may only be validly called to analyze a ptrauth operation + // with no deactivation symbol, so if we have one it isn't compatible. + if (!getDeactivationSymbol()->isNullValue()) + return false; + // If the keys are different, there's no chance for this to be compatible. if (getKey() != Key) return false; diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index e3e8d895a63f4..2073e0d42d8e3 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -539,7 +539,8 @@ struct ConstantPtrAuthKeyType { ConstantPtrAuth *create(TypeClass *Ty) const { return new ConstantPtrAuth(Operands[0], cast(Operands[1]), - cast(Operands[2]), Operands[3]); + cast(Operands[2]), Operands[3], + Operands[4]); } }; diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 604730e0d3004..26c4f4ec784cd 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -1699,7 +1699,9 @@ LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key, LLVMValueRef Disc, LLVMValueRef AddrDisc) { return wrap(ConstantPtrAuth::get( unwrap(Ptr), unwrap(Key), - unwrap(Disc), unwrap(AddrDisc))); + unwrap(Disc), unwrap(AddrDisc), + ConstantPointerNull::get( + cast(unwrap(AddrDisc)->getType())))); } /*-- Opcode mapping */ diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index cd39970f5111f..85d3690dd8306 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -620,7 +620,8 @@ bool CallBase::hasReadingOperandBundles() const { // ptrauth) forces a callsite to be at least readonly. return hasOperandBundlesOtherThan({LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && getIntrinsicID() != Intrinsic::assume; } @@ -628,7 +629,8 @@ bool CallBase::hasClobberingOperandBundles() const { return hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && getIntrinsicID() != Intrinsic::assume; } diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index 335c210c10e1a..10aba759185a7 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -55,6 +55,8 @@ static StringRef knownBundleName(unsigned BundleTagID) { return "convergencectrl"; case LLVMContext::OB_align: return "align"; + case LLVMContext::OB_deactivation_symbol: + return "deactivation-symbol"; default: llvm_unreachable("unknown bundle id"); } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 67522022ede4d..fd49cd86504df 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -2748,6 +2748,14 @@ void Verifier::visitConstantPtrAuth(const ConstantPtrAuth *CPA) { Check(CPA->getDiscriminator()->getBitWidth() == 64, "signed ptrauth constant discriminator must be i64 constant integer"); + + Check(CPA->getDeactivationSymbol()->getType()->isPointerTy(), + "signed ptrauth constant deactivation symbol must be a pointer"); + + Check(isa(CPA->getDeactivationSymbol()) || + CPA->getDeactivationSymbol()->isNullValue(), + "signed ptrauth constant deactivation symbol must be a global value " + "or null"); } bool Verifier::verifyAttributeCount(AttributeList Attrs, unsigned Params) { diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp index 861c284253f7a..8e4a5ea5fc612 100644 --- a/llvm/lib/Object/Archive.cpp +++ b/llvm/lib/Object/Archive.cpp @@ -582,7 +582,8 @@ Expected Archive::Child::getBuffer() const { if (!FullNameOrErr) return FullNameOrErr.takeError(); const std::string &FullName = *FullNameOrErr; - ErrorOr> Buf = MemoryBuffer::getFile(FullName); + ErrorOr> Buf = + MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator=*/false); if (std::error_code EC = Buf.getError()) return errorCodeToError(EC); Parent->ThinBuffers.push_back(std::move(*Buf)); diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 0d190ea448931..f5281ea69b512 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1590,24 +1590,31 @@ parseBoundsCheckingOptions(StringRef Params) { Options.Rt = { /*MinRuntime=*/false, /*MayReturn=*/true, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "rt-abort") { Options.Rt = { /*MinRuntime=*/false, /*MayReturn=*/false, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "min-rt") { Options.Rt = { /*MinRuntime=*/true, /*MayReturn=*/true, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "min-rt-abort") { Options.Rt = { /*MinRuntime=*/true, /*MayReturn=*/false, + /*HandlerPreserveAllRegs=*/false, }; } else if (ParamName == "merge") { Options.Merge = true; + } else if (ParamName == "handler-preserve-all-regs") { + if (Options.Rt) + Options.Rt->HandlerPreserveAllRegs = true; } else { StringRef ParamEQ; StringRef Val; diff --git a/llvm/lib/SandboxIR/Constant.cpp b/llvm/lib/SandboxIR/Constant.cpp index 9de88ef2cf0a0..eb14797af081c 100644 --- a/llvm/lib/SandboxIR/Constant.cpp +++ b/llvm/lib/SandboxIR/Constant.cpp @@ -412,10 +412,12 @@ PointerType *NoCFIValue::getType() const { } ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { auto *LLVMC = llvm::ConstantPtrAuth::get( cast(Ptr->Val), cast(Key->Val), - cast(Disc->Val), cast(AddrDisc->Val)); + cast(Disc->Val), cast(AddrDisc->Val), + cast(DeactivationSymbol->Val)); return cast(Ptr->getContext().getOrCreateConstant(LLVMC)); } @@ -439,6 +441,11 @@ Constant *ConstantPtrAuth::getAddrDiscriminator() const { cast(Val)->getAddrDiscriminator()); } +Constant *ConstantPtrAuth::getDeactivationSymbol() const { + return Ctx.getOrCreateConstant( + cast(Val)->getDeactivationSymbol()); +} + ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { auto *LLVMC = cast(Val)->getWithSameSchema( cast(Pointer->Val)); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 3aa77bd47930f..8267414e78955 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -176,7 +176,12 @@ class AArch64AsmPrinter : public AsmPrinter { const MachineOperand *AUTAddrDisc, Register Scratch, std::optional PACKey, - uint64_t PACDisc, Register PACAddrDisc); + uint64_t PACDisc, Register PACAddrDisc, Value *DS); + + // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true + // if no instruction should be emitted because the deactivation symbol is + // defined in the current module so this function emitted a NOP instead. + bool emitDeactivationSymbolRelocation(Value *DS); // Emit the sequence for PAC. void emitPtrauthSign(const MachineInstr *MI); @@ -214,11 +219,9 @@ class AArch64AsmPrinter : public AsmPrinter { // authenticating) void LowerLOADgotAUTH(const MachineInstr &MI); - const MCExpr *emitPAuthRelocationAsIRelative(const MCExpr *Target, - uint16_t Disc, - AArch64PACKey::ID KeyID, - bool HasAddressDiversity, - bool IsDSOLocal); + const MCExpr *emitPAuthRelocationAsIRelative( + const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr); /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. @@ -2113,11 +2116,31 @@ void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) { LRCheckMethod); } +bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) { + if (!DS) + return false; + + if (isa(DS)) { + // Just emit the nop directly. + EmitToStreamer(MCInstBuilder(AArch64::HINT).addImm(0)); + return true; + } + MCSymbol *Dot = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Dot); + const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Dot, OutContext); + + const MCExpr *DSExpr = MCSymbolRefExpr::create( + OutContext.getOrCreateSymbol(DS->getName()), OutContext); + OutStreamer->emitRelocDirective(*DeactDotExpr, "R_AARCH64_PATCHINST", DSExpr, + SMLoc()); + return false; +} + void AArch64AsmPrinter::emitPtrauthAuthResign( Register AUTVal, AArch64PACKey::ID AUTKey, uint64_t AUTDisc, const MachineOperand *AUTAddrDisc, Register Scratch, std::optional PACKey, uint64_t PACDisc, - Register PACAddrDisc) { + Register PACAddrDisc, Value *DS) { const bool IsAUTPAC = PACKey.has_value(); // We expand AUT/AUTPAC into a sequence of the form @@ -2164,15 +2187,17 @@ void AArch64AsmPrinter::emitPtrauthAuthResign( bool AUTZero = AUTDiscReg == AArch64::XZR; unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero); - // autiza x16 ; if AUTZero - // autia x16, x17 ; if !AUTZero - MCInst AUTInst; - AUTInst.setOpcode(AUTOpc); - AUTInst.addOperand(MCOperand::createReg(AUTVal)); - AUTInst.addOperand(MCOperand::createReg(AUTVal)); - if (!AUTZero) - AUTInst.addOperand(MCOperand::createReg(AUTDiscReg)); - EmitToStreamer(*OutStreamer, AUTInst); + if (!emitDeactivationSymbolRelocation(DS)) { + // autiza x16 ; if AUTZero + // autia x16, x17 ; if !AUTZero + MCInst AUTInst; + AUTInst.setOpcode(AUTOpc); + AUTInst.addOperand(MCOperand::createReg(AUTVal)); + AUTInst.addOperand(MCOperand::createReg(AUTVal)); + if (!AUTZero) + AUTInst.addOperand(MCOperand::createReg(AUTDiscReg)); + EmitToStreamer(*OutStreamer, AUTInst); + } // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done. if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap)) @@ -2236,6 +2261,9 @@ void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) { bool IsZeroDisc = DiscReg == AArch64::XZR; unsigned Opc = getPACOpcodeForKey(Key, IsZeroDisc); + if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol())) + return; + // paciza x16 ; if IsZeroDisc // pacia x16, x17 ; if !IsZeroDisc MCInst PACInst; @@ -2356,15 +2384,17 @@ static void emitAddress(MCStreamer &Streamer, MCRegister Reg, } static bool targetSupportsPAuthRelocation(const Triple &TT, - const MCExpr *Target) { + const MCExpr *Target, + const MCExpr *DSExpr) { // No released version of glibc supports PAuth relocations. if (TT.isOSGlibc()) return false; // We emit PAuth constants as IRELATIVE relocations in cases where the // constant cannot be represented as a PAuth relocation: - // 1) The signed value is not a symbol. - return !isa(Target); + // 1) There is a deactivation symbol. + // 2) The signed value is not a symbol. + return !DSExpr && !isa(Target); } static bool targetSupportsIRelativeRelocation(const Triple &TT) { @@ -2417,14 +2447,27 @@ static bool targetSupportsIRelativeRelocation(const Triple &TT) { // mov x1, #12345 // b __emupac_pacda // .popsection +// +// Example (signed null pointer, not address discriminated, with deactivation +// symbol ds): +// +// .8byte .Lpauth_ifunc0 +// .pushsection .text.startup,"ax",@progbits +// .Lpauth_ifunc0: +// mov x0, #0 +// mov x1, #12345 +// .reloc ., R_AARCH64_PATCHINST, ds +// b __emupac_pacda +// ret +// .popsection const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, - bool HasAddressDiversity, bool IsDSOLocal) { + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) { const Triple &TT = TM.getTargetTriple(); // We only emit an IRELATIVE relocation if the target supports IRELATIVE and // does not support the kind of PAuth relocation that we are trying to emit. - if (targetSupportsPAuthRelocation(TT, Target) || + if (targetSupportsPAuthRelocation(TT, Target, DSExpr) || !targetSupportsIRelativeRelocation(TT)) return nullptr; @@ -2435,6 +2478,7 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( std::unique_ptr STI( TM.getTarget().createMCSubtargetInfo(TT, "", "")); assert(STI && "Unable to create subtarget info"); + this->STI = static_cast(&*STI); MCSymbol *Place = OutStreamer->getContext().createTempSymbol(); OutStreamer->emitLabel(Place); @@ -2468,6 +2512,16 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( emitMOVZ(AArch64::X1, Disc, 0); } + if (DSExpr) { + MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol(); + OutStreamer->emitLabel(PrePACInst); + + auto *PrePACInstExpr = + MCSymbolRefExpr::create(PrePACInst, OutStreamer->getContext()); + OutStreamer->emitRelocDirective(*PrePACInstExpr, "R_AARCH64_PATCHINST", + DSExpr, SMLoc()); + } + // We don't know the subtarget because this is being emitted for a global // initializer. Because the performance of IFUNC resolvers is unimportant, we // always call the EmuPAC runtime, which will end up using the PAC instruction @@ -2478,6 +2532,12 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( MCSymbolRefExpr::create(EmuPAC, OutStreamer->getContext()); OutStreamer->emitInstruction(MCInstBuilder(AArch64::B).addExpr(EmuPACRef), *STI); + + // We need a RET despite the above tail call because the deactivation symbol + // may replace the tail call with a NOP. + if (DSExpr) + OutStreamer->emitInstruction( + MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); OutStreamer->popSection(); return MCSymbolRefExpr::create(IRelativeSym, AArch64::S_FUNCINIT, @@ -2509,6 +2569,13 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { Sym = MCConstantExpr::create(Offset.getSExtValue(), Ctx); } + const MCExpr *DSExpr = nullptr; + if (auto *DS = dyn_cast(CPA.getDeactivationSymbol())) { + if (isa(DS)) + return Sym; + DSExpr = MCSymbolRefExpr::create(getSymbol(DS), Ctx); + } + uint64_t KeyID = CPA.getKey()->getZExtValue(); // We later rely on valid KeyID value in AArch64PACKeyIDToString call from // AArch64AuthMCExpr::printImpl, so fail fast. @@ -2529,9 +2596,13 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { // Check if we need to represent this with an IRELATIVE and emit it if so. if (auto *IFuncSym = emitPAuthRelocationAsIRelative( Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), - BaseGVB && BaseGVB->isDSOLocal())) + BaseGVB && BaseGVB->isDSOLocal(), DSExpr)) return IFuncSym; + if (DSExpr) + report_fatal_error("deactivation symbols unsupported in constant " + "expressions on this target"); + // Finally build the complete @AUTH expr. return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), Ctx); @@ -3136,17 +3207,18 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { } case AArch64::AUTx16x17: - emitPtrauthAuthResign(AArch64::X16, - (AArch64PACKey::ID)MI->getOperand(0).getImm(), - MI->getOperand(1).getImm(), &MI->getOperand(2), - AArch64::X17, std::nullopt, 0, 0); + emitPtrauthAuthResign( + AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(), + MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17, + std::nullopt, 0, 0, MI->getDeactivationSymbol()); return; case AArch64::AUTxMxN: emitPtrauthAuthResign(MI->getOperand(0).getReg(), (AArch64PACKey::ID)MI->getOperand(3).getImm(), MI->getOperand(4).getImm(), &MI->getOperand(5), - MI->getOperand(1).getReg(), std::nullopt, 0, 0); + MI->getOperand(1).getReg(), std::nullopt, 0, 0, + MI->getDeactivationSymbol()); return; case AArch64::AUTPAC: @@ -3154,7 +3226,8 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { AArch64::X16, (AArch64PACKey::ID)MI->getOperand(0).getImm(), MI->getOperand(1).getImm(), &MI->getOperand(2), AArch64::X17, (AArch64PACKey::ID)MI->getOperand(3).getImm(), - MI->getOperand(4).getImm(), MI->getOperand(5).getReg()); + MI->getOperand(4).getImm(), MI->getOperand(5).getReg(), + MI->getDeactivationSymbol()); return; case AArch64::PAC: @@ -3635,6 +3708,9 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { return; } + if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol())) + return; + // Finally, do the automated lowerings for everything else. MCInst TmpInst; MCInstLowering.Lower(MI, TmpInst); diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 08466667c0fa5..b721c1f533726 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -1557,7 +1557,10 @@ void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) { extractPtrauthBlendDiscriminators(AUTDisc, CurDAG); if (!Subtarget->isX16X17Safer()) { - SDValue Ops[] = {Val, AUTKey, AUTConstDisc, AUTAddrDisc}; + std::vector Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc}; + // Copy deactivation symbol if present. + if (N->getNumOperands() > 4) + Ops.push_back(N->getOperand(4)); SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTxMxN, DL, MVT::i64, MVT::i64, Ops); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d4099b56b6d6e..dd70d729ffc91 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -10203,6 +10203,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, if (InGlue.getNode()) Ops.push_back(InGlue); + if (CLI.DeactivationSymbol) + Ops.push_back(DAG.getDeactivationSymbol(CLI.DeactivationSymbol)); + // If we're doing a tall call, use a TC_RETURN here rather than an // actual call instruction. if (IsTailCall) { diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 6871c2d504cf6..61a8f764e39ed 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -2347,6 +2347,7 @@ class BImm pattern> let Inst{25-0} = addr; let DecoderMethod = "DecodeUnconditionalBranch"; + let supportsDeactivationSymbol = true; } class BranchImm pattern> @@ -2404,6 +2405,7 @@ class SignAuthOneData opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = Rn; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthZero opcode_prefix, bits<2> opcode, string asm, @@ -2417,6 +2419,7 @@ class SignAuthZero opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = 0b11111; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthTwoOperand opc, string asm, diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 03bad8ff8ac8a..b4d8649b31d6d 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -2215,6 +2215,7 @@ let Predicates = [HasPAuth] in { let Size = 12; let Defs = [X16, X17]; let usesCustomInserter = 1; + let supportsDeactivationSymbol = true; } // A standalone pattern is used, so that literal 0 can be passed as $Disc. diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index 55694efafeed1..7907a3c283624 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -1421,6 +1421,7 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, } else if (Info.CFIType) { MIB->setCFIType(MF, Info.CFIType->getZExtValue()); } + MIB->setDeactivationSymbol(MF, Info.DeactivationSymbol); MIB.add(Info.Callee); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 8e35ba77d69aa..71ea9ef6fc050 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -143,14 +143,6 @@ class AMDGPUCodeGenPrepareImpl bool canBreakPHINode(const PHINode &I); - /// \returns True if binary operation \p I is a signed binary operation, false - /// otherwise. - bool isSigned(const BinaryOperator &I) const; - - /// \returns True if the condition of 'select' operation \p I comes from a - /// signed 'icmp' operation, false otherwise. - bool isSigned(const SelectInst &I) const; - /// Return true if \p T is a legal scalar floating point type. bool isLegalFloatingTy(const Type *T) const; @@ -304,16 +296,6 @@ bool AMDGPUCodeGenPrepareImpl::run() { return MadeChange; } -bool AMDGPUCodeGenPrepareImpl::isSigned(const BinaryOperator &I) const { - return I.getOpcode() == Instruction::AShr || - I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::SRem; -} - -bool AMDGPUCodeGenPrepareImpl::isSigned(const SelectInst &I) const { - return isa(I.getOperand(0)) && - cast(I.getOperand(0))->isSigned(); -} - bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const { return Ty->isFloatTy() || Ty->isDoubleTy() || (Ty->isHalfTy() && ST.has16BitInsts()); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 0b89e5f4cf316..8b1a09caf907d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -242,7 +242,7 @@ void SPIRVGlobalRegistry::invalidateMachineInstr(MachineInstr *MI) { // remove. const SPIRVSubtarget &ST = MI->getMF()->getSubtarget(); - const SPIRVInstrInfo *TII = ST.getInstrInfo(); + [[maybe_unused]] const SPIRVInstrInfo *TII = ST.getInstrInfo(); assert(!TII->isAliasingInstr(*MI) && "Cannot invalidate aliasing instructions."); assert(MI->getOpcode() != SPIRV::OpFunction && diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp index e31d7c6a86476..f061272d3fad4 100644 --- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp +++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp @@ -1270,7 +1270,7 @@ void SystemZAsmPrinter::emitFunctionBodyEnd() { static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, bool StackProtector, bool FPRMask, bool VRMask, - bool EHBlock, bool HasName) { + bool EHBlock, bool HasArgAreaLength, bool HasName) { enum class PPA1Flag1 : uint8_t { DSA64Bit = (0x80 >> 0), VarArg = (0x80 >> 7), @@ -1282,8 +1282,9 @@ static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, LLVM_MARK_AS_BITMASK_ENUM(ExternalProcedure) }; enum class PPA1Flag3 : uint8_t { + HasArgAreaLength = (0x80 >> 1), FPRMask = (0x80 >> 2), - LLVM_MARK_AS_BITMASK_ENUM(FPRMask) + LLVM_MARK_AS_BITMASK_ENUM(HasArgAreaLength) }; enum class PPA1Flag4 : uint8_t { EPMOffsetPresent = (0x80 >> 0), @@ -1307,6 +1308,9 @@ static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, if (StackProtector) Flags2 |= PPA1Flag2::STACKPROTECTOR; + if (HasArgAreaLength) + Flags3 |= PPA1Flag3::HasArgAreaLength; // Add emit ArgAreaLength flag. + // SavedGPRMask, SavedFPRMask, and SavedVRMask are precomputed in. if (FPRMask) Flags3 |= PPA1Flag3::FPRMask; // Add emit FPR mask flag. @@ -1339,6 +1343,9 @@ static void emitPPA1Flags(std::unique_ptr &OutStreamer, bool VarArg, OutStreamer->emitInt8(static_cast(Flags2)); // Flags 2. OutStreamer->AddComment("PPA1 Flags 3"); + if ((Flags3 & PPA1Flag3::HasArgAreaLength) == PPA1Flag3::HasArgAreaLength) + OutStreamer->AddComment( + " Bit 1: 1 = Argument Area Length is in optional area"); if ((Flags3 & PPA1Flag3::FPRMask) == PPA1Flag3::FPRMask) OutStreamer->AddComment(" Bit 2: 1 = FP Reg Mask is in optional area"); OutStreamer->emitInt8( @@ -1477,12 +1484,26 @@ void SystemZAsmPrinter::emitPPA1(MCSymbol *FnEndSym) { bool NeedEmitEHBlock = !MF->getLandingPads().empty(); + // Optional Argument Area Length. + // Note: This represents the length of the argument area that we reserve + // in our stack for setting up arguments for calls to other + // routines. If this optional field is not set, LE will reserve + // 128 bytes for the argument area. This optional field is + // created if greater than 128 bytes is required - to guarantee + // the required space is reserved on stack extension in the new + // extension. This optional field is also created if the + // routine has alloca(). This may reduce stack space + // if alloca() call causes a stack extension. + bool HasArgAreaLength = + (AllocaReg != 0) || (MFFrame.getMaxCallFrameSize() > 128); + bool HasName = MF->getFunction().hasName() && MF->getFunction().getName().size() > 0; emitPPA1Flags(OutStreamer, MF->getFunction().isVarArg(), MFFrame.hasStackProtectorIndex(), SavedFPRMask != 0, - TargetHasVector && SavedVRMask != 0, NeedEmitEHBlock, HasName); + TargetHasVector && SavedVRMask != 0, NeedEmitEHBlock, + HasArgAreaLength, HasName); OutStreamer->AddComment("Length/4 of Parms"); OutStreamer->emitInt16( @@ -1490,6 +1511,11 @@ void SystemZAsmPrinter::emitPPA1(MCSymbol *FnEndSym) { OutStreamer->AddComment("Length of Code"); OutStreamer->emitAbsoluteSymbolDiff(FnEndSym, CurrentFnEPMarkerSym, 4); + if (HasArgAreaLength) { + OutStreamer->AddComment("Argument Area Length"); + OutStreamer->emitInt32(MFFrame.getMaxCallFrameSize()); + } + // Emit saved FPR mask and offset to FPR save area (0x20 of flags 3). if (SavedFPRMask) { OutStreamer->AddComment("FPR mask"); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 8e4edefec42fd..9543d97616ae3 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3077,6 +3077,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { } case Intrinsic::ptrauth_auth: case Intrinsic::ptrauth_resign: { + // We don't support this optimization on intrinsic calls with deactivation + // symbols, which are represented using operand bundles. + if (II->hasOperandBundles()) + break; + // (sign|resign) + (auth|resign) can be folded by omitting the middle // sign+auth component if the key and discriminator match. bool NeedSign = II->getIntrinsicID() == Intrinsic::ptrauth_resign; @@ -3088,6 +3093,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // whatever we replace this sequence with. Value *AuthKey = nullptr, *AuthDisc = nullptr, *BasePtr; if (const auto *CI = dyn_cast(Ptr)) { + // We don't support this optimization on intrinsic calls with deactivation + // symbols, which are represented using operand bundles. + if (CI->hasOperandBundles()) + break; + BasePtr = CI->getArgOperand(0); if (CI->getIntrinsicID() == Intrinsic::ptrauth_sign) { if (CI->getArgOperand(1) != Key || CI->getArgOperand(2) != Disc) @@ -3110,9 +3120,10 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (NeedSign && isa(II->getArgOperand(4))) { auto *SignKey = cast(II->getArgOperand(3)); auto *SignDisc = cast(II->getArgOperand(4)); - auto *SignAddrDisc = ConstantPointerNull::get(Builder.getPtrTy()); + auto *Null = ConstantPointerNull::get(Builder.getPtrTy()); auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey, - SignDisc, SignAddrDisc); + SignDisc, /*AddrDisc=*/Null, + /*DeactivationSymbol=*/Null); replaceInstUsesWith( *II, ConstantExpr::getPointerCast(NewCPA, II->getType())); return eraseInstFromFunction(*II); diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp index 9239ae8741afb..b5a8f79e26436 100644 --- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp +++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp @@ -178,6 +178,8 @@ getRuntimeCallName(const BoundsCheckingPass::Options::Runtime &Opts) { Name += "_minimal"; if (!Opts.MayReturn) Name += "_abort"; + else if (Opts.HandlerPreserveAllRegs) + Name += "_preserve"; return Name; } @@ -267,7 +269,10 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI, TrapCall->setDoesNotReturn(); IRB.CreateUnreachable(); } - + // The preserve-all logic is somewhat duplicated in CGExpr.cpp for + // local-bounds. Make sure to change that too. + if (Opts.Rt && Opts.Rt->HandlerPreserveAllRegs && MayReturn) + TrapCall->setCallingConv(CallingConv::PreserveAll); if (!MayReturn && SingleTrapBB && !DebugTrapBB) ReuseTrapBB = TrapBB; diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 9021d8b289baf..6e36006890df4 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -526,8 +526,9 @@ Value *Mapper::mapValue(const Value *V) { if (isa(C)) return getVM()[V] = ConstantVector::get(Ops); if (isa(C)) - return getVM()[V] = ConstantPtrAuth::get(Ops[0], cast(Ops[1]), - cast(Ops[2]), Ops[3]); + return getVM()[V] = + ConstantPtrAuth::get(Ops[0], cast(Ops[1]), + cast(Ops[2]), Ops[3], Ops[4]); // If this is a no-operand constant, it must be because the type was remapped. if (isa(C)) return getVM()[V] = PoisonValue::get(NewTy); diff --git a/llvm/test/Assembler/invalid-ptrauth-const6.ll b/llvm/test/Assembler/invalid-ptrauth-const6.ll new file mode 100644 index 0000000000000..6e8e1d386acc8 --- /dev/null +++ b/llvm/test/Assembler/invalid-ptrauth-const6.ll @@ -0,0 +1,6 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +@var = global i32 0 + +; CHECK: error: constant ptrauth deactivation symbol must be a pointer +@ptr = global ptr ptrauth (ptr @var, i32 0, i64 65535, ptr null, i64 0) diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll index e21786e5ee330..53cbe2d6ffd37 100644 --- a/llvm/test/Bitcode/compatibility.ll +++ b/llvm/test/Bitcode/compatibility.ll @@ -217,9 +217,13 @@ declare void @g.f1() ; CHECK: @g.sanitize_address_dyninit = global i32 0, sanitize_address_dyninit ; CHECK: @g.sanitize_multiple = global i32 0, sanitize_memtag, sanitize_address_dyninit +@ds = external global i32 + ; ptrauth constant @auth_var = global ptr ptrauth (ptr @g1, i32 0, i64 65535, ptr null) ; CHECK: @auth_var = global ptr ptrauth (ptr @g1, i32 0, i64 65535) +@auth_var.ds = global ptr ptrauth (ptr @g1, i32 0, i64 65535, ptr null, ptr @ds) +; CHECK: @auth_var.ds = global ptr ptrauth (ptr @g1, i32 0, i64 65535, ptr null, ptr @ds) ;; Aliases ; Format: @ = [Linkage] [Visibility] [DLLStorageClass] [ThreadLocal] diff --git a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll index 5628e17b4936e..01e5b3f6673ae 100644 --- a/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ b/llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -14,6 +14,7 @@ ; CHECK-NEXT: