21 changes: 4 additions & 17 deletions clang/lib/Sema/SemaTemplate.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2418,23 +2418,16 @@ struct ConvertConstructorToDeductionGuideTransform {
QualType Result = SemaRef.BuildFunctionType(DeducedType, ParamTypes, Loc,
DeductionGuideName, EPI);
TypeSourceInfo *TSI = SemaRef.Context.getTrivialTypeSourceInfo(Result, Loc);
if (NestedPattern)
TSI = SemaRef.SubstType(TSI, OuterInstantiationArgs, Loc,
DeductionGuideName);

FunctionProtoTypeLoc FPTL =
TSI->getTypeLoc().castAs<FunctionProtoTypeLoc>();

// Build the parameters, needed during deduction / substitution.
SmallVector<ParmVarDecl*, 4> Params;
for (auto T : ParamTypes) {
auto *TSI = SemaRef.Context.getTrivialTypeSourceInfo(T, Loc);
if (NestedPattern)
TSI = SemaRef.SubstType(TSI, OuterInstantiationArgs, Loc,
DeclarationName());
ParmVarDecl *NewParam =
ParmVarDecl::Create(SemaRef.Context, DC, Loc, Loc, nullptr,
TSI->getType(), TSI, SC_None, nullptr);
ParmVarDecl *NewParam = ParmVarDecl::Create(
SemaRef.Context, DC, Loc, Loc, nullptr, T,
SemaRef.Context.getTrivialTypeSourceInfo(T, Loc), SC_None, nullptr);
NewParam->setScopeInfo(0, Params.size());
FPTL.setParam(Params.size(), NewParam);
Params.push_back(NewParam);
Expand Down Expand Up @@ -2677,14 +2670,8 @@ FunctionTemplateDecl *Sema::DeclareImplicitDeductionGuideFromInitList(
if (BuildingDeductionGuides.isInvalid())
return nullptr;

ClassTemplateDecl *Pattern =
Transform.NestedPattern ? Transform.NestedPattern : Transform.Template;
ContextRAII SavedContext(*this, Pattern->getTemplatedDecl());

auto *DG = cast<FunctionTemplateDecl>(
return cast<FunctionTemplateDecl>(
Transform.buildSimpleDeductionGuide(ParamTypes));
SavedContext.pop();
return DG;
}

void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
Expand Down
2 changes: 2 additions & 0 deletions clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -973,6 +973,8 @@ void WalkAST::checkMsg_decodeValueOfObjCType(const ObjCMessageExpr *ME) {
if (VT < VersionTuple(11, 0))
return;
break;
case llvm::Triple::XROS:
break;
default:
return;
}
Expand Down
41 changes: 41 additions & 0 deletions clang/test/Driver/xros-driver.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// RUN: %clang -target arm64-apple-xros1 -c -### %s 2>&1 | FileCheck --check-prefix=VERSION1 %s
// RUN: %clang -target arm64-apple-xros -c -### %s 2>&1 | FileCheck --check-prefix=VERSION1 %s

// RUN: %clang -target arm64-apple-xros1-simulator -c -### %s 2>&1 | FileCheck --check-prefix=VERSION1_ASi %s

// RUN: not %clang -target arm64-apple-xros1000 -c -### %s 2>&1 | FileCheck --check-prefix=INVALID-VERSION %s

// RUN: %clang -target arm64-apple-xros1 -### %s 2>&1 | FileCheck --check-prefix=LINK %s
// RUN: %clang -target arm64-apple-xros1-simulator -### %s 2>&1 | FileCheck --check-prefix=LINK-SIM %s

// RUN: %clang -target arm64-apple-xros1 -c -x objective-c -fobjc-arc -### %s 2>&1 | FileCheck --check-prefix=ARC %s
// RUN: %clang -target arm64-apple-xros -c -x objective-c -fobjc-arc -### %s 2>&1 | FileCheck --check-prefix=OBJC-RUNTIME %s
// RUN: %clang -target arm64-apple-xros2 -c -x objective-c -fobjc-arc -### %s 2>&1 | FileCheck --check-prefix=OBJC-RUNTIME2 %s

// RUN: %clang -target arm64-apple-xros1 -c -x objective-c -fobjc-arc -### %s 2>&1 | FileCheck --check-prefix=ARC %s
// RUN: %clang -target arm64-apple-xros1-simulator -c -x objective-c -fobjc-arc -### %s 2>&1 | FileCheck --check-prefix=ARC %s

// RUN: %clang -target arm64-apple-xros -c -### %s 2>&1 | FileCheck --check-prefix=SSP_ON %s

// RUN: %clang -target arm64e-apple-xros -c -### %s 2>&1 | FileCheck --check-prefix=CPU-ARM64E %s
// RUN: %clang -target arm64-apple-xros -c -### %s 2>&1 | FileCheck --check-prefix=CPU-ARM64 %s
// RUN: %clang -target arm64-apple-xros-simulator -c -### %s 2>&1 | FileCheck --check-prefix=CPU-ARM64-SIM %s

// VERSION1: "-cc1"{{.*}} "-triple" "arm64-apple-xros1.0.0"
// VERSION1_ASi: "-cc1"{{.*}} "-triple" "arm64-apple-xros1.0.0-simulator"
// INVALID-VERSION: error: invalid version number in

// VERSION1-NOT: -faligned-alloc-unavailable

// LINK: "-platform_version" "xros" "1.0.0" "1.0.0"
// LINK-SIM: "-platform_version" "xros-simulator" "1.0.0" "1.0.0"

// OBJC-RUNTIME: "-fobjc-runtime=ios-17.0.0.0"
// OBJC-RUNTIME2: "-fobjc-runtime=ios-18.0.0.0"
// ARC-NOT: error:

// SSP_ON: "-stack-protector" "1"

// CPU-ARM64E: "-cc1"{{.*}} "-triple" "arm64e-apple-xros1.0.0"{{.*}} "-target-cpu" "apple-a12"{{.*}}
// CPU-ARM64: "-cc1"{{.*}} "-triple" "arm64-apple-xros1.0.0"{{.*}} "-target-cpu" "apple-a12"
// CPU-ARM64-SIM: "-cc1"{{.*}} "-triple" "arm64-apple-xros1.0.0-simulator"{{.*}} "-target-cpu" "apple-m1"
3 changes: 3 additions & 0 deletions clang/test/Frontend/xros-version.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
// RUN: %clang_cc1 -triple arm64-apple-xros1 -dM -E -o - %s | FileCheck %s

// CHECK: __ENVIRONMENT_OS_VERSION_MIN_REQUIRED__ 10000
19 changes: 1 addition & 18 deletions clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// RUN: %clang_cc1 -std=c++20 -verify %s
// expected-no-diagnostics

template<class T> struct S {
template<class U> struct N {
Expand Down Expand Up @@ -57,21 +58,3 @@ template<class X> struct requires_clause {
requires_clause<int>::B req(1, 2);
using RC = decltype(req);
using RC = requires_clause<int>::B<int>;

template<typename X> struct nested_init_list {
template<C<X> Y>
struct B { // #INIT_LIST_INNER
X x;
Y y;
};
};

nested_init_list<int>::B nil {1, 2};
using NIL = decltype(nil);
using NIL = nested_init_list<int>::B<int>;

// expected-error@+1 {{no viable constructor or deduction guide for deduction of template arguments of 'B'}}
nested_init_list<int>::B nil_invalid {1, ""};
// expected-note@#INIT_LIST_INNER {{candidate template ignored: substitution failure [with Y = const char *]: constraints not satisfied for class template 'B' [with Y = const char *]}}
// expected-note@#INIT_LIST_INNER {{candidate function template not viable: requires 1 argument, but 2 were provided}}
// expected-note@#INIT_LIST_INNER {{candidate function template not viable: requires 0 arguments, but 2 were provided}}
7 changes: 7 additions & 0 deletions libcxx/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,13 @@ if (NOT "${LIBCXX_HARDENING_MODE}" IN_LIST LIBCXX_SUPPORTED_HARDENING_MODES)
message(FATAL_ERROR
"Unsupported hardening mode: '${LIBCXX_HARDENING_MODE}'. Supported values are ${LIBCXX_SUPPORTED_HARDENING_MODES}.")
endif()
set(LIBCXX_ASSERTION_HANDLER_FILE
"${CMAKE_CURRENT_SOURCE_DIR}/vendor/llvm/default_assertion_handler.in"
CACHE STRING
"Specify the path to a header that contains a custom implementation of the
assertion handler that gets invoked when a hardening assertion fails. If
provided, this header will be included by the library, replacing the
default assertion handler.")
option(LIBCXX_ENABLE_RANDOM_DEVICE
"Whether to include support for std::random_device in the library. Disabling
this can be useful when building the library for platforms that don't have
Expand Down
46 changes: 46 additions & 0 deletions libcxx/docs/BuildingLibcxx.rst
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,15 @@ libc++ Feature Options
Use the specified GCC toolchain and standard library when building the native
stdlib benchmark tests.

.. option:: LIBCXX_ASSERTION_HANDLER_FILE:PATH

**Default**:: ``"${CMAKE_CURRENT_SOURCE_DIR}/vendor/llvm/default_assertion_handler.in"``

Specify the path to a header that contains a custom implementation of the
assertion handler that gets invoked when a hardening assertion fails. If
provided, this header will be included by the library, replacing the
default assertion handler.


libc++ ABI Feature Options
--------------------------
Expand Down Expand Up @@ -473,6 +482,43 @@ LLVM-specific options
others.


.. _assertion-handler:

Overriding the default assertion handler
==========================================

When the library wants to terminate due to an unforeseen condition (such as
a hardening assertion failure), the program is aborted through a special verbose
termination function. The library provides a default function that prints an
error message and calls ``std::abort()``. Note that this function is provided by
the static or shared library, so it is only available when deploying to
a platform where the compiled library is sufficiently recent. On older
platforms, the program will terminate in an unspecified unsuccessful manner, but
the quality of diagnostics won't be great.

However, vendors can also override that mechanism at CMake configuration time.
When a hardening assertion fails, the library invokes the
``_LIBCPP_ASSERTION_HANDLER`` macro. A vendor may provide a header that contains
a custom definition of this macro and specify the path to the header via the
``LIBCXX_ASSERTION_HANDLER_FILE`` CMake variable. If provided, this header will
be included by the library and replace the default implementation. The header
must not include any standard library headers (directly or transitively) because
doing so will almost always create a circular dependency. The
``_LIBCPP_ASSERTION_HANDLER(message)`` macro takes a single parameter that
contains an error message explaining the hardening failure and some details
about the source location that triggered it.

When a hardening assertion fails, it means that the program is about to invoke
library undefined behavior. For this reason, the custom assertion handler is
generally expected to terminate the program. If a custom assertion handler
decides to avoid doing so (e.g. it chooses to log and continue instead), it does
so at its own risk -- this approach should only be used in non-production builds
and with an understanding of potential consequences. Furthermore, the custom
assertion handler should not throw any exceptions as it may be invoked from
standard library functions that are marked ``noexcept`` (so throwing will result
in ``std::terminate`` being called).


Using Alternate ABI libraries
=============================

Expand Down
4 changes: 4 additions & 0 deletions libcxx/docs/ReleaseNotes/18.rst
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@ Deprecations and Removals
macro is provided to restore the previous behavior, and it will be supported in the LLVM 18 release only.
In LLVM 19 and beyond, ``_LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT`` will not be honored anymore.

- The only supported way to customize the assertion handler that gets invoked when a hardening assertion fails
is now by setting the ``LIBCXX_ASSERTION_HANDLER_FILE`` CMake variable and providing a custom header. See
the documentation on overriding the default assertion handler for details.

- The ``_LIBCPP_AVAILABILITY_CUSTOM_VERBOSE_ABORT_PROVIDED`` macro is not honored anymore in LLVM 18.
Please see the updated documentation about the hardening modes in libc++ and in particular the
``_LIBCPP_VERBOSE_ABORT`` macro for details.
Expand Down
64 changes: 0 additions & 64 deletions libcxx/docs/UsingLibcxx.rst
Original file line number Diff line number Diff line change
Expand Up @@ -146,70 +146,6 @@ IWYU, you should run the tool like so:
If you would prefer to not use that flag, then you can replace ``/path/to/include-what-you-use/share/libcxx.imp``
file with the libc++-provided ``libcxx.imp`` file.

.. _termination-handler:

Overriding the default termination handler
==========================================

When the library wants to terminate due to an unforeseen condition (such as a hardening assertion
failure), the program is aborted through a special verbose termination function. The library provides
a default function that prints an error message and calls ``std::abort()``. Note that this function is
provided by the static or shared library, so it is only available when deploying to a platform where
the compiled library is sufficiently recent. On older platforms, the program will terminate in an
unspecified unsuccessful manner, but the quality of diagnostics won't be great.

However, users can also override that mechanism at two different levels. First, the mechanism can be
overridden at compile time by defining the ``_LIBCPP_VERBOSE_ABORT(format, args...)`` variadic macro.
When that macro is defined, it will be called with a format string as the first argument, followed by
a series of arguments to format using printf-style formatting. Compile-time customization may be
useful to get precise control over code generation, however it is also inconvenient to use in
some cases. Indeed, compile-time customization of the verbose termination function requires that all
translation units be compiled with a consistent definition for ``_LIBCPP_VERBOSE_ABORT`` to avoid ODR
violations, which can add complexity in the build system of users.

Otherwise, if compile-time customization is not necessary, link-time customization of the handler is also
possible, similarly to how replacing ``operator new`` works. This mechanism trades off fine-grained control
over the call site where the termination is initiated in exchange for better ergonomics. Link-time
customization is done by simply defining the following function in exactly one translation unit of your
program:

.. code-block:: cpp
void __libcpp_verbose_abort(char const* format, ...)
This mechanism is similar to how one can replace the default definition of ``operator new``
and ``operator delete``. For example:

.. code-block:: cpp
// In HelloWorldHandler.cpp
#include <version> // must include any libc++ header before defining the function (C compatibility headers excluded)
void std::__libcpp_verbose_abort(char const* format, ...) {
std::va_list list;
va_start(list, format);
std::vfprintf(stderr, format, list);
va_end(list);
std::abort();
}
// In HelloWorld.cpp
#include <vector>
int main() {
std::vector<int> v;
int& x = v[0]; // Your termination function will be called here if hardening is enabled.
}
Also note that the verbose termination function should never return. Since assertions in libc++
catch undefined behavior, your code will proceed with undefined behavior if your function is called
and does return.

Furthermore, exceptions should not be thrown from the function. Indeed, many functions in the
library are ``noexcept``, and any exception thrown from the termination function will result
in ``std::terminate`` being called.

Libc++ Configuration Macros
===========================

Expand Down
10 changes: 9 additions & 1 deletion libcxx/include/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1020,9 +1020,11 @@ endforeach()

configure_file("__config_site.in" "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site" @ONLY)
configure_file("module.modulemap.in" "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap" @ONLY)
configure_file("${LIBCXX_ASSERTION_HANDLER_FILE}" "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler" COPYONLY)

set(_all_includes "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site"
"${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap")
"${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap"
"${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler")
foreach(f ${files})
set(src "${CMAKE_CURRENT_SOURCE_DIR}/${f}")
set(dst "${LIBCXX_GENERATED_INCLUDE_DIR}/${f}")
Expand Down Expand Up @@ -1059,6 +1061,12 @@ if (LIBCXX_INSTALL_HEADERS)
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
COMPONENT cxx-headers)

# Install the generated __assertion_handler file to the generic include dir.
install(FILES "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler"
DESTINATION "${LIBCXX_INSTALL_INCLUDE_DIR}"
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
COMPONENT cxx-headers)

# Install the generated modulemap file to the generic include dir.
install(FILES "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap"
DESTINATION "${LIBCXX_INSTALL_INCLUDE_DIR}"
Expand Down
6 changes: 3 additions & 3 deletions libcxx/include/__assert
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
#ifndef _LIBCPP___ASSERT
#define _LIBCPP___ASSERT

#include <__assertion_handler> // Note: this include is generated by CMake and is potentially vendor-provided.
#include <__config>
#include <__verbose_abort>

#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
Expand All @@ -20,8 +20,8 @@
#define _LIBCPP_ASSERT(expression, message) \
(__builtin_expect(static_cast<bool>(expression), 1) \
? (void)0 \
: _LIBCPP_VERBOSE_ABORT( \
"%s:%d: assertion %s failed: %s\n", __builtin_FILE(), __builtin_LINE(), #expression, message))
: _LIBCPP_ASSERTION_HANDLER(__FILE__ ":" _LIBCPP_TOSTRING(__LINE__) ": assertion " _LIBCPP_TOSTRING( \
expression) " failed: " message "\n"))

// TODO: __builtin_assume can currently inhibit optimizations. Until this has been fixed and we can add
// assumptions without a clear optimization intent, disable that to avoid worsening the code generation.
Expand Down
14 changes: 4 additions & 10 deletions libcxx/include/string
Original file line number Diff line number Diff line change
Expand Up @@ -659,7 +659,6 @@ _LIBCPP_PUSH_MACROS
#else
# define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
#endif
#define _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED false

_LIBCPP_BEGIN_NAMESPACE_STD

Expand Down Expand Up @@ -1896,38 +1895,33 @@ private:
#endif
}

// ASan: short string is poisoned if and only if this function returns true.
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __asan_short_string_is_annotated() const _NOEXCEPT {
return _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED && !__libcpp_is_constant_evaluated();
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT {
(void) __current_size;
#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
if (!__libcpp_is_constant_evaluated())
__annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1);
#endif
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT {
#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
if (!__libcpp_is_constant_evaluated())
__annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1);
#endif
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT {
(void) __n;
#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
if (!__libcpp_is_constant_evaluated())
__annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n);
#endif
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
(void) __old_size;
#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
if (!__libcpp_is_constant_evaluated())
__annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1);
#endif
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
// This test ensures that enabling assertions with the legacy `_LIBCPP_ENABLE_ASSERTIONS` now enables the extensive
// hardening mode.

// `check_assertion.h` is only available starting from C++11 and requires Unix headers.
// UNSUPPORTED: c++03, !has-unix-headers
// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
// UNSUPPORTED: c++03, !has-unix-headers, no-localization
// The ability to set a custom abort message is required to compare the assertion message.
// XFAIL: availability-verbose_abort-missing
// Note that GCC doesn't support `-Wno-macro-redefined`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@

// This test ensures that we can override any hardening mode with the debug mode on a per-TU basis.

// `check_assertion.h` is only available starting from C++11.
// UNSUPPORTED: c++03
// `check_assertion.h` requires Unix headers.
// REQUIRES: has-unix-headers
// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
// UNSUPPORTED: c++03, !has-unix-headers, no-localization
// The ability to set a custom abort message is required to compare the assertion message.
// XFAIL: availability-verbose_abort-missing
// ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@

// This test ensures that we can override any hardening mode with the extensive hardening mode on a per-TU basis.

// `check_assertion.h` is only available starting from C++11.
// UNSUPPORTED: c++03
// `check_assertion.h` requires Unix headers.
// REQUIRES: has-unix-headers
// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
// UNSUPPORTED: c++03, !has-unix-headers, no-localization
// The ability to set a custom abort message is required to compare the assertion message.
// XFAIL: availability-verbose_abort-missing
// ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@

// This test ensures that we can override any hardening mode with the fast mode on a per-TU basis.

// `check_assertion.h` is only available starting from C++11.
// UNSUPPORTED: c++03
// `check_assertion.h` requires Unix headers.
// REQUIRES: has-unix-headers
// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
// UNSUPPORTED: c++03, !has-unix-headers, no-localization
// The ability to set a custom abort message is required to compare the assertion message.
// XFAIL: availability-verbose_abort-missing
// ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@

// This test ensures that we can override any hardening mode with the unchecked mode on a per-TU basis.

// `check_assertion.h` is only available starting from C++11.
// UNSUPPORTED: c++03
// `check_assertion.h` requires Unix headers.
// REQUIRES: has-unix-headers
// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
// UNSUPPORTED: c++03, !has-unix-headers, no-localization
// ADDITIONAL_COMPILE_FLAGS: -U_LIBCPP_HARDENING_MODE -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE

#include <cassert>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
// Regression test to error in deque::__annotate_from_to in deque,
// with origin in deque::__add_back_capacity.

// REQUIRES: has-unix-headers
// UNSUPPORTED: c++03
// `check_assertion.h` is only available starting from C++11 and requires Unix headers and regex support.
// UNSUPPORTED: c++03, !has-unix-headers, no-localization

#include <deque>
#include <cstdio>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

// REQUIRES: asan
// UNSUPPORTED: c++03

#include <cassert>
#include <string>
#include <array>
#include <deque>
#include "test_macros.h"
#include "asan_testing.h"
#include "min_allocator.h"

// This tests exists to check if strings work well with deque, as those
// may be partialy annotated, we cannot simply call
// is_double_ended_contiguous_container_asan_correct, as it assumes that
// object memory inside is not annotated, so we check everything in a more careful way.

template <typename D>
void verify_inside(D const& d) {
for (size_t i = 0; i < d.size(); ++i) {
assert(is_string_asan_correct(d[i]));
}
}

template <typename S, size_t N>
S get_s(char c) {
S s;
for (size_t i = 0; i < N; ++i)
s.push_back(c);

return s;
}

template <class C, class S>
void test_string() {
size_t const N = sizeof(S) < 256 ? (4096 / sizeof(S)) : 16;

{
C d1a(1), d1b(N), d1c(N + 1), d1d(32 * N);
verify_inside(d1a);
verify_inside(d1b);
verify_inside(d1c);
verify_inside(d1d);
}
{
C d2;
for (size_t i = 0; i < 16 * N; ++i) {
d2.push_back(get_s<S, 1>(i % 10 + 'a'));
verify_inside(d2);
d2.push_back(get_s<S, 222>(i % 10 + 'b'));
verify_inside(d2);

d2.pop_front();
verify_inside(d2);
}
}
{
C d3;
for (size_t i = 0; i < 16 * N; ++i) {
d3.push_front(get_s<S, 1>(i % 10 + 'a'));
verify_inside(d3);
d3.push_front(get_s<S, 222>(i % 10 + 'b'));
verify_inside(d3);

d3.pop_back();
verify_inside(d3);
}
}
{
C d4;
for (size_t i = 0; i < 16 * N; ++i) {
// When there is no SSO, all elements inside should not be poisoned,
// so we can verify deque poisoning.
d4.push_front(get_s<S, 333>(i % 10 + 'a'));
verify_inside(d4);
assert(is_double_ended_contiguous_container_asan_correct(d4));
d4.push_back(get_s<S, 222>(i % 10 + 'b'));
verify_inside(d4);
assert(is_double_ended_contiguous_container_asan_correct(d4));
}
}
{
C d5;
for (size_t i = 0; i < 5 * N; ++i) {
// In d4 we never had poisoned memory inside deque.
// Here we start with SSO, so part of the inside of the container,
// will be poisoned.
d5.push_front(S());
verify_inside(d5);
}
for (size_t i = 0; i < d5.size(); ++i) {
// We change the size to have long string.
// Memory owne by deque should not be poisoned by string.
d5[i].resize(1000);
verify_inside(d5);
}

assert(is_double_ended_contiguous_container_asan_correct(d5));

d5.erase(d5.begin() + 2);
verify_inside(d5);

d5.erase(d5.end() - 2);
verify_inside(d5);

assert(is_double_ended_contiguous_container_asan_correct(d5));
}
{
C d6a;
assert(is_double_ended_contiguous_container_asan_correct(d6a));

C d6b(N + 2, get_s<S, 1000>('a'));
d6b.push_front(get_s<S, 1001>('b'));
while (!d6b.empty()) {
d6b.pop_back();
assert(is_double_ended_contiguous_container_asan_correct(d6b));
}

C d6c(N + 2, get_s<S, 1002>('c'));
while (!d6c.empty()) {
d6c.pop_back();
assert(is_double_ended_contiguous_container_asan_correct(d6c));
}
}
{
C d7(9 * N + 2);

d7.insert(d7.begin() + 1, S());
verify_inside(d7);

d7.insert(d7.end() - 3, S());
verify_inside(d7);

d7.insert(d7.begin() + 2 * N, get_s<S, 1>('a'));
verify_inside(d7);

d7.insert(d7.end() - 2 * N, get_s<S, 1>('b'));
verify_inside(d7);

d7.insert(d7.begin() + 2 * N, 3 * N, get_s<S, 1>('c'));
verify_inside(d7);

// It may not be short for big element types, but it will be checked correctly:
d7.insert(d7.end() - 2 * N, 3 * N, get_s<S, 2>('d'));
verify_inside(d7);

d7.erase(d7.begin() + 2);
verify_inside(d7);

d7.erase(d7.end() - 2);
verify_inside(d7);
}
}

template <class S>
void test_container() {
test_string<std::deque<S, std::allocator<S>>, S>();
test_string<std::deque<S, min_allocator<S>>, S>();
test_string<std::deque<S, safe_allocator<S>>, S>();
}

int main(int, char**) {
// Those tests support only types based on std::basic_string.
test_container<std::string>();
test_container<std::wstring>();
#if TEST_STD_VER >= 11
test_container<std::u16string>();
test_container<std::u32string>();
#endif
#if TEST_STD_VER >= 20
test_container<std::u8string>();
#endif

return 0;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

// REQUIRES: asan
// UNSUPPORTED: c++03

// <string>

// Basic test if ASan annotations work for short strings.

#include <string>
#include <cassert>
#include <cstdlib>

#include "asan_testing.h"
#include "min_allocator.h"
#include "test_iterators.h"
#include "test_macros.h"

extern "C" void __sanitizer_set_death_callback(void (*callback)(void));

void do_exit() { exit(0); }

int main(int, char**) {
{
typedef cpp17_input_iterator<char*> MyInputIter;
// Should not trigger ASan.
std::basic_string<char, std::char_traits<char>, safe_allocator<char>> v;
char i[] = {'a', 'b', 'c', 'd'};

v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 4));
assert(v[0] == 'a');
assert(is_string_asan_correct(v));
}

__sanitizer_set_death_callback(do_exit);
{
using T = char;
using C = std::basic_string<T, std::char_traits<T>, safe_allocator<T>>;
const T t[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g'};
C c(std::begin(t), std::end(t));
assert(is_string_asan_correct(c));
assert(__sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) !=
0);
volatile T foo = c[c.size() + 1]; // should trigger ASAN. Use volatile to prevent being optimized away.
assert(false); // if we got here, ASAN didn't trigger
((void)foo);
}

return 0;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

// REQUIRES: asan
// UNSUPPORTED: c++03

#include <cassert>
#include <string>
#include <vector>
#include <array>
#include "test_macros.h"
#include "asan_testing.h"
#include "min_allocator.h"

// This tests exists to check if strings work well with vector, as those
// may be partialy annotated, we cannot simply call
// is_contiguous_container_asan_correct, as it assumes that
// object memory inside is not annotated, so we check everything in a more careful way.

template <typename D>
void verify_inside(D const& d) {
for (size_t i = 0; i < d.size(); ++i) {
assert(is_string_asan_correct(d[i]));
}
}

template <typename S, size_t N>
S get_s(char c) {
S s;
for (size_t i = 0; i < N; ++i)
s.push_back(c);

return s;
}

template <class C, class S>
void test_string() {
size_t const N = sizeof(S) < 256 ? (4096 / sizeof(S)) : 16;

{
C d1a(1), d1b(N), d1c(N + 1), d1d(32 * N);
verify_inside(d1a);
verify_inside(d1b);
verify_inside(d1c);
verify_inside(d1d);
}
{
C d2;
for (size_t i = 0; i < 16 * N; ++i) {
d2.push_back(get_s<S, 1>(i % 10 + 'a'));
verify_inside(d2);
d2.push_back(get_s<S, 222>(i % 10 + 'b'));
verify_inside(d2);

d2.erase(d2.cbegin());
verify_inside(d2);
}
}
{
C d3;
for (size_t i = 0; i < 16 * N; ++i) {
d3.push_back(get_s<S, 1>(i % 10 + 'a'));
verify_inside(d3);
d3.push_back(get_s<S, 222>(i % 10 + 'b'));
verify_inside(d3);

d3.pop_back();
verify_inside(d3);
}
}
{
C d4;
for (size_t i = 0; i < 16 * N; ++i) {
// When there is no SSO, all elements inside should not be poisoned,
// so we can verify vector poisoning.
d4.push_back(get_s<S, 333>(i % 10 + 'a'));
verify_inside(d4);
assert(is_contiguous_container_asan_correct(d4));
d4.push_back(get_s<S, 222>(i % 10 + 'b'));
verify_inside(d4);
assert(is_contiguous_container_asan_correct(d4));
}
}
{
C d5;
for (size_t i = 0; i < 5 * N; ++i) {
// In d4 we never had poisoned memory inside vector.
// Here we start with SSO, so part of the inside of the container,
// will be poisoned.
d5.push_back(S());
verify_inside(d5);
}
for (size_t i = 0; i < d5.size(); ++i) {
// We change the size to have long string.
// Memory owne by vector should not be poisoned by string.
d5[i].resize(1000);
verify_inside(d5);
}

assert(is_contiguous_container_asan_correct(d5));

d5.erase(d5.begin() + 2);
verify_inside(d5);

d5.erase(d5.end() - 2);
verify_inside(d5);

assert(is_contiguous_container_asan_correct(d5));
}
{
C d6a;
assert(is_contiguous_container_asan_correct(d6a));

C d6b(N + 2, get_s<S, 1000>('a'));
d6b.push_back(get_s<S, 1001>('b'));
while (!d6b.empty()) {
d6b.pop_back();
assert(is_contiguous_container_asan_correct(d6b));
}

C d6c(N + 2, get_s<S, 1002>('c'));
while (!d6c.empty()) {
d6c.pop_back();
assert(is_contiguous_container_asan_correct(d6c));
}
}
{
C d7(9 * N + 2);

d7.insert(d7.begin() + 1, S());
verify_inside(d7);

d7.insert(d7.end() - 3, S());
verify_inside(d7);

d7.insert(d7.begin() + 2 * N, get_s<S, 1>('a'));
verify_inside(d7);

d7.insert(d7.end() - 2 * N, get_s<S, 1>('b'));
verify_inside(d7);

d7.insert(d7.begin() + 2 * N, 3 * N, get_s<S, 1>('c'));
verify_inside(d7);

// It may not be short for big element types, but it will be checked correctly:
d7.insert(d7.end() - 2 * N, 3 * N, get_s<S, 2>('d'));
verify_inside(d7);

d7.erase(d7.begin() + 2);
verify_inside(d7);

d7.erase(d7.end() - 2);
verify_inside(d7);
}
}

template <class S>
void test_container() {
test_string<std::vector<S, std::allocator<S>>, S>();
test_string<std::vector<S, min_allocator<S>>, S>();
test_string<std::vector<S, safe_allocator<S>>, S>();
}

int main(int, char**) {
// Those tests support only types based on std::basic_string.
test_container<std::string>();
test_container<std::wstring>();
#if TEST_STD_VER >= 11
test_container<std::u16string>();
test_container<std::u32string>();
#endif
#if TEST_STD_VER >= 20
test_container<std::u8string>();
#endif

return 0;
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

// UNSUPPORTED: c++03, c++11, c++14
// UNSUPPORTED: no-exceptions
// REQUIRES: has-unix-headers
// `check_assertion.h` requires Unix headers and regex support.
// UNSUPPORTED: !has-unix-headers, no-localization

// UNSUPPORTED: libcpp-has-no-incomplete-pstl

Expand Down
29 changes: 5 additions & 24 deletions libcxx/test/support/asan_testing.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,35 +56,16 @@ TEST_CONSTEXPR bool is_double_ended_contiguous_container_asan_correct(const std:
#endif

#if TEST_HAS_FEATURE(address_sanitizer)
template <typename S>
bool is_string_short(S const& s) {
// We do not have access to __is_long(), but we can check if strings
// buffer is inside strings memory. If strings memory contains its content,
// SSO is in use. To check it, we can just confirm that the beginning is in
// the string object memory block.
// &s - beginning of objects memory
// &s[0] - beginning of the buffer
// (&s+1) - end of objects memory
return (void*)std::addressof(s) <= (void*)std::addressof(s[0]) &&
(void*)std::addressof(s[0]) < (void*)(std::addressof(s) + 1);
}

template <typename ChrT, typename TraitsT, typename Alloc>
TEST_CONSTEXPR bool is_string_asan_correct(const std::basic_string<ChrT, TraitsT, Alloc>& c) {
if (TEST_IS_CONSTANT_EVALUATED)
return true;

if (!is_string_short(c) || _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED) {
if (std::__asan_annotate_container_with_allocator<Alloc>::value)
return __sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) !=
0;
else
return __sanitizer_verify_contiguous_container(
c.data(), c.data() + c.capacity() + 1, c.data() + c.capacity() + 1) != 0;
} else {
return __sanitizer_verify_contiguous_container(std::addressof(c), std::addressof(c) + 1, std::addressof(c) + 1) !=
0;
}
if (std::__asan_annotate_container_with_allocator<Alloc>::value)
return __sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) != 0;
else
return __sanitizer_verify_contiguous_container(
c.data(), c.data() + c.capacity() + 1, c.data() + c.capacity() + 1) != 0;
}
#else
# include <string>
Expand Down
33 changes: 21 additions & 12 deletions libcxx/test/support/check_assertion.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <cstdio>
#include <cstdlib>
#include <exception>
#include <regex>
#include <string>
#include <string_view>
#include <utility>
Expand Down Expand Up @@ -234,18 +235,26 @@ struct DeathTest {
};

#ifdef _LIBCPP_VERSION
void std::__libcpp_verbose_abort(char const* format, ...) {
// Extract information from the error message. This has to stay synchronized with
// how we format assertions in the library.
va_list list;
va_start(list, format);
char const* file = va_arg(list, char const*);
int line = va_arg(list, int);
char const* expression = va_arg(list, char const*); (void)expression;
char const* message = va_arg(list, char const*);
va_end(list);

if (GlobalMatcher().Matches(file, line, message)) {
void std::__libcpp_verbose_abort(char const* printf_format, ...) {
// Extract information from the error message. This has to stay synchronized with how we format assertions in the
// library.
va_list args;
va_start(args, printf_format);
char const* message = va_arg(args, char const*);

std::regex message_format("(.*):(\\d+): assertion (.*) failed: (.*)\\n");

std::cmatch match_result;
bool has_match = std::regex_match(message, match_result, message_format);
assert(has_match);
assert(match_result.size() == 5);

std::string file = match_result[1];
int line = std::stoi(match_result[2]);
// Omitting `expression` in `match_result[3]`
std::string failure_reason = match_result[4];

if (GlobalMatcher().Matches(file.c_str(), line, failure_reason.c_str())) {
std::exit(DeathTest::RK_MatchFound);
}
std::exit(DeathTest::RK_MatchFailure);
Expand Down
3 changes: 3 additions & 0 deletions libcxx/utils/libcxx/header_information.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ def is_modulemap_header(header):
if header == "__config_site":
return False

if header == "__assertion_handler":
return False

# exclude libc++abi files
if header in ["cxxabi.h", "__cxxabi_config.h"]:
return False
Expand Down
23 changes: 23 additions & 0 deletions libcxx/vendor/llvm/default_assertion_handler.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef _LIBCPP___ASSERTION_HANDLER
#define _LIBCPP___ASSERTION_HANDLER

#include <__config>
#include <__verbose_abort>

#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif

// TODO(hardening): in production, trap rather than abort.
#define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message)

#endif // _LIBCPP___ASSERTION_HANDLER
5 changes: 5 additions & 0 deletions llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "llvm/CodeGen/ExpandMemCmp.h"
#include "llvm/CodeGen/ExpandReductions.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/GlobalMerge.h"
#include "llvm/CodeGen/IndirectBrExpand.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/InterleavedLoadCombine.h"
Expand Down Expand Up @@ -280,6 +281,9 @@ template <typename DerivedT> class CodeGenPassBuilder {
inconvertibleErrorCode());
}

/// Target can override this to add GlobalMergePass before all IR passes.
void addGlobalMergePass(AddIRPass &) const {}

/// Add passes that optimize instruction level parallelism for out-of-order
/// targets. These passes are run while the machine code is still in SSA
/// form, so they can use MachineTraceMetrics to control their heuristics.
Expand Down Expand Up @@ -593,6 +597,7 @@ CodeGenPassBuilder<Derived>::getPassNameFromLegacyName(StringRef Name) const {

template <typename Derived>
void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
derived().addGlobalMergePass(addPass);
if (TM.useEmulatedTLS())
addPass(LowerEmuTLSPass());

Expand Down
50 changes: 50 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalMerge.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
//===- llvm/CodeGen/GlobalMerge.h -------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_CODEGEN_GLOBALMERGE_H
#define LLVM_CODEGEN_GLOBALMERGE_H

#include "llvm/IR/PassManager.h"

namespace llvm {

class TargetMachine;

struct GlobalMergeOptions {
// FIXME: Infer the maximum possible offset depending on the actual users
// (these max offsets are different for the users inside Thumb or ARM
// functions), see the code that passes in the offset in the ARM backend
// for more information.
unsigned MaxOffset = 0;
bool GroupByUse = true;
bool IgnoreSingleUse = true;
bool MergeConst = false;
/// Whether we should merge global variables that have external linkage.
bool MergeExternal = true;
/// Whether we should try to optimize for size only.
/// Currently, this applies a dead simple heuristic: only consider globals
/// used in minsize functions for merging.
/// FIXME: This could learn about optsize, and be used in the cost model.
bool SizeOnly = false;
};

// FIXME: This pass must run before AsmPrinterPass::doInitialization!
class GlobalMergePass : public PassInfoMixin<GlobalMergePass> {
const TargetMachine *TM;
GlobalMergeOptions Options;

public:
GlobalMergePass(const TargetMachine *TM, GlobalMergeOptions Options)
: TM(TM), Options(Options) {}

PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
};

} // namespace llvm

#endif // LLVM_CODEGEN_GLOBALMERGE_H
3 changes: 2 additions & 1 deletion llvm/include/llvm/CodeGen/MachinePassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
#ifndef MODULE_PASS
#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR)
#endif
MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ())
MODULE_PASS("global-merge", GlobalMergePass, (TM, GlobalMergeOptions()))
MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass, ())
MODULE_PASS("lower-emutls", LowerEmuTLSPass, ())
MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ())
MODULE_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
#undef MODULE_PASS

Expand Down
18 changes: 4 additions & 14 deletions llvm/include/llvm/Support/GenericLoopInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -241,24 +241,14 @@ template <class BlockT, class LoopT> class LoopBase {
bool isLoopLatch(const BlockT *BB) const {
assert(!isInvalid() && "Loop not in a valid state!");
assert(contains(BB) && "block does not belong to the loop");

BlockT *Header = getHeader();
auto PredBegin = GraphTraits<Inverse<BlockT *>>::child_begin(Header);
auto PredEnd = GraphTraits<Inverse<BlockT *>>::child_end(Header);
return std::find(PredBegin, PredEnd, BB) != PredEnd;
return llvm::is_contained(inverse_children<BlockT *>(getHeader()), BB);
}

/// Calculate the number of back edges to the loop header.
unsigned getNumBackEdges() const {
assert(!isInvalid() && "Loop not in a valid state!");
unsigned NumBackEdges = 0;
BlockT *H = getHeader();

for (const auto Pred : children<Inverse<BlockT *>>(H))
if (contains(Pred))
++NumBackEdges;

return NumBackEdges;
return llvm::count_if(inverse_children<BlockT *>(getHeader()),
[&](BlockT *Pred) { return contains(Pred); });
}

//===--------------------------------------------------------------------===//
Expand Down Expand Up @@ -336,7 +326,7 @@ template <class BlockT, class LoopT> class LoopBase {
void getLoopLatches(SmallVectorImpl<BlockT *> &LoopLatches) const {
assert(!isInvalid() && "Loop not in a valid state!");
BlockT *H = getHeader();
for (const auto Pred : children<Inverse<BlockT *>>(H))
for (const auto Pred : inverse_children<BlockT *>(H))
if (contains(Pred))
LoopLatches.push_back(Pred);
}
Expand Down
167 changes: 85 additions & 82 deletions llvm/lib/CodeGen/GlobalMerge.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
//
// ===---------------------------------------------------------------------===//

#include "llvm/CodeGen/GlobalMerge.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SetVector.h"
Expand Down Expand Up @@ -137,88 +138,99 @@ STATISTIC(NumMerged, "Number of globals merged");

namespace {

class GlobalMerge : public FunctionPass {
const TargetMachine *TM = nullptr;

// FIXME: Infer the maximum possible offset depending on the actual users
// (these max offsets are different for the users inside Thumb or ARM
// functions), see the code that passes in the offset in the ARM backend
// for more information.
unsigned MaxOffset;

/// Whether we should try to optimize for size only.
/// Currently, this applies a dead simple heuristic: only consider globals
/// used in minsize functions for merging.
/// FIXME: This could learn about optsize, and be used in the cost model.
bool OnlyOptimizeForSize = false;

/// Whether we should merge global variables that have external linkage.
bool MergeExternalGlobals = false;
class GlobalMergeImpl {
const TargetMachine *TM = nullptr;
GlobalMergeOptions Opt;
bool IsMachO = false;

private:
bool doMerge(SmallVectorImpl<GlobalVariable *> &Globals, Module &M,
bool isConst, unsigned AddrSpace) const;

/// Merge everything in \p Globals for which the corresponding bit
/// in \p GlobalSet is set.
bool doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
const BitVector &GlobalSet, Module &M, bool isConst,
unsigned AddrSpace) const;

/// Check if the given variable has been identified as must keep
/// \pre setMustKeepGlobalVariables must have been called on the Module that
/// contains GV
bool isMustKeepGlobalVariable(const GlobalVariable *GV) const {
return MustKeepGlobalVariables.count(GV);
}

bool IsMachO = false;
/// Collect every variables marked as "used" or used in a landing pad
/// instruction for this Module.
void setMustKeepGlobalVariables(Module &M);

bool doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
Module &M, bool isConst, unsigned AddrSpace) const;
/// Collect every variables marked as "used"
void collectUsedGlobalVariables(Module &M, StringRef Name);

/// Merge everything in \p Globals for which the corresponding bit
/// in \p GlobalSet is set.
bool doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
const BitVector &GlobalSet, Module &M, bool isConst,
unsigned AddrSpace) const;
/// Keep track of the GlobalVariable that must not be merged away
SmallSetVector<const GlobalVariable *, 16> MustKeepGlobalVariables;

/// Check if the given variable has been identified as must keep
/// \pre setMustKeepGlobalVariables must have been called on the Module that
/// contains GV
bool isMustKeepGlobalVariable(const GlobalVariable *GV) const {
return MustKeepGlobalVariables.count(GV);
}
public:
GlobalMergeImpl(const TargetMachine *TM, GlobalMergeOptions Opt)
: TM(TM), Opt(Opt) {}
bool run(Module &M);
};

/// Collect every variables marked as "used" or used in a landing pad
/// instruction for this Module.
void setMustKeepGlobalVariables(Module &M);
class GlobalMerge : public FunctionPass {
const TargetMachine *TM = nullptr;
GlobalMergeOptions Opt;

/// Collect every variables marked as "used"
void collectUsedGlobalVariables(Module &M, StringRef Name);
public:
static char ID; // Pass identification, replacement for typeid.

/// Keep track of the GlobalVariable that must not be merged away
SmallSetVector<const GlobalVariable *, 16> MustKeepGlobalVariables;
explicit GlobalMerge() : FunctionPass(ID) {
Opt.MaxOffset = GlobalMergeMaxOffset;
initializeGlobalMergePass(*PassRegistry::getPassRegistry());
}

public:
static char ID; // Pass identification, replacement for typeid.
explicit GlobalMerge(const TargetMachine *TM, unsigned MaximalOffset,
bool OnlyOptimizeForSize, bool MergeExternalGlobals)
: FunctionPass(ID), TM(TM) {
Opt.MaxOffset = MaximalOffset;
Opt.SizeOnly = OnlyOptimizeForSize;
Opt.MergeExternal = MergeExternalGlobals;
initializeGlobalMergePass(*PassRegistry::getPassRegistry());
}

explicit GlobalMerge()
: FunctionPass(ID), MaxOffset(GlobalMergeMaxOffset) {
initializeGlobalMergePass(*PassRegistry::getPassRegistry());
}
bool doInitialization(Module &M) override {
GlobalMergeImpl P(TM, Opt);
return P.run(M);
}
bool runOnFunction(Function &F) override { return false; }

explicit GlobalMerge(const TargetMachine *TM, unsigned MaximalOffset,
bool OnlyOptimizeForSize, bool MergeExternalGlobals)
: FunctionPass(ID), TM(TM), MaxOffset(MaximalOffset),
OnlyOptimizeForSize(OnlyOptimizeForSize),
MergeExternalGlobals(MergeExternalGlobals) {
initializeGlobalMergePass(*PassRegistry::getPassRegistry());
}
StringRef getPassName() const override { return "Merge internal globals"; }

bool doInitialization(Module &M) override;
bool runOnFunction(Function &F) override;
bool doFinalization(Module &M) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
FunctionPass::getAnalysisUsage(AU);
}
};

StringRef getPassName() const override { return "Merge internal globals"; }
} // end anonymous namespace

void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
FunctionPass::getAnalysisUsage(AU);
}
};
PreservedAnalyses GlobalMergePass::run(Module &M, ModuleAnalysisManager &) {
GlobalMergeImpl P(TM, Options);
bool Changed = P.run(M);
if (!Changed)
return PreservedAnalyses::all();

} // end anonymous namespace
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
return PA;
}

char GlobalMerge::ID = 0;

INITIALIZE_PASS(GlobalMerge, DEBUG_TYPE, "Merge global variables", false, false)

bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
Module &M, bool isConst, unsigned AddrSpace) const {
bool GlobalMergeImpl::doMerge(SmallVectorImpl<GlobalVariable *> &Globals,
Module &M, bool isConst,
unsigned AddrSpace) const {
auto &DL = M.getDataLayout();
// FIXME: Find better heuristics
llvm::stable_sort(
Expand Down Expand Up @@ -333,7 +345,7 @@ bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
Function *ParentFn = I->getParent()->getParent();

// If we're only optimizing for size, ignore non-minsize functions.
if (OnlyOptimizeForSize && !ParentFn->hasMinSize())
if (Opt.SizeOnly && !ParentFn->hasMinSize())
continue;

size_t UGSIdx = GlobalUsesByFunction[ParentFn];
Expand Down Expand Up @@ -434,9 +446,9 @@ bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
return Changed;
}

bool GlobalMerge::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
const BitVector &GlobalSet, Module &M, bool isConst,
unsigned AddrSpace) const {
bool GlobalMergeImpl::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
const BitVector &GlobalSet, Module &M,
bool isConst, unsigned AddrSpace) const {
assert(Globals.size() > 1);

Type *Int32Ty = Type::getInt32Ty(M.getContext());
Expand Down Expand Up @@ -467,7 +479,7 @@ bool GlobalMerge::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
unsigned Padding = alignTo(MergedSize, Alignment) - MergedSize;
MergedSize += Padding;
MergedSize += DL.getTypeAllocSize(Ty);
if (MergedSize > MaxOffset) {
if (MergedSize > Opt.MaxOffset) {
break;
}
if (Padding) {
Expand Down Expand Up @@ -563,7 +575,7 @@ bool GlobalMerge::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
return Changed;
}

void GlobalMerge::collectUsedGlobalVariables(Module &M, StringRef Name) {
void GlobalMergeImpl::collectUsedGlobalVariables(Module &M, StringRef Name) {
// Extract global variables from llvm.used array
const GlobalVariable *GV = M.getGlobalVariable(Name);
if (!GV || !GV->hasInitializer()) return;
Expand All @@ -577,7 +589,7 @@ void GlobalMerge::collectUsedGlobalVariables(Module &M, StringRef Name) {
MustKeepGlobalVariables.insert(G);
}

void GlobalMerge::setMustKeepGlobalVariables(Module &M) {
void GlobalMergeImpl::setMustKeepGlobalVariables(Module &M) {
collectUsedGlobalVariables(M, "llvm.used");
collectUsedGlobalVariables(M, "llvm.compiler.used");

Expand All @@ -604,7 +616,7 @@ void GlobalMerge::setMustKeepGlobalVariables(Module &M) {
}
}

bool GlobalMerge::doInitialization(Module &M) {
bool GlobalMergeImpl::run(Module &M) {
if (!EnableGlobalMerge)
return false;

Expand Down Expand Up @@ -632,7 +644,7 @@ bool GlobalMerge::doInitialization(Module &M) {
if (TM && !TM->shouldAssumeDSOLocal(M, &GV))
continue;

if (!(MergeExternalGlobals && GV.hasExternalLinkage()) &&
if (!(Opt.MergeExternal && GV.hasExternalLinkage()) &&
!GV.hasInternalLinkage())
continue;

Expand All @@ -659,7 +671,7 @@ bool GlobalMerge::doInitialization(Module &M) {
continue;

Type *Ty = GV.getValueType();
if (DL.getTypeAllocSize(Ty) < MaxOffset) {
if (DL.getTypeAllocSize(Ty) < Opt.MaxOffset) {
if (TM &&
TargetLoweringObjectFile::getKindForGlobal(&GV, *TM).isBSS())
BSSGlobals[{AddressSpace, Section}].push_back(&GV);
Expand All @@ -686,15 +698,6 @@ bool GlobalMerge::doInitialization(Module &M) {
return Changed;
}

bool GlobalMerge::runOnFunction(Function &F) {
return false;
}

bool GlobalMerge::doFinalization(Module &M) {
MustKeepGlobalVariables.clear();
return false;
}

Pass *llvm::createGlobalMergePass(const TargetMachine *TM, unsigned Offset,
bool OnlyOptimizeForSize,
bool MergeExternalByDefault) {
Expand Down
59 changes: 37 additions & 22 deletions llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ static const unsigned MaxParallelChains = 64;
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
const SDValue *Parts, unsigned NumParts,
MVT PartVT, EVT ValueVT, const Value *V,
SDValue InChain,
std::optional<CallingConv::ID> CC);

/// getCopyFromParts - Create a value that contains the specified legal parts
Expand All @@ -163,6 +164,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
static SDValue
getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
SDValue InChain,
std::optional<CallingConv::ID> CC = std::nullopt,
std::optional<ISD::NodeType> AssertOp = std::nullopt) {
// Let the target assemble the parts if it wants to
Expand All @@ -173,7 +175,7 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,

if (ValueVT.isVector())
return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
CC);
InChain, CC);

assert(NumParts > 0 && "No parts to assemble!");
SDValue Val = Parts[0];
Expand All @@ -194,10 +196,10 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

if (RoundParts > 2) {
Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
PartVT, HalfVT, V);
Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
RoundParts / 2, PartVT, HalfVT, V);
Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
InChain);
Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
PartVT, HalfVT, V, InChain);
} else {
Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
Expand All @@ -213,7 +215,7 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
unsigned OddParts = NumParts - RoundParts;
EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
OddVT, V, CC);
OddVT, V, InChain, CC);

// Combine the round and odd parts.
Lo = Val;
Expand Down Expand Up @@ -243,7 +245,8 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split");
EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
InChain, CC);
}
}

Expand Down Expand Up @@ -283,10 +286,20 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
// FP_ROUND's are always exact here.
if (ValueVT.bitsLT(Val.getValueType()))
return DAG.getNode(
ISD::FP_ROUND, DL, ValueVT, Val,
DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
if (ValueVT.bitsLT(Val.getValueType())) {

SDValue NoChange =
DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));

if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
llvm::Attribute::StrictFP)) {
return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
NoChange);
}

return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
}

return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
}
Expand Down Expand Up @@ -324,6 +337,7 @@ static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
const SDValue *Parts, unsigned NumParts,
MVT PartVT, EVT ValueVT, const Value *V,
SDValue InChain,
std::optional<CallingConv::ID> CallConv) {
assert(ValueVT.isVector() && "Not a vector value");
assert(NumParts > 0 && "No parts to assemble!");
Expand Down Expand Up @@ -362,17 +376,17 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
// If the register was not expanded, truncate or copy the value,
// as appropriate.
for (unsigned i = 0; i != NumParts; ++i)
Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
PartVT, IntermediateVT, V, CallConv);
Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
V, InChain, CallConv);
} else if (NumParts > 0) {
// If the intermediate type was expanded, build the intermediate
// operands from the parts.
assert(NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!");
unsigned Factor = NumParts / NumIntermediates;
for (unsigned i = 0; i != NumIntermediates; ++i)
Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
PartVT, IntermediateVT, V, CallConv);
Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
IntermediateVT, V, InChain, CallConv);
}

// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
Expand Down Expand Up @@ -926,7 +940,7 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
}

Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
RegisterVT, ValueVT, V, CallConv);
RegisterVT, ValueVT, V, Chain, CallConv);
Part += NumRegs;
Parts.clear();
}
Expand Down Expand Up @@ -10628,9 +10642,9 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
CLI.CallConv, VT);

ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
NumRegs, RegisterVT, VT, nullptr,
CLI.CallConv, AssertOp));
ReturnValues.push_back(getCopyFromParts(
CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
CLI.Chain, CLI.CallConv, AssertOp));
CurReg += NumRegs;
}

Expand Down Expand Up @@ -11109,8 +11123,9 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
MVT VT = ValueVTs[0].getSimpleVT();
MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
std::optional<ISD::NodeType> AssertOp;
SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
nullptr, F.getCallingConv(), AssertOp);
SDValue ArgValue =
getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
F.getCallingConv(), AssertOp);

MachineFunction& MF = SDB->DAG.getMachineFunction();
MachineRegisterInfo& RegInfo = MF.getRegInfo();
Expand Down Expand Up @@ -11182,7 +11197,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
AssertOp = ISD::AssertZext;

ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
PartVT, VT, nullptr,
PartVT, VT, nullptr, NewRoot,
F.getCallingConv(), AssertOp));
}

Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/DebugInfo/DWARF/DWARFTypePrinter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,7 @@ DWARFTypePrinter::appendUnqualifiedNameBefore(DWARFDie D,
Word = true;
StringRef Name = NamePtr;
static constexpr StringRef MangledPrefix = "_STN|";
if (Name.starts_with(MangledPrefix)) {
Name = Name.drop_front(MangledPrefix.size());
if (Name.consume_front(MangledPrefix)) {
auto Separator = Name.find('|');
assert(Separator != StringRef::npos);
StringRef BaseName = Name.substr(0, Separator);
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,8 @@ Error SymbolizableObjectFile::addSymbol(const SymbolRef &Symbol,
SymbolAddress = OpdExtractor->getAddress(&OpdOffset);
}
// Mach-O symbol table names have leading underscore, skip it.
if (Module->isMachO() && !SymbolName.empty() && SymbolName[0] == '_')
SymbolName = SymbolName.drop_front();
if (Module->isMachO())
SymbolName.consume_front("_");

if (Obj.isELF() && ELFSymbolRef(Symbol).getBinding() != ELF::STB_LOCAL)
ELFSymIdx = 0;
Expand Down
27 changes: 27 additions & 0 deletions llvm/lib/Passes/PassBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
#include "llvm/CodeGen/ExpandLargeFpConvert.h"
#include "llvm/CodeGen/ExpandMemCmp.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/GlobalMerge.h"
#include "llvm/CodeGen/HardwareLoops.h"
#include "llvm/CodeGen/IndirectBrExpand.h"
#include "llvm/CodeGen/InterleavedAccess.h"
Expand Down Expand Up @@ -1082,6 +1083,32 @@ Expected<bool> parseWinEHPrepareOptions(StringRef Params) {
"WinEHPreparePass");
}

Expected<GlobalMergeOptions> parseGlobalMergeOptions(StringRef Params) {
GlobalMergeOptions Result;
while (!Params.empty()) {
StringRef ParamName;
std::tie(ParamName, Params) = Params.split(';');

bool Enable = !ParamName.consume_front("no-");
if (ParamName == "group-by-use")
Result.GroupByUse = Enable;
else if (ParamName == "ignore-single-use")
Result.IgnoreSingleUse = Enable;
else if (ParamName == "merge-const")
Result.MergeConst = Enable;
else if (ParamName == "merge-external")
Result.MergeExternal = Enable;
else if (ParamName.consume_front("max-offset=")) {
if (ParamName.getAsInteger(0, Result.MaxOffset))
return make_error<StringError>(
formatv("invalid GlobalMergePass parameter '{0}' ", ParamName)
.str(),
inconvertibleErrorCode());
}
}
return Result;
}

} // namespace

/// Tests whether a pass name starts with a valid prefix for a default pipeline
Expand Down
7 changes: 7 additions & 0 deletions llvm/lib/Passes/PassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,13 @@ MODULE_PASS_WITH_PARAMS(
"asan", "AddressSanitizerPass",
[](AddressSanitizerOptions Opts) { return AddressSanitizerPass(Opts); },
parseASanPassOptions, "kernel")
MODULE_PASS_WITH_PARAMS(
"global-merge", "GlobalMergePass",
[TM = TM](GlobalMergeOptions Opts) { return GlobalMergePass(TM, Opts); },
parseGlobalMergeOptions,
"group-by-use;ignore-single-use;max-offset=N;merge-const;merge-external;"
"no-group-by-use;no-ignore-single-use;no-merge-const;no-merge-external;"
"size-only")
MODULE_PASS_WITH_PARAMS(
"globaldce", "GlobalDCEPass",
[](bool InLTOPostLink) { return GlobalDCEPass(InLTOPostLink); },
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Support/Caching.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ Expected<FileCache> llvm::localCache(const Twine &CacheNameRef,
// This CacheStream will move the temporary file into the cache when done.
return std::make_unique<CacheStream>(
std::make_unique<raw_fd_ostream>(Temp->FD, /* ShouldClose */ false),
AddBuffer, std::move(*Temp), std::string(EntryPath.str()),
ModuleName.str(), Task);
AddBuffer, std::move(*Temp), std::string(EntryPath), ModuleName.str(),
Task);
};
};
}
2 changes: 1 addition & 1 deletion llvm/lib/Support/FileCollector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ void FileCollector::PathCanonicalizer::updateWithRealPath(
// cases? What if there is nothing on disk?
if (sys::fs::real_path(Directory, RealPath))
return;
CachedDirs[Directory] = std::string(RealPath.str());
CachedDirs[Directory] = std::string(RealPath);
} else {
RealPath = DirWithSymlink->second;
}
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Support/GraphWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ std::string llvm::createGraphFilename(const Twine &Name, int &FD) {
}

errs() << "Writing '" << Filename << "'... ";
return std::string(Filename.str());
return std::string(Filename);
}

// Execute the graph viewer. Return true if there were errors.
Expand Down
8 changes: 4 additions & 4 deletions llvm/lib/Support/LockFileManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ LockFileManager::LockFileManager(StringRef FileName)
this->FileName = FileName;
if (std::error_code EC = sys::fs::make_absolute(this->FileName)) {
std::string S("failed to obtain absolute path for ");
S.append(std::string(this->FileName.str()));
S.append(std::string(this->FileName));
setError(EC, S);
return;
}
Expand All @@ -181,7 +181,7 @@ LockFileManager::LockFileManager(StringRef FileName)
if (std::error_code EC = sys::fs::createUniqueFile(
UniqueLockFileName, UniqueLockFileID, UniqueLockFileName)) {
std::string S("failed to create unique file ");
S.append(std::string(UniqueLockFileName.str()));
S.append(std::string(UniqueLockFileName));
setError(EC, S);
return;
}
Expand All @@ -202,7 +202,7 @@ LockFileManager::LockFileManager(StringRef FileName)
// We failed to write out PID, so report the error, remove the
// unique lock file, and fail.
std::string S("failed to write to ");
S.append(std::string(UniqueLockFileName.str()));
S.append(std::string(UniqueLockFileName));
setError(Out.error(), S);
sys::fs::remove(UniqueLockFileName);
return;
Expand Down Expand Up @@ -248,7 +248,7 @@ LockFileManager::LockFileManager(StringRef FileName)
// ownership.
if ((EC = sys::fs::remove(LockFileName))) {
std::string S("failed to remove lockfile ");
S.append(std::string(UniqueLockFileName.str()));
S.append(std::string(UniqueLockFileName));
setError(EC, S);
return;
}
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Support/Path.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1145,7 +1145,7 @@ void directory_entry::replace_filename(const Twine &Filename, file_type Type,
basic_file_status Status) {
SmallString<128> PathStr = path::parent_path(Path);
path::append(PathStr, Filename);
this->Path = std::string(PathStr.str());
this->Path = std::string(PathStr);
this->Type = Type;
this->Status = Status;
}
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Support/Process.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ Process::FindInEnvPath(StringRef EnvName, StringRef FileName,
SmallString<128> FilePath(Dir);
path::append(FilePath, FileName);
if (fs::exists(Twine(FilePath))) {
FoundPath = std::string(FilePath.str());
FoundPath = std::string(FilePath);
break;
}
}
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Support/Unix/Program.inc
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ ErrorOr<std::string> sys::findProgramByName(StringRef Name,
SmallString<128> FilePath(Path);
sys::path::append(FilePath, Name);
if (sys::fs::can_execute(FilePath.c_str()))
return std::string(FilePath.str()); // Found the executable!
return std::string(FilePath); // Found the executable!
}
return errc::no_such_file_or_directory;
}
Expand Down
12 changes: 6 additions & 6 deletions llvm/lib/Support/VirtualFileSystem.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -325,14 +325,14 @@ RealFileSystem::openFileForRead(const Twine &Name) {

llvm::ErrorOr<std::string> RealFileSystem::getCurrentWorkingDirectory() const {
if (WD && *WD)
return std::string(WD->get().Specified.str());
return std::string(WD->get().Specified);
if (WD)
return WD->getError();

SmallString<128> Dir;
if (std::error_code EC = llvm::sys::fs::current_path(Dir))
return EC;
return std::string(Dir.str());
return std::string(Dir);
}

std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
Expand Down Expand Up @@ -1091,7 +1091,7 @@ class InMemoryFileSystem::DirIterator : public llvm::vfs::detail::DirIterImpl {
}
break;
}
CurrentEntry = directory_entry(std::string(Path.str()), Type);
CurrentEntry = directory_entry(std::string(Path), Type);
} else {
// When we're at the end, make CurrentEntry invalid and DirIterImpl will
// do the rest.
Expand Down Expand Up @@ -1146,7 +1146,7 @@ std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) {
llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);

if (!Path.empty())
WorkingDirectory = std::string(Path.str());
WorkingDirectory = std::string(Path);
return {};
}

Expand Down Expand Up @@ -1252,7 +1252,7 @@ class llvm::vfs::RedirectingFSDirIterImpl
Type = sys::fs::file_type::regular_file;
break;
}
CurrentEntry = directory_entry(std::string(PathStr.str()), Type);
CurrentEntry = directory_entry(std::string(PathStr), Type);
} else {
CurrentEntry = directory_entry();
}
Expand Down Expand Up @@ -1328,7 +1328,7 @@ RedirectingFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
Path.toVector(AbsolutePath);
if (std::error_code EC = makeAbsolute(AbsolutePath))
return EC;
WorkingDirectory = std::string(AbsolutePath.str());
WorkingDirectory = std::string(AbsolutePath);
return {};
}

Expand Down
238 changes: 177 additions & 61 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Large diffs are not rendered by default.

42 changes: 26 additions & 16 deletions llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4086,14 +4086,17 @@ MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
SDValue Control;
unsigned ROpc, MOpc;

#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
if (!PreferBEXTR) {
assert(Subtarget->hasBMI2() && "We must have BMI2's BZHI then.");
// If we can't make use of BEXTR then we can't fuse shift+mask stages.
// Let's perform the mask first, and apply shift later. Note that we need to
// widen the mask to account for the fact that we'll apply shift afterwards!
Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
ROpc = NVT == MVT::i64 ? X86::BZHI64rr : X86::BZHI32rr;
MOpc = NVT == MVT::i64 ? X86::BZHI64rm : X86::BZHI32rm;
ROpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BZHI64rr)
: GET_EGPR_IF_ENABLED(X86::BZHI32rr);
MOpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BZHI64rm)
: GET_EGPR_IF_ENABLED(X86::BZHI32rm);
unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
} else {
Expand All @@ -4108,8 +4111,10 @@ MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
} else {
assert(Subtarget->hasBMI() && "We must have BMI1's BEXTR then.");
// BMI requires the immediate to placed in a register.
ROpc = NVT == MVT::i64 ? X86::BEXTR64rr : X86::BEXTR32rr;
MOpc = NVT == MVT::i64 ? X86::BEXTR64rm : X86::BEXTR32rm;
ROpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BEXTR64rr)
: GET_EGPR_IF_ENABLED(X86::BEXTR32rr);
MOpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BEXTR64rm)
: GET_EGPR_IF_ENABLED(X86::BEXTR32rm);
unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
}
Expand Down Expand Up @@ -5482,25 +5487,30 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
switch (NVT.SimpleTy) {
default: llvm_unreachable("Unsupported VT!");
case MVT::i32:
Opc = UseMULXHi ? X86::MULX32Hrr :
UseMULX ? X86::MULX32rr :
IsSigned ? X86::IMUL32r : X86::MUL32r;
MOpc = UseMULXHi ? X86::MULX32Hrm :
UseMULX ? X86::MULX32rm :
IsSigned ? X86::IMUL32m : X86::MUL32m;
Opc = UseMULXHi ? X86::MULX32Hrr
: UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX32rr)
: IsSigned ? X86::IMUL32r
: X86::MUL32r;
MOpc = UseMULXHi ? X86::MULX32Hrm
: UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX32rm)
: IsSigned ? X86::IMUL32m
: X86::MUL32m;
LoReg = UseMULX ? X86::EDX : X86::EAX;
HiReg = X86::EDX;
break;
case MVT::i64:
Opc = UseMULXHi ? X86::MULX64Hrr :
UseMULX ? X86::MULX64rr :
IsSigned ? X86::IMUL64r : X86::MUL64r;
MOpc = UseMULXHi ? X86::MULX64Hrm :
UseMULX ? X86::MULX64rm :
IsSigned ? X86::IMUL64m : X86::MUL64m;
Opc = UseMULXHi ? X86::MULX64Hrr
: UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX64rr)
: IsSigned ? X86::IMUL64r
: X86::MUL64r;
MOpc = UseMULXHi ? X86::MULX64Hrm
: UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX64rm)
: IsSigned ? X86::IMUL64m
: X86::MUL64m;
LoReg = UseMULX ? X86::RDX : X86::RAX;
HiReg = X86::RDX;
break;
#undef GET_EGPR_IF_ENABLED
}

SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
Expand Down
16 changes: 11 additions & 5 deletions llvm/lib/Target/X86/X86InstrArithmetic.td
Original file line number Diff line number Diff line change
Expand Up @@ -1338,17 +1338,23 @@ defm ANDN32 : AndN<Xi32, "_EVEX">, EVEX, Requires<[HasBMI, HasEGPR, In64BitMode]
defm ANDN64 : AndN<Xi64, "_EVEX">, EVEX, REX_W, Requires<[HasBMI, HasEGPR, In64BitMode]>;
}

let Predicates = [HasBMI], AddedComplexity = -6 in {
multiclass Andn_Pats<string suffix> {
def : Pat<(and (not GR32:$src1), GR32:$src2),
(ANDN32rr GR32:$src1, GR32:$src2)>;
(!cast<Instruction>(ANDN32rr#suffix) GR32:$src1, GR32:$src2)>;
def : Pat<(and (not GR64:$src1), GR64:$src2),
(ANDN64rr GR64:$src1, GR64:$src2)>;
(!cast<Instruction>(ANDN64rr#suffix) GR64:$src1, GR64:$src2)>;
def : Pat<(and (not GR32:$src1), (loadi32 addr:$src2)),
(ANDN32rm GR32:$src1, addr:$src2)>;
(!cast<Instruction>(ANDN32rm#suffix) GR32:$src1, addr:$src2)>;
def : Pat<(and (not GR64:$src1), (loadi64 addr:$src2)),
(ANDN64rm GR64:$src1, addr:$src2)>;
(!cast<Instruction>(ANDN64rm#suffix) GR64:$src1, addr:$src2)>;
}

let Predicates = [HasBMI, NoEGPR], AddedComplexity = -6 in
defm : Andn_Pats<"">;

let Predicates = [HasBMI, HasEGPR], AddedComplexity = -6 in
defm : Andn_Pats<"_EVEX">;

//===----------------------------------------------------------------------===//
// MULX Instruction
//
Expand Down
58 changes: 43 additions & 15 deletions llvm/lib/Target/X86/X86InstrMisc.td
Original file line number Diff line number Diff line change
Expand Up @@ -1241,43 +1241,49 @@ let Predicates = [HasBMI, In64BitMode], Defs = [EFLAGS] in {
defm BLSI64 : Bls<"blsi", MRM3r, MRM3m, Xi64, "_EVEX">, EVEX;
}

let Predicates = [HasBMI] in {
multiclass Bls_Pats<string suffix> {
// FIXME(1): patterns for the load versions are not implemented
// FIXME(2): By only matching `add_su` and `ineg_su` we may emit
// extra `mov` instructions if `src` has future uses. It may be better
// to always match if `src` has more users.
def : Pat<(and GR32:$src, (add_su GR32:$src, -1)),
(BLSR32rr GR32:$src)>;
(!cast<Instruction>(BLSR32rr#suffix) GR32:$src)>;
def : Pat<(and GR64:$src, (add_su GR64:$src, -1)),
(BLSR64rr GR64:$src)>;
(!cast<Instruction>(BLSR64rr#suffix) GR64:$src)>;

def : Pat<(xor GR32:$src, (add_su GR32:$src, -1)),
(BLSMSK32rr GR32:$src)>;
(!cast<Instruction>(BLSMSK32rr#suffix) GR32:$src)>;
def : Pat<(xor GR64:$src, (add_su GR64:$src, -1)),
(BLSMSK64rr GR64:$src)>;
(!cast<Instruction>(BLSMSK64rr#suffix) GR64:$src)>;

def : Pat<(and GR32:$src, (ineg_su GR32:$src)),
(BLSI32rr GR32:$src)>;
(!cast<Instruction>(BLSI32rr#suffix) GR32:$src)>;
def : Pat<(and GR64:$src, (ineg_su GR64:$src)),
(BLSI64rr GR64:$src)>;
(!cast<Instruction>(BLSI64rr#suffix) GR64:$src)>;

// Versions to match flag producing ops.
def : Pat<(and_flag_nocf GR32:$src, (add_su GR32:$src, -1)),
(BLSR32rr GR32:$src)>;
(!cast<Instruction>(BLSR32rr#suffix) GR32:$src)>;
def : Pat<(and_flag_nocf GR64:$src, (add_su GR64:$src, -1)),
(BLSR64rr GR64:$src)>;
(!cast<Instruction>(BLSR64rr#suffix) GR64:$src)>;

def : Pat<(xor_flag_nocf GR32:$src, (add_su GR32:$src, -1)),
(BLSMSK32rr GR32:$src)>;
(!cast<Instruction>(BLSMSK32rr#suffix) GR32:$src)>;
def : Pat<(xor_flag_nocf GR64:$src, (add_su GR64:$src, -1)),
(BLSMSK64rr GR64:$src)>;
(!cast<Instruction>(BLSMSK64rr#suffix) GR64:$src)>;

def : Pat<(and_flag_nocf GR32:$src, (ineg_su GR32:$src)),
(BLSI32rr GR32:$src)>;
(!cast<Instruction>(BLSI32rr#suffix) GR32:$src)>;
def : Pat<(and_flag_nocf GR64:$src, (ineg_su GR64:$src)),
(BLSI64rr GR64:$src)>;
(!cast<Instruction>(BLSI64rr#suffix) GR64:$src)>;
}

let Predicates = [HasBMI, NoEGPR] in
defm : Bls_Pats<"">;

let Predicates = [HasBMI, HasEGPR] in
defm : Bls_Pats<"_EVEX">;

multiclass Bmi4VOp3<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node,
X86FoldableSchedWrite sched, string Suffix = ""> {
let SchedRW = [sched], Form = MRMSrcReg4VOp3 in
Expand Down Expand Up @@ -1324,7 +1330,7 @@ def AndMask64 : ImmLeaf<i64, [{
}]>;

// Use BEXTR for 64-bit 'and' with large immediate 'mask'.
let Predicates = [HasBMI, NoBMI2, NoTBM] in {
let Predicates = [HasBMI, NoBMI2, NoTBM, NoEGPR] in {
def : Pat<(and GR64:$src, AndMask64:$mask),
(BEXTR64rr GR64:$src,
(SUBREG_TO_REG (i64 0),
Expand All @@ -1335,8 +1341,19 @@ let Predicates = [HasBMI, NoBMI2, NoTBM] in {
(MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
}

let Predicates = [HasBMI, NoBMI2, NoTBM, HasEGPR] in {
def : Pat<(and GR64:$src, AndMask64:$mask),
(BEXTR64rr_EVEX GR64:$src,
(SUBREG_TO_REG (i64 0),
(MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
(BEXTR64rm_EVEX addr:$src,
(SUBREG_TO_REG (i64 0),
(MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
}

// Use BZHI for 64-bit 'and' with large immediate 'mask'.
let Predicates = [HasBMI2, NoTBM] in {
let Predicates = [HasBMI2, NoTBM, NoEGPR] in {
def : Pat<(and GR64:$src, AndMask64:$mask),
(BZHI64rr GR64:$src,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)),
Expand All @@ -1347,6 +1364,17 @@ let Predicates = [HasBMI2, NoTBM] in {
(MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
}

let Predicates = [HasBMI2, NoTBM, HasEGPR] in {
def : Pat<(and GR64:$src, AndMask64:$mask),
(BZHI64rr_EVEX GR64:$src,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)),
(MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
(BZHI64rm_EVEX addr:$src,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)),
(MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
}

multiclass bmi_pdep_pext<string mnemonic, RegisterClass RC,
X86MemOperand x86memop, SDPatternOperator OpNode,
PatFrag ld_frag, string Suffix = ""> {
Expand Down
47 changes: 27 additions & 20 deletions llvm/lib/Target/X86/X86InstrShiftRotate.td
Original file line number Diff line number Diff line change
Expand Up @@ -284,32 +284,32 @@ defm SHRX64: ShiftX<"shrx", Xi64>, XD;
defm SHLX32: ShiftX<"shlx", Xi32>, PD;
defm SHLX64: ShiftX<"shlx", Xi64>, PD;

multiclass RORX_Pats {
multiclass RORX_Pats<string suffix> {
// Prefer RORX which is non-destructive and doesn't update EFLAGS.
let AddedComplexity = 10 in {
def : Pat<(rotr GR32:$src, (i8 imm:$shamt)),
(RORX32ri GR32:$src, imm:$shamt)>;
(!cast<Instruction>(RORX32ri#suffix) GR32:$src, imm:$shamt)>;
def : Pat<(rotr GR64:$src, (i8 imm:$shamt)),
(RORX64ri GR64:$src, imm:$shamt)>;
(!cast<Instruction>(RORX64ri#suffix) GR64:$src, imm:$shamt)>;

def : Pat<(rotl GR32:$src, (i8 imm:$shamt)),
(RORX32ri GR32:$src, (ROT32L2R_imm8 imm:$shamt))>;
(!cast<Instruction>(RORX32ri#suffix) GR32:$src, (ROT32L2R_imm8 imm:$shamt))>;
def : Pat<(rotl GR64:$src, (i8 imm:$shamt)),
(RORX64ri GR64:$src, (ROT64L2R_imm8 imm:$shamt))>;
(!cast<Instruction>(RORX64ri#suffix) GR64:$src, (ROT64L2R_imm8 imm:$shamt))>;
}

def : Pat<(rotr (loadi32 addr:$src), (i8 imm:$shamt)),
(RORX32mi addr:$src, imm:$shamt)>;
(!cast<Instruction>(RORX32mi#suffix) addr:$src, imm:$shamt)>;
def : Pat<(rotr (loadi64 addr:$src), (i8 imm:$shamt)),
(RORX64mi addr:$src, imm:$shamt)>;
(!cast<Instruction>(RORX64mi#suffix) addr:$src, imm:$shamt)>;

def : Pat<(rotl (loadi32 addr:$src), (i8 imm:$shamt)),
(RORX32mi addr:$src, (ROT32L2R_imm8 imm:$shamt))>;
(!cast<Instruction>(RORX32mi#suffix) addr:$src, (ROT32L2R_imm8 imm:$shamt))>;
def : Pat<(rotl (loadi64 addr:$src), (i8 imm:$shamt)),
(RORX64mi addr:$src, (ROT64L2R_imm8 imm:$shamt))>;
(!cast<Instruction>(RORX64mi#suffix) addr:$src, (ROT64L2R_imm8 imm:$shamt))>;
}

multiclass ShiftX_Pats<SDNode op> {
multiclass ShiftX_Pats<SDNode op, string suffix = ""> {
// Prefer SARX/SHRX/SHLX over SAR/SHR/SHL with variable shift BUT not
// immediate shift, i.e. the following code is considered better
//
Expand All @@ -325,16 +325,16 @@ multiclass ShiftX_Pats<SDNode op> {
//
let AddedComplexity = 1 in {
def : Pat<(op GR32:$src1, GR8:$src2),
(!cast<Instruction>(NAME#"32rr") GR32:$src1,
(!cast<Instruction>(NAME#"32rr"#suffix) GR32:$src1,
(INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
def : Pat<(op GR64:$src1, GR8:$src2),
(!cast<Instruction>(NAME#"64rr") GR64:$src1,
(!cast<Instruction>(NAME#"64rr"#suffix) GR64:$src1,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
def : Pat<(op GR32:$src1, (shiftMask32 GR8:$src2)),
(!cast<Instruction>(NAME#"32rr") GR32:$src1,
(!cast<Instruction>(NAME#"32rr"#suffix) GR32:$src1,
(INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
def : Pat<(op GR64:$src1, (shiftMask64 GR8:$src2)),
(!cast<Instruction>(NAME#"64rr") GR64:$src1,
(!cast<Instruction>(NAME#"64rr"#suffix) GR64:$src1,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
}
// We prefer to use
Expand All @@ -348,22 +348,29 @@ multiclass ShiftX_Pats<SDNode op> {
//
// This priority is enforced by IsProfitableToFoldLoad.
def : Pat<(op (loadi32 addr:$src1), GR8:$src2),
(!cast<Instruction>(NAME#"32rm") addr:$src1,
(!cast<Instruction>(NAME#"32rm"#suffix) addr:$src1,
(INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
def : Pat<(op (loadi64 addr:$src1), GR8:$src2),
(!cast<Instruction>(NAME#"64rm") addr:$src1,
(!cast<Instruction>(NAME#"64rm"#suffix) addr:$src1,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
def : Pat<(op (loadi32 addr:$src1), (shiftMask32 GR8:$src2)),
(!cast<Instruction>(NAME#"32rm") addr:$src1,
(!cast<Instruction>(NAME#"32rm"#suffix) addr:$src1,
(INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
def : Pat<(op (loadi64 addr:$src1), (shiftMask64 GR8:$src2)),
(!cast<Instruction>(NAME#"64rm") addr:$src1,
(!cast<Instruction>(NAME#"64rm"#suffix) addr:$src1,
(INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
}

let Predicates = [HasBMI2] in {
defm : RORX_Pats;
let Predicates = [HasBMI2, NoEGPR] in {
defm : RORX_Pats<"">;
defm SARX : ShiftX_Pats<sra>;
defm SHRX : ShiftX_Pats<srl>;
defm SHLX : ShiftX_Pats<shl>;
}

let Predicates = [HasBMI2, HasEGPR] in {
defm : RORX_Pats<"_EVEX">;
defm SARX : ShiftX_Pats<sra, "_EVEX">;
defm SHRX : ShiftX_Pats<srl, "_EVEX">;
defm SHLX : ShiftX_Pats<shl, "_EVEX">;
}
51 changes: 5 additions & 46 deletions llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
Original file line number Diff line number Diff line change
Expand Up @@ -1094,52 +1094,11 @@ define <4 x i1> @isnan_v4bf16(<4 x bfloat> %x) nounwind {
ret <4 x i1> %1
}

define i1 @isnan_bf16_strictfp(bfloat %x) strictfp nounwind {
; GFX7CHECK-LABEL: isnan_bf16_strictfp:
; GFX7CHECK: ; %bb.0:
; GFX7CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7CHECK-NEXT: v_bfe_u32 v0, v0, 16, 15
; GFX7CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX7CHECK-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX7CHECK-NEXT: s_setpc_b64 s[30:31]
;
; GFX8CHECK-LABEL: isnan_bf16_strictfp:
; GFX8CHECK: ; %bb.0:
; GFX8CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; GFX8CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v0
; GFX8CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX8CHECK-NEXT: s_setpc_b64 s[30:31]
;
; GFX9CHECK-LABEL: isnan_bf16_strictfp:
; GFX9CHECK: ; %bb.0:
; GFX9CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; GFX9CHECK-NEXT: s_movk_i32 s4, 0x7f80
; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v0
; GFX9CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9CHECK-NEXT: s_setpc_b64 s[30:31]
;
; GFX10CHECK-LABEL: isnan_bf16_strictfp:
; GFX10CHECK: ; %bb.0:
; GFX10CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
; GFX11CHECK-LABEL: isnan_bf16_strictfp:
; GFX11CHECK: ; %bb.0:
; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0
; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
%1 = call i1 @llvm.is.fpclass.bf16(bfloat %x, i32 3) strictfp ; nan
ret i1 %1
}
; FIXME: Broken for gfx6/7
; define i1 @isnan_bf16_strictfp(bfloat %x) strictfp nounwind {
; %1 = call i1 @llvm.is.fpclass.bf16(bfloat %x, i32 3) strictfp ; nan
; ret i1 %1
; }

define i1 @isinf_bf16(bfloat %x) nounwind {
; GFX7CHECK-LABEL: isinf_bf16:
Expand Down
3 changes: 3 additions & 0 deletions llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
Original file line number Diff line number Diff line change
Expand Up @@ -1316,6 +1316,9 @@ define i1 @isnan_f16_strictfp(half %x) strictfp nounwind {
; GFX7SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7SELDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7SELDAG-NEXT: s_movk_i32 s4, 0x7c00
; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7SELDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7SELDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7SELDAG-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; GFX7SELDAG-NEXT: v_cmp_lt_i32_e32 vcc, s4, v0
; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
Expand Down
110 changes: 0 additions & 110 deletions llvm/test/CodeGen/AMDGPU/strict_fp_casts.ll

This file was deleted.

Loading