diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 8b702af082fc2..2526d0cd56dba 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -77,6 +77,7 @@ add_header_library( libc.include.errno libc.src.errno.errno libc.src.__support.CPP.limits + libc.src.__support.common ) add_header_library( @@ -98,6 +99,7 @@ add_header_library( DEPENDS libc.src.__support.CPP.type_traits libc.src.__support.FPUtil.fp_bits + libc.src.__support.common ) add_header_library( @@ -112,6 +114,7 @@ add_header_library( str_to_float HDRS str_to_float.h + detailed_powers_of_ten.h DEPENDS .ctype_utils .high_precision_decimal @@ -121,6 +124,7 @@ add_header_library( libc.src.__support.CPP.limits libc.src.__support.FPUtil.fp_bits libc.src.__support.builtin_wrappers + libc.src.__support.common libc.src.errno.errno ) diff --git a/libc/src/__support/CPP/bitset.h b/libc/src/__support/CPP/bitset.h index 534985661c4ce..d3fc0eeda83a6 100644 --- a/libc/src/__support/CPP/bitset.h +++ b/libc/src/__support/CPP/bitset.h @@ -78,7 +78,7 @@ template struct bitset { static constexpr size_t NUMBER_OF_UNITS = (NumberOfBits + BITS_PER_UNIT - 1) / BITS_PER_UNIT; - static inline size_t mask(size_t Index) { + static constexpr size_t mask(size_t Index) { return size_t{1} << (Index % BITS_PER_UNIT); } size_t Data[NUMBER_OF_UNITS] = {0}; diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h index 353a17e6e1e04..7f735656736ac 100644 --- a/libc/src/__support/FPUtil/BasicOperations.h +++ b/libc/src/__support/FPUtil/BasicOperations.h @@ -12,19 +12,20 @@ #include "FPBits.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" namespace __llvm_libc { namespace fputil { template , int> = 0> -static inline T abs(T x) { +LIBC_INLINE T abs(T x) { FPBits bits(x); bits.set_sign(0); return T(bits); } template , int> = 0> -static inline T fmin(T x, T y) { +LIBC_INLINE T fmin(T x, T y) { FPBits bitx(x), bity(y); if (bitx.is_nan()) { @@ -42,7 +43,7 @@ static inline T fmin(T x, T y) { } template , int> = 0> -static inline T fmax(T x, T y) { +LIBC_INLINE T fmax(T x, T y) { FPBits bitx(x), bity(y); if (bitx.is_nan()) { @@ -60,7 +61,7 @@ static inline T fmax(T x, T y) { } template , int> = 0> -static inline T fdim(T x, T y) { +LIBC_INLINE T fdim(T x, T y) { FPBits bitx(x), bity(y); if (bitx.is_nan()) { diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt index 448a1549a9715..84e5346a175ae 100644 --- a/libc/src/__support/FPUtil/CMakeLists.txt +++ b/libc/src/__support/FPUtil/CMakeLists.txt @@ -46,6 +46,7 @@ add_header_library( .fp_bits .fenv_impl libc.src.__support.CPP.type_traits + libc.src.__support.common libc.include.math libc.include.errno libc.src.errno.errno @@ -71,6 +72,7 @@ add_header_library( .platform_defs libc.src.__support.CPP.bit libc.src.__support.CPP.type_traits + libc.src.__support.common libc.include.math libc.include.errno libc.src.errno.errno @@ -83,6 +85,7 @@ add_header_library( DEPENDS .fp_bits libc.src.__support.CPP.type_traits + libc.src.__support.common ) add_header_library( @@ -94,6 +97,7 @@ add_header_library( .manipulation_functions .normal_float libc.src.__support.CPP.type_traits + libc.src.__support.common ) add_header_library( @@ -118,6 +122,7 @@ add_header_library( libc.src.__support.builtin_wrappers libc.src.__support.CPP.bit libc.src.__support.CPP.type_traits + libc.src.__support.common libc.src.__support.uint128 ) @@ -166,6 +171,7 @@ add_header_library( PolyEval.h DEPENDS .multiply_add + libc.src.__support.common ) add_header_library( diff --git a/libc/src/__support/FPUtil/DivisionAndRemainderOperations.h b/libc/src/__support/FPUtil/DivisionAndRemainderOperations.h index 6a55525108d68..7b9d7fa49d7ac 100644 --- a/libc/src/__support/FPUtil/DivisionAndRemainderOperations.h +++ b/libc/src/__support/FPUtil/DivisionAndRemainderOperations.h @@ -14,6 +14,7 @@ #include "NormalFloat.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" namespace __llvm_libc { namespace fputil { @@ -23,7 +24,7 @@ static constexpr int QUOTIENT_LSB_BITS = 3; // The implementation is a bit-by-bit algorithm which uses integer division // to evaluate the quotient and remainder. template , int> = 0> -static inline T remquo(T x, T y, int &q) { +LIBC_INLINE T remquo(T x, T y, int &q) { FPBits xbits(x), ybits(y); if (xbits.is_nan()) return x; diff --git a/libc/src/__support/FPUtil/FEnvImpl.h b/libc/src/__support/FPUtil/FEnvImpl.h index f507b304518f9..85818a67dfa05 100644 --- a/libc/src/__support/FPUtil/FEnvImpl.h +++ b/libc/src/__support/FPUtil/FEnvImpl.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_FENVIMPL_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if defined(LLVM_LIBC_ARCH_AARCH64) #if defined(__APPLE__) @@ -27,21 +28,21 @@ namespace fputil { // All dummy functions silently succeed. -static inline int clear_except(int) { return 0; } +LIBC_INLINE int clear_except(int) { return 0; } -static inline int test_except(int) { return 0; } +LIBC_INLINE int test_except(int) { return 0; } -static inline int set_except(int) { return 0; } +LIBC_INLINE int set_except(int) { return 0; } -static inline int raise_except(int) { return 0; } +LIBC_INLINE int raise_except(int) { return 0; } -static inline int get_round() { return FE_TONEAREST; } +LIBC_INLINE int get_round() { return FE_TONEAREST; } -static inline int set_round(int) { return 0; } +LIBC_INLINE int set_round(int) { return 0; } -static inline int get_env(fenv_t *) { return 0; } +LIBC_INLINE int get_env(fenv_t *) { return 0; } -static inline int set_env(const fenv_t *) { return 0; } +LIBC_INLINE int set_env(const fenv_t *) { return 0; } } // namespace fputil } // namespace __llvm_libc diff --git a/libc/src/__support/FPUtil/FMA.h b/libc/src/__support/FPUtil/FMA.h index 82db04e8f04ce..f14a978f53e06 100644 --- a/libc/src/__support/FPUtil/FMA.h +++ b/libc/src/__support/FPUtil/FMA.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_FMA_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if defined(LIBC_TARGET_HAS_FMA) @@ -27,7 +28,7 @@ namespace __llvm_libc { namespace fputil { -template static inline T fma(T x, T y, T z) { +template LIBC_INLINE T fma(T x, T y, T z) { return generic::fma(x, y, z); } diff --git a/libc/src/__support/FPUtil/Hypot.h b/libc/src/__support/FPUtil/Hypot.h index de22691f7de79..86817b7496541 100644 --- a/libc/src/__support/FPUtil/Hypot.h +++ b/libc/src/__support/FPUtil/Hypot.h @@ -16,6 +16,7 @@ #include "src/__support/CPP/type_traits.h" #include "src/__support/UInt128.h" #include "src/__support/builtin_wrappers.h" +#include "src/__support/common.h" namespace __llvm_libc { namespace fputil { @@ -23,7 +24,7 @@ namespace fputil { namespace internal { template -static inline T find_leading_one(T mant, int &shift_length) { +LIBC_INLINE T find_leading_one(T mant, int &shift_length) { shift_length = 0; if (mant > 0) { shift_length = (sizeof(mant) * 8) - 1 - unsafe_clz(mant); @@ -98,7 +99,7 @@ template <> struct DoubleLength { // - HYPOT(x, y) is NaN if x or y is NaN. // template , int> = 0> -static inline T hypot(T x, T y) { +LIBC_INLINE T hypot(T x, T y) { using FPBits_t = FPBits; using UIntType = typename FPBits::UIntType; using DUIntType = typename DoubleLength::Type; diff --git a/libc/src/__support/FPUtil/ManipulationFunctions.h b/libc/src/__support/FPUtil/ManipulationFunctions.h index 60c0e4887cd8d..946d35c03b1ce 100644 --- a/libc/src/__support/FPUtil/ManipulationFunctions.h +++ b/libc/src/__support/FPUtil/ManipulationFunctions.h @@ -16,6 +16,7 @@ #include "src/__support/CPP/bit.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" #include #include @@ -24,7 +25,7 @@ namespace __llvm_libc { namespace fputil { template , int> = 0> -static inline T frexp(T x, int &exp) { +LIBC_INLINE T frexp(T x, int &exp) { FPBits bits(x); if (bits.is_inf_or_nan()) return x; @@ -40,7 +41,7 @@ static inline T frexp(T x, int &exp) { } template , int> = 0> -static inline T modf(T x, T &iptr) { +LIBC_INLINE T modf(T x, T &iptr) { FPBits bits(x); if (bits.is_zero() || bits.is_nan()) { iptr = x; @@ -61,14 +62,14 @@ static inline T modf(T x, T &iptr) { } template , int> = 0> -static inline T copysign(T x, T y) { +LIBC_INLINE T copysign(T x, T y) { FPBits xbits(x); xbits.set_sign(FPBits(y).get_sign()); return T(xbits); } template , int> = 0> -static inline int ilogb(T x) { +LIBC_INLINE int ilogb(T x) { // TODO: Raise appropriate floating point exceptions and set errno to the // an appropriate error value wherever relevant. FPBits bits(x); @@ -96,7 +97,7 @@ static inline int ilogb(T x) { } template , int> = 0> -static inline T logb(T x) { +LIBC_INLINE T logb(T x) { FPBits bits(x); if (bits.is_zero()) { // TODO(Floating point exception): Raise div-by-zero exception. @@ -114,7 +115,7 @@ static inline T logb(T x) { } template , int> = 0> -static inline T ldexp(T x, int exp) { +LIBC_INLINE T ldexp(T x, int exp) { FPBits bits(x); if (bits.is_zero() || bits.is_inf_or_nan() || exp == 0) return x; @@ -140,7 +141,7 @@ static inline T ldexp(T x, int exp) { } template , int> = 0> -static inline T nextafter(T from, T to) { +LIBC_INLINE T nextafter(T from, T to) { FPBits from_bits(from); if (from_bits.is_nan()) return from; diff --git a/libc/src/__support/FPUtil/NearestIntegerOperations.h b/libc/src/__support/FPUtil/NearestIntegerOperations.h index 3a9d0d121c6a9..968e7d499ae27 100644 --- a/libc/src/__support/FPUtil/NearestIntegerOperations.h +++ b/libc/src/__support/FPUtil/NearestIntegerOperations.h @@ -13,6 +13,7 @@ #include "FPBits.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" #include #include @@ -21,7 +22,7 @@ namespace __llvm_libc { namespace fputil { template , int> = 0> -static inline T trunc(T x) { +LIBC_INLINE T trunc(T x) { FPBits bits(x); // If x is infinity or NaN, return it. @@ -52,7 +53,7 @@ static inline T trunc(T x) { } template , int> = 0> -static inline T ceil(T x) { +LIBC_INLINE T ceil(T x) { FPBits bits(x); // If x is infinity NaN or zero, return it. @@ -90,7 +91,7 @@ static inline T ceil(T x) { } template , int> = 0> -static inline T floor(T x) { +LIBC_INLINE T floor(T x) { FPBits bits(x); if (bits.get_sign()) { return -ceil(-x); @@ -100,7 +101,7 @@ static inline T floor(T x) { } template , int> = 0> -static inline T round(T x) { +LIBC_INLINE T round(T x) { using UIntType = typename FPBits::UIntType; FPBits bits(x); @@ -151,7 +152,7 @@ static inline T round(T x) { } template , int> = 0> -static inline T round_using_current_rounding_mode(T x) { +LIBC_INLINE T round_using_current_rounding_mode(T x) { using UIntType = typename FPBits::UIntType; FPBits bits(x); @@ -232,7 +233,7 @@ namespace internal { template && cpp::is_integral_v, int> = 0> -static inline I rounded_float_to_signed_integer(F x) { +LIBC_INLINE I rounded_float_to_signed_integer(F x) { constexpr I INTEGER_MIN = (I(1) << (sizeof(I) * 8 - 1)); constexpr I INTEGER_MAX = -(INTEGER_MIN + 1); FPBits bits(x); @@ -273,14 +274,14 @@ static inline I rounded_float_to_signed_integer(F x) { template && cpp::is_integral_v, int> = 0> -static inline I round_to_signed_integer(F x) { +LIBC_INLINE I round_to_signed_integer(F x) { return internal::rounded_float_to_signed_integer(round(x)); } template && cpp::is_integral_v, int> = 0> -static inline I round_to_signed_integer_using_current_rounding_mode(F x) { +LIBC_INLINE I round_to_signed_integer_using_current_rounding_mode(F x) { return internal::rounded_float_to_signed_integer( round_using_current_rounding_mode(x)); } diff --git a/libc/src/__support/FPUtil/PolyEval.h b/libc/src/__support/FPUtil/PolyEval.h index b6faa0f897880..43b8ff4e8e1f6 100644 --- a/libc/src/__support/FPUtil/PolyEval.h +++ b/libc/src/__support/FPUtil/PolyEval.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_POLYEVAL_H #include "multiply_add.h" +#include "src/__support/common.h" // Evaluate polynomial using Horner's Scheme: // With polyeval(x, a_0, a_1, ..., a_n) = a_n * x^n + ... + a_1 * x + a_0, we @@ -21,10 +22,10 @@ namespace __llvm_libc { namespace fputil { -template static inline T polyeval(T, T a0) { return a0; } +template LIBC_INLINE T polyeval(T, T a0) { return a0; } template -static inline T polyeval(T x, T a0, Ts... a) { +LIBC_INLINE T polyeval(T x, T a0, Ts... a) { return multiply_add(x, polyeval(x, a...), a0); } diff --git a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h index 22c3513360cbf..213966ab98f4f 100644 --- a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h +++ b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_AARCH64_FENVIMPL_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if !defined(LLVM_LIBC_ARCH_AARCH64) || defined(__APPLE__) #error "Invalid include" @@ -50,7 +51,7 @@ struct FEnv { static constexpr uint32_t ExceptionStatusFlagsBitPosition = 0; static constexpr uint32_t ExceptionControlFlagsBitPosition = 8; - static inline uint32_t getStatusValueForExcept(int excepts) { + LIBC_INLINE uint32_t getStatusValueForExcept(int excepts) { return (excepts & FE_INVALID ? INVALID : 0) | (excepts & FE_DIVBYZERO ? DIVBYZERO : 0) | (excepts & FE_OVERFLOW ? OVERFLOW : 0) | @@ -58,7 +59,7 @@ struct FEnv { (excepts & FE_INEXACT ? INEXACT : 0); } - static inline int exceptionStatusToMacro(uint32_t status) { + LIBC_INLINE int exceptionStatusToMacro(uint32_t status) { return (status & INVALID ? FE_INVALID : 0) | (status & DIVBYZERO ? FE_DIVBYZERO : 0) | (status & OVERFLOW ? FE_OVERFLOW : 0) | @@ -75,7 +76,7 @@ struct FEnv { static void writeStatusWord(uint32_t fpsr) { __arm_wsr("fpsr", fpsr); } }; -static inline int enable_except(int excepts) { +LIBC_INLINE int enable_except(int excepts) { uint32_t newExcepts = FEnv::getStatusValueForExcept(excepts); uint32_t controlWord = FEnv::getControlWord(); int oldExcepts = @@ -85,7 +86,7 @@ static inline int enable_except(int excepts) { return FEnv::exceptionStatusToMacro(oldExcepts); } -static inline int disable_except(int excepts) { +LIBC_INLINE int disable_except(int excepts) { uint32_t disabledExcepts = FEnv::getStatusValueForExcept(excepts); uint32_t controlWord = FEnv::getControlWord(); int oldExcepts = @@ -95,14 +96,14 @@ static inline int disable_except(int excepts) { return FEnv::exceptionStatusToMacro(oldExcepts); } -static inline int get_except() { +LIBC_INLINE int get_except() { uint32_t controlWord = FEnv::getControlWord(); int enabledExcepts = (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F; return FEnv::exceptionStatusToMacro(enabledExcepts); } -static inline int clear_except(int excepts) { +LIBC_INLINE int clear_except(int excepts) { uint32_t statusWord = FEnv::getStatusWord(); uint32_t toClear = FEnv::getStatusValueForExcept(excepts); statusWord &= ~(toClear << FEnv::ExceptionStatusFlagsBitPosition); @@ -110,14 +111,14 @@ static inline int clear_except(int excepts) { return 0; } -static inline int test_except(int excepts) { +LIBC_INLINE int test_except(int excepts) { uint32_t toTest = FEnv::getStatusValueForExcept(excepts); uint32_t statusWord = FEnv::getStatusWord(); return FEnv::exceptionStatusToMacro( (statusWord >> FEnv::ExceptionStatusFlagsBitPosition) & toTest); } -static inline int set_except(int excepts) { +LIBC_INLINE int set_except(int excepts) { uint32_t statusWord = FEnv::getStatusWord(); uint32_t statusValue = FEnv::getStatusValueForExcept(excepts); statusWord |= (statusValue << FEnv::ExceptionStatusFlagsBitPosition); @@ -125,7 +126,7 @@ static inline int set_except(int excepts) { return 0; } -static inline int raise_except(int excepts) { +LIBC_INLINE int raise_except(int excepts) { float zero = 0.0f; float one = 1.0f; float largeValue = float(FPBits(FPBits::MAX_NORMAL)); @@ -185,7 +186,7 @@ static inline int raise_except(int excepts) { return result; } -static inline int get_round() { +LIBC_INLINE int get_round() { uint32_t roundingMode = (FEnv::getControlWord() >> FEnv::RoundingControlBitPosition) & 0x3; switch (roundingMode) { @@ -202,7 +203,7 @@ static inline int get_round() { } } -static inline int set_round(int mode) { +LIBC_INLINE int set_round(int mode) { uint16_t bitValue; switch (mode) { case FE_TONEAREST: @@ -229,14 +230,14 @@ static inline int set_round(int mode) { return 0; } -static inline int get_env(fenv_t *envp) { +LIBC_INLINE int get_env(fenv_t *envp) { FEnv::FPState *state = reinterpret_cast(envp); state->ControlWord = FEnv::getControlWord(); state->StatusWord = FEnv::getStatusWord(); return 0; } -static inline int set_env(const fenv_t *envp) { +LIBC_INLINE int set_env(const fenv_t *envp) { if (envp == FE_DFL_ENV) { // Default status and control words bits are all zeros so we just // write zeros. diff --git a/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h b/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h index 5cc096112061a..a916dc54f3a54 100644 --- a/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h +++ b/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_AARCH64_FENV_DARWIN_IMPL_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if !defined(LLVM_LIBC_ARCH_AARCH64) || !defined(__APPLE__) #error "Invalid include" @@ -60,7 +61,7 @@ struct FEnv { // __fpcr_flush_to_zero bit in the FPCR register. This control bit is // located in a different place from FE_FLUSHTOZERO status bit relative to // the other exceptions. - static inline uint32_t exception_value_from_status(int status) { + LIBC_INLINE uint32_t exception_value_from_status(int status) { return (status & FE_INVALID ? EX_INVALID : 0) | (status & FE_DIVBYZERO ? EX_DIVBYZERO : 0) | (status & FE_OVERFLOW ? EX_OVERFLOW : 0) | @@ -69,7 +70,7 @@ struct FEnv { (status & FE_FLUSHTOZERO ? EX_FLUSHTOZERO : 0); } - static inline uint32_t exception_value_from_control(int control) { + LIBC_INLINE uint32_t exception_value_from_control(int control) { return (control & __fpcr_trap_invalid ? EX_INVALID : 0) | (control & __fpcr_trap_divbyzero ? EX_DIVBYZERO : 0) | (control & __fpcr_trap_overflow ? EX_OVERFLOW : 0) | @@ -78,7 +79,7 @@ struct FEnv { (control & __fpcr_flush_to_zero ? EX_FLUSHTOZERO : 0); } - static inline int exception_value_to_status(uint32_t excepts) { + LIBC_INLINE int exception_value_to_status(uint32_t excepts) { return (excepts & EX_INVALID ? FE_INVALID : 0) | (excepts & EX_DIVBYZERO ? FE_DIVBYZERO : 0) | (excepts & EX_OVERFLOW ? FE_OVERFLOW : 0) | @@ -87,7 +88,7 @@ struct FEnv { (excepts & EX_FLUSHTOZERO ? FE_FLUSHTOZERO : 0); } - static inline int exception_value_to_control(uint32_t excepts) { + LIBC_INLINE int exception_value_to_control(uint32_t excepts) { return (excepts & EX_INVALID ? __fpcr_trap_invalid : 0) | (excepts & EX_DIVBYZERO ? __fpcr_trap_divbyzero : 0) | (excepts & EX_OVERFLOW ? __fpcr_trap_overflow : 0) | @@ -105,7 +106,7 @@ struct FEnv { static void set_status_word(uint32_t fpsr) { __arm_wsr("fpsr", fpsr); } }; -static inline int enable_except(int excepts) { +LIBC_INLINE int enable_except(int excepts) { uint32_t new_excepts = FEnv::exception_value_from_status(excepts); uint32_t control_word = FEnv::get_control_word(); uint32_t old_excepts = FEnv::exception_value_from_control(control_word); @@ -116,7 +117,7 @@ static inline int enable_except(int excepts) { return FEnv::exception_value_to_status(old_excepts); } -static inline int disable_except(int excepts) { +LIBC_INLINE int disable_except(int excepts) { uint32_t disabled_excepts = FEnv::exception_value_from_status(excepts); uint32_t control_word = FEnv::get_control_word(); uint32_t old_excepts = FEnv::exception_value_from_control(control_word); @@ -125,13 +126,13 @@ static inline int disable_except(int excepts) { return FEnv::exception_value_to_status(old_excepts); } -static inline int get_except() { +LIBC_INLINE int get_except() { uint32_t control_word = FEnv::get_control_word(); uint32_t enabled_excepts = FEnv::exception_value_from_control(control_word); return FEnv::exception_value_to_status(enabled_excepts); } -static inline int clear_except(int excepts) { +LIBC_INLINE int clear_except(int excepts) { uint32_t status_word = FEnv::get_status_word(); uint32_t except_value = FEnv::exception_value_from_status(excepts); status_word &= ~FEnv::exception_value_to_status(except_value); @@ -139,13 +140,13 @@ static inline int clear_except(int excepts) { return 0; } -static inline int test_except(int excepts) { +LIBC_INLINE int test_except(int excepts) { uint32_t statusWord = FEnv::get_status_word(); uint32_t ex_value = FEnv::exception_value_from_status(excepts); return statusWord & FEnv::exception_value_to_status(ex_value); } -static inline int set_except(int excepts) { +LIBC_INLINE int set_except(int excepts) { uint32_t status_word = FEnv::get_status_word(); uint32_t new_exceptions = FEnv::exception_value_from_status(excepts); status_word |= FEnv::exception_value_to_status(new_exceptions); @@ -153,7 +154,7 @@ static inline int set_except(int excepts) { return 0; } -static inline int raise_except(int excepts) { +LIBC_INLINE int raise_except(int excepts) { float zero = 0.0f; float one = 1.0f; float large_value = float(FPBits(FPBits::MAX_NORMAL)); @@ -212,7 +213,7 @@ static inline int raise_except(int excepts) { return result; } -static inline int get_round() { +LIBC_INLINE int get_round() { uint32_t rounding_mode = (FEnv::get_control_word() >> FEnv::ROUNDING_CONTROL_BIT_POSITION) & 0x3; switch (rounding_mode) { @@ -229,7 +230,7 @@ static inline int get_round() { } } -static inline int set_round(int mode) { +LIBC_INLINE int set_round(int mode) { uint16_t bit_value; switch (mode) { case FE_TONEAREST: @@ -256,14 +257,14 @@ static inline int set_round(int mode) { return 0; } -static inline int get_env(fenv_t *envp) { +LIBC_INLINE int get_env(fenv_t *envp) { FEnv::FPState *state = reinterpret_cast(envp); state->ControlWord = FEnv::get_control_word(); state->StatusWord = FEnv::get_status_word(); return 0; } -static inline int set_env(const fenv_t *envp) { +LIBC_INLINE int set_env(const fenv_t *envp) { if (envp == FE_DFL_ENV) { // Default status and control words bits are all zeros so we just // write zeros. diff --git a/libc/src/__support/FPUtil/aarch64/nearest_integer.h b/libc/src/__support/FPUtil/aarch64/nearest_integer.h index e6c83e956ccba..1d94e66d0e1dc 100644 --- a/libc/src/__support/FPUtil/aarch64/nearest_integer.h +++ b/libc/src/__support/FPUtil/aarch64/nearest_integer.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_AARCH64_NEAREST_INTEGER_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if !defined(LLVM_LIBC_ARCH_AARCH64) #error "Invalid include" @@ -18,13 +19,13 @@ namespace __llvm_libc { namespace fputil { -static inline float nearest_integer(float x) { +LIBC_INLINE float nearest_integer(float x) { float result; __asm__ __volatile__("frintn %s0, %s1\n\t" : "=w"(result) : "w"(x)); return result; } -static inline double nearest_integer(double x) { +LIBC_INLINE double nearest_integer(double x) { double result; __asm__ __volatile__("frintn %d0, %d1\n\t" : "=w"(result) : "w"(x)); return result; diff --git a/libc/src/__support/FPUtil/generic/FMA.h b/libc/src/__support/FPUtil/generic/FMA.h index da730992fce8f..69868d4e5ba23 100644 --- a/libc/src/__support/FPUtil/generic/FMA.h +++ b/libc/src/__support/FPUtil/generic/FMA.h @@ -21,7 +21,7 @@ namespace __llvm_libc { namespace fputil { namespace generic { -template static inline T fma(T x, T y, T z); +template LIBC_INLINE T fma(T x, T y, T z); // TODO(lntue): Implement fmaf that is correctly rounded to all rounding modes. // The implementation below only is only correct for the default rounding mode, @@ -78,7 +78,7 @@ namespace internal { // Extract the sticky bits and shift the `mantissa` to the right by // `shift_length`. -static inline bool shift_mantissa(int shift_length, UInt128 &mant) { +LIBC_INLINE bool shift_mantissa(int shift_length, UInt128 &mant) { if (shift_length >= 128) { mant = 0; return true; // prod_mant is non-zero. diff --git a/libc/src/__support/FPUtil/generic/FMod.h b/libc/src/__support/FPUtil/generic/FMod.h index c20bd8950b9d5..e0694ba11b7e4 100644 --- a/libc/src/__support/FPUtil/generic/FMod.h +++ b/libc/src/__support/FPUtil/generic/FMod.h @@ -122,7 +122,7 @@ template struct FModExceptionalInputHandler { static_assert(cpp::is_floating_point_v, "FModCStandardWrapper instantiated with invalid type."); - static bool PreCheck(T x, T y, T &out) { + LIBC_INLINE static bool PreCheck(T x, T y, T &out) { using FPB = fputil::FPBits; const T quiet_NaN = FPB::build_quiet_nan(0); FPB sx(x), sy(y); @@ -168,8 +168,8 @@ template class FModDivisionSimpleHelper { using intU_t = typename FPBits::UIntType; public: - inline constexpr static intU_t execute(int exp_diff, int sides_zeroes_count, - intU_t m_x, intU_t m_y) { + LIBC_INLINE constexpr static intU_t + execute(int exp_diff, int sides_zeroes_count, intU_t m_x, intU_t m_y) { while (exp_diff > sides_zeroes_count) { exp_diff -= sides_zeroes_count; m_x <<= sides_zeroes_count; @@ -187,8 +187,8 @@ template class FModDivisionInvMultHelper { using intU_t = typename FPB::UIntType; public: - inline constexpr static intU_t execute(int exp_diff, int sides_zeroes_count, - intU_t m_x, intU_t m_y) { + LIBC_INLINE constexpr static intU_t + execute(int exp_diff, int sides_zeroes_count, intU_t m_x, intU_t m_y) { if (exp_diff > sides_zeroes_count) { intU_t inv_hy = (cpp::numeric_limits::max() / m_y); while (exp_diff > sides_zeroes_count) { @@ -223,7 +223,7 @@ class FMod { using FPB = FPBits; using intU_t = typename FPB::UIntType; - inline static constexpr FPB eval_internal(FPB sx, FPB sy) { + LIBC_INLINE static constexpr FPB eval_internal(FPB sx, FPB sy) { if (likely(sx.uintval() <= sy.uintval())) { if (sx.uintval() < sy.uintval()) @@ -300,7 +300,7 @@ class FMod { } public: - static inline T eval(T x, T y) { + LIBC_INLINE static T eval(T x, T y) { if (T out; Wrapper::PreCheck(x, y, out)) return out; FPB sx(x), sy(y); diff --git a/libc/src/__support/FPUtil/generic/sqrt.h b/libc/src/__support/FPUtil/generic/sqrt.h index 125d864672bbd..c972c5f40cd59 100644 --- a/libc/src/__support/FPUtil/generic/sqrt.h +++ b/libc/src/__support/FPUtil/generic/sqrt.h @@ -34,8 +34,8 @@ template <> struct SpecialLongDouble { #endif // SPECIAL_X86_LONG_DOUBLE template -static inline void normalize(int &exponent, - typename FPBits::UIntType &mantissa) { +LIBC_INLINE void normalize(int &exponent, + typename FPBits::UIntType &mantissa) { const int shift = unsafe_clz(mantissa) - (8 * sizeof(mantissa) - 1 - MantissaWidth::VALUE); exponent -= shift; @@ -64,7 +64,7 @@ inline void normalize(int &exponent, UInt128 &mantissa) { // Correctly rounded IEEE 754 SQRT for all rounding modes. // Shift-and-add algorithm. template -static inline cpp::enable_if_t, T> sqrt(T x) { +LIBC_INLINE cpp::enable_if_t, T> sqrt(T x) { if constexpr (internal::SpecialLongDouble::VALUE) { // Special 80-bit long double. diff --git a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h index cf3b2e4aa8d4f..64fb5e0d8a66c 100644 --- a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h +++ b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h @@ -29,12 +29,12 @@ inline void normalize(int &exponent, UInt128 &mantissa) { // if constexpr statement in sqrt.h still requires x86::sqrt to be declared // even when it's not used. -static inline long double sqrt(long double x); +LIBC_INLINE long double sqrt(long double x); // Correctly rounded SQRT for all rounding modes. // Shift-and-add algorithm. #if defined(SPECIAL_X86_LONG_DOUBLE) -static inline long double sqrt(long double x) { +LIBC_INLINE long double sqrt(long double x) { using UIntType = typename FPBits::UIntType; constexpr UIntType ONE = UIntType(1) << int(MantissaWidth::VALUE); diff --git a/libc/src/__support/FPUtil/multiply_add.h b/libc/src/__support/FPUtil/multiply_add.h index b7889c63871c0..384035e76840c 100644 --- a/libc/src/__support/FPUtil/multiply_add.h +++ b/libc/src/__support/FPUtil/multiply_add.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_MULTIPLY_ADD_H #include "src/__support/architectures.h" +#include "src/__support/common.h" namespace __llvm_libc { namespace fputil { @@ -18,7 +19,7 @@ namespace fputil { // multiply_add(x, y, z) = x*y + z // which uses FMA instructions to speed up if available. -template static inline T multiply_add(T x, T y, T z) { +template LIBC_INLINE T multiply_add(T x, T y, T z) { return x * y + z; } diff --git a/libc/src/__support/FPUtil/nearest_integer.h b/libc/src/__support/FPUtil/nearest_integer.h index e6377fe9dc602..bf2bf4b797c69 100644 --- a/libc/src/__support/FPUtil/nearest_integer.h +++ b/libc/src/__support/FPUtil/nearest_integer.h @@ -26,7 +26,7 @@ namespace fputil { // Notice that for AARCH64 and x86-64 with SSE4.2 support, we will use their // corresponding rounding instruction instead. And in those cases, the results // are rounded to the nearest integer, tie-to-even. -static inline float nearest_integer(float x) { +LIBC_INLINE float nearest_integer(float x) { if (x < 0x1p24f && x > -0x1p24f) { float r = x < 0 ? (x - 0x1.0p23f) + 0x1.0p23f : (x + 0x1.0p23f) - 0x1.0p23f; float diff = x - r; @@ -42,7 +42,7 @@ static inline float nearest_integer(float x) { return x; } -static inline double nearest_integer(double x) { +LIBC_INLINE double nearest_integer(double x) { if (x < 0x1p53 && x > -0x1p53) { double r = x < 0 ? (x - 0x1.0p52) + 0x1.0p52 : (x + 0x1.0p52) - 0x1.0p52; double diff = x - r; diff --git a/libc/src/__support/FPUtil/x86_64/FEnvImpl.h b/libc/src/__support/FPUtil/x86_64/FEnvImpl.h index 0f7e2b57a778e..5c30ee35efae3 100644 --- a/libc/src/__support/FPUtil/x86_64/FEnvImpl.h +++ b/libc/src/__support/FPUtil/x86_64/FEnvImpl.h @@ -68,7 +68,7 @@ static constexpr uint16_t MXCSR_EXCEPTION_CONTOL_BIT_POISTION = 7; // Exception flags are individual bits in the corresponding registers. // So, we just OR the bit values to get the full set of exceptions. -static inline uint16_t get_status_value_for_except(int excepts) { +LIBC_INLINE uint16_t get_status_value_for_except(int excepts) { // We will make use of the fact that exception control bits are single // bit flags in the control registers. return (excepts & FE_INVALID ? ExceptionFlags::INVALID_F : 0) | @@ -81,7 +81,7 @@ static inline uint16_t get_status_value_for_except(int excepts) { (excepts & FE_INEXACT ? ExceptionFlags::INEXACT_F : 0); } -static inline int exception_status_to_macro(uint16_t status) { +LIBC_INLINE int exception_status_to_macro(uint16_t status) { return (status & ExceptionFlags::INVALID_F ? FE_INVALID : 0) | #ifdef __FE_DENORM (status & ExceptionFlags::DENORMAL_F ? __FE_DENORM : 0) | @@ -101,53 +101,53 @@ struct X87StateDescriptor { uint32_t _[5]; }; -static inline uint16_t get_x87_control_word() { +LIBC_INLINE uint16_t get_x87_control_word() { uint16_t w; __asm__ __volatile__("fnstcw %0" : "=m"(w)::); SANITIZER_MEMORY_INITIALIZED(&w, sizeof(w)); return w; } -static inline void write_x87_control_word(uint16_t w) { +LIBC_INLINE void write_x87_control_word(uint16_t w) { __asm__ __volatile__("fldcw %0" : : "m"(w) :); } -static inline uint16_t get_x87_status_word() { +LIBC_INLINE uint16_t get_x87_status_word() { uint16_t w; __asm__ __volatile__("fnstsw %0" : "=m"(w)::); SANITIZER_MEMORY_INITIALIZED(&w, sizeof(w)); return w; } -static inline void clear_x87_exceptions() { +LIBC_INLINE void clear_x87_exceptions() { __asm__ __volatile__("fnclex" : : :); } -static inline uint32_t get_mxcsr() { +LIBC_INLINE uint32_t get_mxcsr() { uint32_t w; __asm__ __volatile__("stmxcsr %0" : "=m"(w)::); SANITIZER_MEMORY_INITIALIZED(&w, sizeof(w)); return w; } -static inline void write_mxcsr(uint32_t w) { +LIBC_INLINE void write_mxcsr(uint32_t w) { __asm__ __volatile__("ldmxcsr %0" : : "m"(w) :); } -static inline void get_x87_state_descriptor(X87StateDescriptor &s) { +LIBC_INLINE void get_x87_state_descriptor(X87StateDescriptor &s) { __asm__ __volatile__("fnstenv %0" : "=m"(s)); SANITIZER_MEMORY_INITIALIZED(&s, sizeof(s)); } -static inline void write_x87_state_descriptor(const X87StateDescriptor &s) { +LIBC_INLINE void write_x87_state_descriptor(const X87StateDescriptor &s) { __asm__ __volatile__("fldenv %0" : : "m"(s) :); } -static inline void fwait() { __asm__ __volatile__("fwait"); } +LIBC_INLINE void fwait() { __asm__ __volatile__("fwait"); } } // namespace internal -static inline int enable_except(int excepts) { +LIBC_INLINE int enable_except(int excepts) { // In the x87 control word and in MXCSR, an exception is blocked // if the corresponding bit is set. That is the reason for all the // bit-flip operations below as we need to turn the bits to zero @@ -174,7 +174,7 @@ static inline int enable_except(int excepts) { return internal::exception_status_to_macro(old_excepts); } -static inline int disable_except(int excepts) { +LIBC_INLINE int disable_except(int excepts) { // In the x87 control word and in MXCSR, an exception is blocked // if the corresponding bit is set. @@ -194,13 +194,13 @@ static inline int disable_except(int excepts) { return internal::exception_status_to_macro(old_excepts); } -static inline int get_except() { +LIBC_INLINE int get_except() { uint16_t mxcsr = static_cast(internal::get_mxcsr()); uint16_t enabled_excepts = ~(mxcsr >> 7) & 0x3F; return internal::exception_status_to_macro(enabled_excepts); } -static inline int clear_except(int excepts) { +LIBC_INLINE int clear_except(int excepts) { internal::X87StateDescriptor state; internal::get_x87_state_descriptor(state); state.status_word &= @@ -213,7 +213,7 @@ static inline int clear_except(int excepts) { return 0; } -static inline int test_except(int excepts) { +LIBC_INLINE int test_except(int excepts) { uint16_t status_value = internal::get_status_value_for_except(excepts); // Check both x87 status word and MXCSR. return internal::exception_status_to_macro( @@ -221,7 +221,7 @@ static inline int test_except(int excepts) { } // Sets the exception flags but does not trigger the exception handler. -static inline int set_except(int excepts) { +LIBC_INLINE int set_except(int excepts) { uint16_t status_value = internal::get_status_value_for_except(excepts); internal::X87StateDescriptor state; internal::get_x87_state_descriptor(state); @@ -235,7 +235,7 @@ static inline int set_except(int excepts) { return 0; } -static inline int raise_except(int excepts) { +LIBC_INLINE int raise_except(int excepts) { uint16_t status_value = internal::get_status_value_for_except(excepts); // We set the status flag for exception one at a time and call the @@ -287,7 +287,7 @@ static inline int raise_except(int excepts) { return 0; } -static inline int get_round() { +LIBC_INLINE int get_round() { uint16_t bit_value = (internal::get_mxcsr() >> internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION) & 0x3; @@ -305,7 +305,7 @@ static inline int get_round() { } } -static inline int set_round(int mode) { +LIBC_INLINE int set_round(int mode) { uint16_t bit_value; switch (mode) { case FE_TONEAREST: @@ -461,7 +461,7 @@ struct WinExceptionFlags { same order in both. */ -static inline int get_env(fenv_t *envp) { +LIBC_INLINE int get_env(fenv_t *envp) { internal::FPState *state = reinterpret_cast(envp); uint32_t status_word = 0; @@ -505,7 +505,7 @@ static inline int get_env(fenv_t *envp) { return 0; } -static inline int set_env(const fenv_t *envp) { +LIBC_INLINE int set_env(const fenv_t *envp) { const internal::FPState *state = reinterpret_cast(envp); @@ -554,14 +554,14 @@ static inline int set_env(const fenv_t *envp) { return 0; } #else -static inline int get_env(fenv_t *envp) { +LIBC_INLINE int get_env(fenv_t *envp) { internal::FPState *state = reinterpret_cast(envp); internal::get_x87_state_descriptor(state->x87_status); state->mxcsr = internal::get_mxcsr(); return 0; } -static inline int set_env(const fenv_t *envp) { +LIBC_INLINE int set_env(const fenv_t *envp) { // envp contains everything including pieces like the current // top of FPU stack. We cannot arbitrarily change them. So, we first // read the current status and update only those pieces which are diff --git a/libc/src/__support/FPUtil/x86_64/FMA.h b/libc/src/__support/FPUtil/x86_64/FMA.h index d335585ef3015..40da101952c4a 100644 --- a/libc/src/__support/FPUtil/x86_64/FMA.h +++ b/libc/src/__support/FPUtil/x86_64/FMA.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_FMA_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if !defined(LLVM_LIBC_ARCH_X86_64) #error "Invalid include" @@ -26,7 +27,7 @@ namespace __llvm_libc { namespace fputil { template -static inline cpp::enable_if_t, T> fma(T x, T y, T z) { +LIBC_INLINE cpp::enable_if_t, T> fma(T x, T y, T z) { float result; __m128 xmm = _mm_load_ss(&x); // NOLINT __m128 ymm = _mm_load_ss(&y); // NOLINT @@ -37,8 +38,7 @@ static inline cpp::enable_if_t, T> fma(T x, T y, T z) { } template -static inline cpp::enable_if_t, T> fma(T x, T y, - T z) { +LIBC_INLINE cpp::enable_if_t, T> fma(T x, T y, T z) { double result; __m128d xmm = _mm_load_sd(&x); // NOLINT __m128d ymm = _mm_load_sd(&y); // NOLINT diff --git a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h index 5ac45f1d8e6cd..8d3dd30c47dbc 100644 --- a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h +++ b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h @@ -23,7 +23,7 @@ namespace __llvm_libc { namespace fputil { -static inline long double nextafter(long double from, long double to) { +LIBC_INLINE long double nextafter(long double from, long double to) { using FPBits = FPBits; FPBits from_bits(from); if (from_bits.is_nan()) diff --git a/libc/src/__support/FPUtil/x86_64/nearest_integer.h b/libc/src/__support/FPUtil/x86_64/nearest_integer.h index e0c1b1a2d9e2d..f9a2a6a07b556 100644 --- a/libc/src/__support/FPUtil/x86_64/nearest_integer.h +++ b/libc/src/__support/FPUtil/x86_64/nearest_integer.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_SUPPORT_FPUTIL_X86_64_NEAREST_INTEGER_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #if !defined(LLVM_LIBC_ARCH_X86_64) #error "Invalid include" @@ -24,14 +25,14 @@ namespace __llvm_libc { namespace fputil { -static inline float nearest_integer(float x) { +LIBC_INLINE float nearest_integer(float x) { __m128 xmm = _mm_set_ss(x); // NOLINT __m128 ymm = _mm_round_ss(xmm, xmm, _MM_ROUND_NEAREST | _MM_FROUND_NO_EXC); // NOLINT return ymm[0]; } -static inline double nearest_integer(double x) { +LIBC_INLINE double nearest_integer(double x) { __m128d xmm = _mm_set_sd(x); // NOLINT __m128d ymm = _mm_round_sd(xmm, xmm, _MM_ROUND_NEAREST | _MM_FROUND_NO_EXC); // NOLINT diff --git a/libc/src/__support/OSUtil/linux/io.h b/libc/src/__support/OSUtil/linux/io.h index b99f828446fc5..30180b946e7c3 100644 --- a/libc/src/__support/OSUtil/linux/io.h +++ b/libc/src/__support/OSUtil/linux/io.h @@ -16,7 +16,7 @@ namespace __llvm_libc { -static inline void write_to_stderr(const char *msg) { +LIBC_INLINE void write_to_stderr(const char *msg) { __llvm_libc::syscall_impl(SYS_write, 2 /* stderr */, msg, internal::string_length(msg)); } diff --git a/libc/src/__support/OSUtil/linux/quick_exit.h b/libc/src/__support/OSUtil/linux/quick_exit.h index ca755de19522f..e4df9a6d80976 100644 --- a/libc/src/__support/OSUtil/linux/quick_exit.h +++ b/libc/src/__support/OSUtil/linux/quick_exit.h @@ -11,11 +11,13 @@ #include "syscall.h" // For internal syscall function. +#include "src/__support/common.h" + #include // For syscall numbers. namespace __llvm_libc { -static inline void quick_exit(int status) { +LIBC_INLINE void quick_exit(int status) { for (;;) { __llvm_libc::syscall_impl(SYS_exit_group, status); __llvm_libc::syscall_impl(SYS_exit, status); diff --git a/libc/src/__support/builtin_wrappers.h b/libc/src/__support/builtin_wrappers.h index 426d7b43cd1fb..6e3f4274f7533 100644 --- a/libc/src/__support/builtin_wrappers.h +++ b/libc/src/__support/builtin_wrappers.h @@ -12,6 +12,7 @@ #include "named_pair.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" #include "src/__support/compiler_features.h" namespace __llvm_libc { @@ -22,14 +23,14 @@ namespace __llvm_libc { // compiler match for us. namespace __internal { -template static inline int correct_zero(T val, int bits) { +template LIBC_INLINE int correct_zero(T val, int bits) { if (val == T(0)) return sizeof(T(0)) * 8; else return bits; } -template static inline int clz(T val); +template LIBC_INLINE int clz(T val); template <> inline int clz(unsigned int val) { return __builtin_clz(val); } @@ -40,7 +41,7 @@ template <> inline int clz(unsigned long long int val) { return __builtin_clzll(val); } -template static inline int ctz(T val); +template LIBC_INLINE int ctz(T val); template <> inline int ctz(unsigned int val) { return __builtin_ctz(val); } @@ -52,19 +53,19 @@ template <> inline int ctz(unsigned long long int val) { } } // namespace __internal -template static inline int safe_ctz(T val) { +template LIBC_INLINE int safe_ctz(T val) { return __internal::correct_zero(val, __internal::ctz(val)); } -template static inline int unsafe_ctz(T val) { +template LIBC_INLINE int unsafe_ctz(T val) { return __internal::ctz(val); } -template static inline int safe_clz(T val) { +template LIBC_INLINE int safe_clz(T val) { return __internal::correct_zero(val, __internal::clz(val)); } -template static inline int unsafe_clz(T val) { +template LIBC_INLINE int unsafe_clz(T val) { return __internal::clz(val); } diff --git a/libc/src/__support/common.h b/libc/src/__support/common.h index 6ab9940b71ac1..25bc77978cba2 100644 --- a/libc/src/__support/common.h +++ b/libc/src/__support/common.h @@ -25,6 +25,10 @@ #define LLVM_LIBC_FUNCTION_ATTR #endif +#ifndef LIBC_INLINE +#define LIBC_INLINE inline +#endif + // We use OpenMP to declare these functions on the device. #define STR(X) #X #define LLVM_LIBC_DECLARE_DEVICE(name) \ diff --git a/libc/src/__support/detailed_powers_of_ten.h b/libc/src/__support/detailed_powers_of_ten.h index 80dfcb2d6a59e..767b7cf6a65ee 100644 --- a/libc/src/__support/detailed_powers_of_ten.h +++ b/libc/src/__support/detailed_powers_of_ten.h @@ -9,6 +9,8 @@ #ifndef LIBC_SRC_SUPPORT_DETAILED_POWERS_OF_TEN_H #define LIBC_SRC_SUPPORT_DETAILED_POWERS_OF_TEN_H +#include "src/__support/common.h" + #include namespace __llvm_libc { @@ -27,7 +29,7 @@ constexpr int32_t DETAILED_POWERS_OF_TEN_MIN_EXP_10 = -348; constexpr int32_t DETAILED_POWERS_OF_TEN_MAX_EXP_10 = 347; // This rescales the base 10 exponent by a factor of log(10)/log(2). -static inline int64_t exp10_to_exp2(int64_t exp10) { +LIBC_INLINE int64_t exp10_to_exp2(int64_t exp10) { return (217706 * exp10) >> 16; } diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h index b7b1d38070ab2..5fe003f853f35 100644 --- a/libc/src/__support/float_to_string.h +++ b/libc/src/__support/float_to_string.h @@ -14,6 +14,7 @@ #include "src/__support/CPP/type_traits.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/UInt.h" +#include "src/__support/common.h" #include "src/__support/ryu_constants.h" // This implementation is based on the Ryu Printf algorithm by Ulf Adams: @@ -177,7 +178,7 @@ inline cpp::UInt get_table_negative(int exponent, size_t i, return num; } -static inline uint32_t fast_uint_mod_1e9(const cpp::UInt &val) { +LIBC_INLINE uint32_t fast_uint_mod_1e9(const cpp::UInt &val) { // The formula for mult_const is: // 1 + floor((2^(bits in target integer size + log_2(divider))) / divider) // Where divider is 10^9 and target integer size is 128. @@ -189,9 +190,9 @@ static inline uint32_t fast_uint_mod_1e9(const cpp::UInt &val) { return static_cast(val) - (1000000000 * shifted); } -static inline uint32_t mul_shift_mod_1e9(const MantissaInt mantissa, - const cpp::UInt &large, - const int32_t shift_amount) { +LIBC_INLINE uint32_t mul_shift_mod_1e9(const MantissaInt mantissa, + const cpp::UInt &large, + const int32_t shift_amount) { constexpr size_t MANT_INT_SIZE = sizeof(MantissaInt) * 8; cpp::UInt val(large); // TODO: Find a better way to force __uint128_t to be UInt<128> diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index 1579ac876ef47..55f41c1db29ce 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -13,6 +13,7 @@ #include "src/__support/FPUtil/FPBits.h" #include "src/__support/UInt128.h" #include "src/__support/builtin_wrappers.h" +#include "src/__support/common.h" #include "src/__support/ctype_utils.h" #include "src/__support/detailed_powers_of_ten.h" #include "src/__support/high_precision_decimal.h" @@ -22,7 +23,7 @@ namespace __llvm_libc { namespace internal { -template uint32_t inline leading_zeroes(T inputNumber) { +template LIBC_INLINE uint32_t leading_zeroes(T inputNumber) { constexpr uint32_t BITS_IN_T = sizeof(T) * 8; if (inputNumber == 0) { return BITS_IN_T; @@ -51,23 +52,27 @@ template uint32_t inline leading_zeroes(T inputNumber) { return BITS_IN_T - cur_guess; } -template <> uint32_t inline leading_zeroes(uint32_t inputNumber) { +template <> +LIBC_INLINE uint32_t leading_zeroes(uint32_t inputNumber) { return safe_clz(inputNumber); } -template <> uint32_t inline leading_zeroes(uint64_t inputNumber) { +template <> +LIBC_INLINE uint32_t leading_zeroes(uint64_t inputNumber) { return safe_clz(inputNumber); } -static inline uint64_t low64(const UInt128 &num) { +LIBC_INLINE uint64_t low64(const UInt128 &num) { return static_cast(num & 0xffffffffffffffff); } -static inline uint64_t high64(const UInt128 &num) { +LIBC_INLINE uint64_t high64(const UInt128 &num) { return static_cast(num >> 64); } -template inline void set_implicit_bit(fputil::FPBits &) { return; } +template LIBC_INLINE void set_implicit_bit(fputil::FPBits &) { + return; +} #if defined(SPECIAL_X86_LONG_DOUBLE) template <> @@ -85,7 +90,7 @@ inline void set_implicit_bit(fputil::FPBits &result) { // (https://github.com/golang/go/blob/release-branch.go1.16/src/strconv/eisel_lemire.go#L25) // for some optimizations as well as handling 32 bit floats. template -static inline bool +LIBC_INLINE bool eisel_lemire(typename fputil::FPBits::UIntType mantissa, int32_t exp10, typename fputil::FPBits::UIntType *outputMantissa, uint32_t *outputExp2) { @@ -306,7 +311,7 @@ constexpr int32_t NUM_POWERS_OF_TWO = // on the Simple Decimal Conversion algorithm by Nigel Tao, described at this // link: https://nigeltao.github.io/blog/2020/parse-number-f64-simple.html template -static inline void +LIBC_INLINE void simple_decimal_conversion(const char *__restrict numStart, typename fputil::FPBits::UIntType *outputMantissa, uint32_t *outputExp2) { @@ -504,7 +509,7 @@ template <> class ClingerConsts { // exponents, but handles them quickly. This is an implementation of Clinger's // Fast Path, as described above. template -static inline bool +LIBC_INLINE bool clinger_fast_path(typename fputil::FPBits::UIntType mantissa, int32_t exp10, typename fputil::FPBits::UIntType *outputMantissa, uint32_t *outputExp2) { @@ -587,7 +592,7 @@ template <> constexpr int32_t get_lower_bound() { // accuracy. The resulting mantissa and exponent are placed in outputMantissa // and outputExp2. template -static inline void +LIBC_INLINE void decimal_exp_to_float(typename fputil::FPBits::UIntType mantissa, int32_t exp10, const char *__restrict numStart, bool truncated, @@ -649,7 +654,7 @@ decimal_exp_to_float(typename fputil::FPBits::UIntType mantissa, // form, this is mostly just shifting and rounding. This is used for hexadecimal // numbers since a base 16 exponent multiplied by 4 is the base 2 exponent. template -static inline void +LIBC_INLINE void binary_exp_to_float(typename fputil::FPBits::UIntType mantissa, int32_t exp2, bool truncated, typename fputil::FPBits::UIntType *outputMantissa, @@ -736,8 +741,8 @@ binary_exp_to_float(typename fputil::FPBits::UIntType mantissa, int32_t exp2, // checks if the next 4 characters of the string pointer are the start of a // hexadecimal floating point number. Does not advance the string pointer. -static inline bool is_float_hex_start(const char *__restrict src, - const char decimalPoint) { +LIBC_INLINE bool is_float_hex_start(const char *__restrict src, + const char decimalPoint) { if (!(*src == '0' && (*(src + 1) | 32) == 'x')) { return false; } @@ -755,7 +760,7 @@ static inline bool is_float_hex_start(const char *__restrict src, // If the return value is false, then it is assumed that there is no number // here. template -static inline bool +LIBC_INLINE bool decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, char **__restrict strEnd, typename fputil::FPBits::UIntType *outputMantissa, @@ -849,7 +854,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, // If the return value is false, then it is assumed that there is no number // here. template -static inline bool hexadecimal_string_to_float( +LIBC_INLINE bool hexadecimal_string_to_float( const char *__restrict src, const char DECIMAL_POINT, char **__restrict strEnd, typename fputil::FPBits::UIntType *outputMantissa, @@ -940,8 +945,8 @@ static inline bool hexadecimal_string_to_float( // Takes a pointer to a string and a pointer to a string pointer. This function // is used as the backend for all of the string to float functions. template -static inline T strtofloatingpoint(const char *__restrict src, - char **__restrict strEnd) { +LIBC_INLINE T strtofloatingpoint(const char *__restrict src, + char **__restrict strEnd) { using BitsType = typename fputil::FPBits::UIntType; fputil::FPBits result = fputil::FPBits(); const char *original_src = src; diff --git a/libc/src/__support/str_to_integer.h b/libc/src/__support/str_to_integer.h index 269d432e31ecc..8e1ce1132904b 100644 --- a/libc/src/__support/str_to_integer.h +++ b/libc/src/__support/str_to_integer.h @@ -10,6 +10,7 @@ #define LIBC_SRC_SUPPORT_STR_TO_INTEGER_H #include "src/__support/CPP/limits.h" +#include "src/__support/common.h" #include "src/__support/ctype_utils.h" #include "src/__support/str_to_num_result.h" #include @@ -20,14 +21,14 @@ namespace internal { // Returns a pointer to the first character in src that is not a whitespace // character (as determined by isspace()) -static inline const char *first_non_whitespace(const char *__restrict src) { +LIBC_INLINE const char *first_non_whitespace(const char *__restrict src) { while (internal::isspace(*src)) { ++src; } return src; } -static inline int b36_char_to_int(char input) { +LIBC_INLINE int b36_char_to_int(char input) { if (isdigit(input)) return input - '0'; if (isalpha(input)) @@ -37,7 +38,7 @@ static inline int b36_char_to_int(char input) { // checks if the next 3 characters of the string pointer are the start of a // hexadecimal number. Does not advance the string pointer. -static inline bool is_hex_start(const char *__restrict src) { +LIBC_INLINE bool is_hex_start(const char *__restrict src) { return *src == '0' && (*(src + 1) | 32) == 'x' && isalnum(*(src + 2)) && b36_char_to_int(*(src + 2)) < 16; } @@ -45,7 +46,7 @@ static inline bool is_hex_start(const char *__restrict src) { // Takes the address of the string pointer and parses the base from the start of // it. This function will advance |src| to the first valid digit in the inferred // base. -static inline int infer_base(const char *__restrict *__restrict src) { +LIBC_INLINE int infer_base(const char *__restrict *__restrict src) { // A hexadecimal number is defined as "the prefix 0x or 0X followed by a // sequence of the deimal digits and the letters a (or A) through f (or F) // with values 10 through 15 respectively." (C standard 6.4.4.1) @@ -67,8 +68,8 @@ static inline int infer_base(const char *__restrict *__restrict src) { // Takes a pointer to a string and the base to convert to. This function is used // as the backend for all of the string to int functions. template -static inline StrToNumResult strtointeger(const char *__restrict src, - int base) { +LIBC_INLINE StrToNumResult strtointeger(const char *__restrict src, + int base) { unsigned long long result = 0; bool is_number = false; const char *original_src = src; diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index 28593b0318dcb..21b5dcef46a4a 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -29,9 +29,10 @@ add_object_library( libc.config.linux.app_h libc.include.sys_syscall libc.src.__support.CPP.atomic - libc.src.__support.error_or libc.src.__support.CPP.stringstream libc.src.__support.CPP.string_view + libc.src.__support.common + libc.src.__support.error_or libc.src.__support.threads.thread_common COMPILE_OPTIONS -O3 diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp index e0fc69034a5f6..62e24e58b9ec3 100644 --- a/libc/src/__support/threads/linux/thread.cpp +++ b/libc/src/__support/threads/linux/thread.cpp @@ -12,6 +12,7 @@ #include "src/__support/CPP/string_view.h" #include "src/__support/CPP/stringstream.h" #include "src/__support/OSUtil/syscall.h" // For syscall functions. +#include "src/__support/common.h" #include "src/__support/error_or.h" #include "src/__support/threads/linux/futex_word.h" // For FutexWordType @@ -54,7 +55,7 @@ static constexpr unsigned CLONE_SYSCALL_FLAGS = // wake the joining thread. | CLONE_SETTLS; // Setup the thread pointer of the new thread. -static inline ErrorOr alloc_stack(size_t size) { +LIBC_INLINE ErrorOr alloc_stack(size_t size) { long mmap_result = __llvm_libc::syscall_impl(MMAP_SYSCALL_NUMBER, 0, // No special address @@ -69,7 +70,7 @@ static inline ErrorOr alloc_stack(size_t size) { return reinterpret_cast(mmap_result); } -static inline void free_stack(void *stack, size_t size) { +LIBC_INLINE void free_stack(void *stack, size_t size) { __llvm_libc::syscall_impl(SYS_munmap, stack, size); } diff --git a/libc/src/string/memory_utils/bcmp_implementations.h b/libc/src/string/memory_utils/bcmp_implementations.h index 18a4ab829851e..4e31cc92bd838 100644 --- a/libc/src/string/memory_utils/bcmp_implementations.h +++ b/libc/src/string/memory_utils/bcmp_implementations.h @@ -20,7 +20,7 @@ namespace __llvm_libc { -[[maybe_unused]] static inline BcmpReturnType +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) { LLVM_LIBC_LOOP_NOUNROLL for (size_t offset = 0; offset < count; ++offset) @@ -30,7 +30,7 @@ inline_bcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) { } #if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) -[[maybe_unused]] static inline BcmpReturnType +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) { if (count < 256) return generic::Bcmp<16>::loop_and_tail(p1, p2, count); @@ -42,7 +42,7 @@ inline_bcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) { #endif // defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) #if defined(LLVM_LIBC_ARCH_X86) -[[maybe_unused]] static inline BcmpReturnType +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) { if (count <= 32) return x86::sse2::Bcmp<16>::head_tail(p1, p2, count); @@ -54,7 +54,7 @@ inline_bcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) { return x86::sse2::Bcmp<64>::loop_and_tail(p1, p2, count); } -[[maybe_unused]] static inline BcmpReturnType +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) { if (count <= 32) return x86::sse2::Bcmp<16>::head_tail(p1, p2, count); @@ -70,7 +70,7 @@ inline_bcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) { return x86::avx2::Bcmp<64>::loop_and_tail(p1, p2, count); } -[[maybe_unused]] static inline BcmpReturnType +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) { if (count <= 32) return x86::sse2::Bcmp<16>::head_tail(p1, p2, count); @@ -86,8 +86,8 @@ inline_bcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) { return x86::avx512bw::Bcmp<64>::loop_and_tail(p1, p2, count); } -[[maybe_unused]] static inline BcmpReturnType inline_bcmp_x86(CPtr p1, CPtr p2, - size_t count) { +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_x86(CPtr p1, CPtr p2, + size_t count) { if (count == 0) return BcmpReturnType::ZERO(); if (count == 1) @@ -112,8 +112,9 @@ inline_bcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) { #endif // defined(LLVM_LIBC_ARCH_X86) #if defined(LLVM_LIBC_ARCH_AARCH64) -[[maybe_unused]] static inline BcmpReturnType -inline_bcmp_aarch64(CPtr p1, CPtr p2, size_t count) { +[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_aarch64(CPtr p1, + CPtr p2, + size_t count) { if (likely(count <= 32)) { if (unlikely(count >= 16)) { return aarch64::Bcmp<16>::head_tail(p1, p2, count); @@ -159,7 +160,7 @@ inline_bcmp_aarch64(CPtr p1, CPtr p2, size_t count) { } #endif // defined(LLVM_LIBC_ARCH_AARCH64) -static inline BcmpReturnType inline_bcmp(CPtr p1, CPtr p2, size_t count) { +LIBC_INLINE BcmpReturnType inline_bcmp(CPtr p1, CPtr p2, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) return inline_bcmp_x86(p1, p2, count); #elif defined(LLVM_LIBC_ARCH_AARCH64) @@ -173,7 +174,7 @@ static inline BcmpReturnType inline_bcmp(CPtr p1, CPtr p2, size_t count) { #endif } -static inline int inline_bcmp(const void *p1, const void *p2, size_t count) { +LIBC_INLINE int inline_bcmp(const void *p1, const void *p2, size_t count) { return static_cast(inline_bcmp(reinterpret_cast(p1), reinterpret_cast(p2), count)); } diff --git a/libc/src/string/memory_utils/memcmp_implementations.h b/libc/src/string/memory_utils/memcmp_implementations.h index 1dac6e011c8c1..1f0f5b91bc05b 100644 --- a/libc/src/string/memory_utils/memcmp_implementations.h +++ b/libc/src/string/memory_utils/memcmp_implementations.h @@ -20,7 +20,7 @@ #include // size_t namespace __llvm_libc { -[[maybe_unused]] static inline MemcmpReturnType +[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) { LLVM_LIBC_LOOP_NOUNROLL for (size_t offset = 0; offset < count; ++offset) @@ -30,7 +30,7 @@ inline_memcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) { } #if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) -[[maybe_unused]] static inline MemcmpReturnType +[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) { if (unlikely(count >= 384)) { if (auto value = generic::Memcmp<16>::block(p1, p2)) @@ -42,7 +42,7 @@ inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) { #endif // defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) #if defined(LLVM_LIBC_ARCH_X86) -[[maybe_unused]] static inline MemcmpReturnType +[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) { if (unlikely(count >= 384)) { if (auto value = x86::sse2::Memcmp<16>::block(p1, p2)) @@ -52,7 +52,7 @@ inline_memcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) { return x86::sse2::Memcmp<16>::loop_and_tail(p1, p2, count); } -[[maybe_unused]] static inline MemcmpReturnType +[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) { if (count <= 32) return x86::sse2::Memcmp<16>::head_tail(p1, p2, count); @@ -68,7 +68,7 @@ inline_memcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) { return x86::avx2::Memcmp<32>::loop_and_tail(p1, p2, count); } -[[maybe_unused]] static inline MemcmpReturnType +[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) { if (count <= 32) return x86::sse2::Memcmp<16>::head_tail(p1, p2, count); @@ -87,7 +87,7 @@ inline_memcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) { #endif // defined(LLVM_LIBC_ARCH_X86) #if defined(LLVM_LIBC_ARCH_AARCH64) -[[maybe_unused]] static inline MemcmpReturnType +[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) { if (unlikely(count >= 128)) { // [128, ∞] if (auto value = generic::Memcmp<16>::block(p1, p2)) @@ -108,7 +108,7 @@ inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) { } #endif // defined(LLVM_LIBC_ARCH_AARCH64) -static inline MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) { +LIBC_INLINE MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) if (count == 0) return MemcmpReturnType::ZERO(); @@ -146,7 +146,7 @@ static inline MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) { #endif } -static inline int inline_memcmp(const void *p1, const void *p2, size_t count) { +LIBC_INLINE int inline_memcmp(const void *p1, const void *p2, size_t count) { return static_cast(inline_memcmp(reinterpret_cast(p1), reinterpret_cast(p2), count)); } diff --git a/libc/src/string/memory_utils/memcpy_implementations.h b/libc/src/string/memory_utils/memcpy_implementations.h index 8ae237f5d781a..23abdb8003896 100644 --- a/libc/src/string/memory_utils/memcpy_implementations.h +++ b/libc/src/string/memory_utils/memcpy_implementations.h @@ -21,7 +21,7 @@ namespace __llvm_libc { -[[maybe_unused]] static inline void +[[maybe_unused]] LIBC_INLINE void inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src, size_t count) { LLVM_LIBC_LOOP_NOUNROLL @@ -30,7 +30,7 @@ inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src, } #if defined(LLVM_LIBC_ARCH_X86) -[[maybe_unused]] static inline void +[[maybe_unused]] LIBC_INLINE void inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { if (count == 0) return; @@ -60,7 +60,7 @@ inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { return builtin::Memcpy::loop_and_tail(dst, src, count); } -[[maybe_unused]] static inline void +[[maybe_unused]] LIBC_INLINE void inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst, CPtr __restrict src, size_t count) { // Whether to use rep;movsb exclusively, not at all, or only above a certain @@ -89,7 +89,7 @@ inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst, #endif // defined(LLVM_LIBC_ARCH_X86) #if defined(LLVM_LIBC_ARCH_AARCH64) -[[maybe_unused]] static inline void +[[maybe_unused]] LIBC_INLINE void inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) { if (count == 0) return; @@ -117,8 +117,8 @@ inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) { } #endif // defined(LLVM_LIBC_ARCH_AARCH64) -static inline void inline_memcpy(Ptr __restrict dst, CPtr __restrict src, - size_t count) { +LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src, + size_t count) { using namespace __llvm_libc::builtin; #if defined(LLVM_LIBC_ARCH_X86) return inline_memcpy_x86_maybe_interpose_repmovsb(dst, src, count); @@ -133,8 +133,8 @@ static inline void inline_memcpy(Ptr __restrict dst, CPtr __restrict src, #endif } -static inline void inline_memcpy(void *__restrict dst, - const void *__restrict src, size_t count) { +LIBC_INLINE void inline_memcpy(void *__restrict dst, const void *__restrict src, + size_t count) { inline_memcpy(reinterpret_cast(dst), reinterpret_cast(src), count); } diff --git a/libc/src/string/memory_utils/memmove_implementations.h b/libc/src/string/memory_utils/memmove_implementations.h index 7e26b36a86135..0444aedb3bf7e 100644 --- a/libc/src/string/memory_utils/memmove_implementations.h +++ b/libc/src/string/memory_utils/memmove_implementations.h @@ -18,7 +18,7 @@ namespace __llvm_libc { -[[maybe_unused]] static inline void +[[maybe_unused]] LIBC_INLINE void inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { if ((count == 0) || (dst == src)) return; @@ -34,8 +34,8 @@ inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { } template -[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src, - size_t count) { +[[maybe_unused]] LIBC_INLINE void inline_memmove_generic(Ptr dst, CPtr src, + size_t count) { if (count == 0) return; if (count == 1) @@ -65,7 +65,7 @@ template } } -static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { +LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) #if defined(LLVM_LIBC_ARCH_X86) static constexpr size_t kMaxSize = x86::kAvx512F ? 64 @@ -110,7 +110,7 @@ static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { #endif } -static inline void inline_memmove(void *dst, const void *src, size_t count) { +LIBC_INLINE void inline_memmove(void *dst, const void *src, size_t count) { inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), count); } diff --git a/libc/src/string/memory_utils/op_aarch64.h b/libc/src/string/memory_utils/op_aarch64.h index 780191902f7ad..d94e7f4b2b612 100644 --- a/libc/src/string/memory_utils/op_aarch64.h +++ b/libc/src/string/memory_utils/op_aarch64.h @@ -33,7 +33,7 @@ namespace neon { template struct BzeroCacheLine { static constexpr size_t SIZE = Size; - static inline void block(Ptr dst, uint8_t) { + LIBC_INLINE void block(Ptr dst, uint8_t) { static_assert(Size == 64); #if __SIZEOF_POINTER__ == 4 asm("dc zva, %w[dst]" : : [dst] "r"(dst) : "memory"); @@ -42,7 +42,7 @@ template struct BzeroCacheLine { #endif } - static inline void loop_and_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE void loop_and_tail(Ptr dst, uint8_t value, size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -73,11 +73,11 @@ template struct Bcmp { static constexpr size_t SIZE = Size; static constexpr size_t BlockSize = 32; - static const unsigned char *as_u8(CPtr ptr) { + LIBC_INLINE static const unsigned char *as_u8(CPtr ptr) { return reinterpret_cast(ptr); } - static inline BcmpReturnType block(CPtr p1, CPtr p2) { + LIBC_INLINE static BcmpReturnType block(CPtr p1, CPtr p2) { if constexpr (Size == 16) { auto _p1 = as_u8(p1); auto _p2 = as_u8(p2); @@ -113,11 +113,11 @@ template struct Bcmp { return BcmpReturnType::ZERO(); } - static inline BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { return block(p1 + count - SIZE, p2 + count - SIZE); } - static inline BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { if constexpr (Size == 16) { auto _p1 = as_u8(p1); auto _p2 = as_u8(p2); @@ -159,7 +159,8 @@ template struct Bcmp { return BcmpReturnType::ZERO(); } - static inline BcmpReturnType loop_and_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType loop_and_tail(CPtr p1, CPtr p2, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do {