diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h index fd62a790585c8..b69ee190e6c45 100644 --- a/libc/src/__support/FPUtil/FPBits.h +++ b/libc/src/__support/FPUtil/FPBits.h @@ -51,41 +51,45 @@ template struct FPBits { UIntType bits; - void set_mantissa(UIntType mantVal) { + LIBC_INLINE void set_mantissa(UIntType mantVal) { mantVal &= (FloatProp::MANTISSA_MASK); bits &= ~(FloatProp::MANTISSA_MASK); bits |= mantVal; } - UIntType get_mantissa() const { return bits & FloatProp::MANTISSA_MASK; } + LIBC_INLINE UIntType get_mantissa() const { + return bits & FloatProp::MANTISSA_MASK; + } - void set_unbiased_exponent(UIntType expVal) { + LIBC_INLINE void set_unbiased_exponent(UIntType expVal) { expVal = (expVal << (FloatProp::MANTISSA_WIDTH)) & FloatProp::EXPONENT_MASK; bits &= ~(FloatProp::EXPONENT_MASK); bits |= expVal; } - uint16_t get_unbiased_exponent() const { + LIBC_INLINE uint16_t get_unbiased_exponent() const { return uint16_t((bits & FloatProp::EXPONENT_MASK) >> (FloatProp::MANTISSA_WIDTH)); } // The function return mantissa with the implicit bit set iff the current // value is a valid normal number. - constexpr UIntType get_explicit_mantissa() { + LIBC_INLINE constexpr UIntType get_explicit_mantissa() { return ((get_unbiased_exponent() > 0 && !is_inf_or_nan()) ? (FloatProp::MANTISSA_MASK + 1) : 0) | (FloatProp::MANTISSA_MASK & bits); } - void set_sign(bool signVal) { + LIBC_INLINE void set_sign(bool signVal) { bits |= FloatProp::SIGN_MASK; if (!signVal) bits -= FloatProp::SIGN_MASK; } - bool get_sign() const { return (bits & FloatProp::SIGN_MASK) != 0; } + LIBC_INLINE bool get_sign() const { + return (bits & FloatProp::SIGN_MASK) != 0; + } static_assert(sizeof(T) == sizeof(UIntType), "Data type and integral representation have different sizes."); @@ -112,65 +116,65 @@ template struct FPBits { FPBits() : bits(0) {} - T get_val() const { return cpp::bit_cast(bits); } + LIBC_INLINE T get_val() const { return cpp::bit_cast(bits); } - void set_val(T value) { bits = cpp::bit_cast(value); } + LIBC_INLINE void set_val(T value) { bits = cpp::bit_cast(value); } - explicit operator T() const { return get_val(); } + LIBC_INLINE explicit operator T() const { return get_val(); } - UIntType uintval() const { return bits; } + LIBC_INLINE UIntType uintval() const { return bits; } - int get_exponent() const { + LIBC_INLINE int get_exponent() const { return int(get_unbiased_exponent()) - EXPONENT_BIAS; } - bool is_zero() const { + LIBC_INLINE bool is_zero() const { // Remove sign bit by shift return (bits << 1) == 0; } - bool is_inf() const { + LIBC_INLINE bool is_inf() const { return (bits & FloatProp::EXP_MANT_MASK) == FloatProp::EXPONENT_MASK; } - bool is_nan() const { + LIBC_INLINE bool is_nan() const { return (bits & FloatProp::EXP_MANT_MASK) > FloatProp::EXPONENT_MASK; } - bool is_quiet_nan() const { + LIBC_INLINE bool is_quiet_nan() const { return (bits & FloatProp::EXP_MANT_MASK) == (FloatProp::EXPONENT_MASK | FloatProp::QUIET_NAN_MASK); } - bool is_inf_or_nan() const { + LIBC_INLINE bool is_inf_or_nan() const { return (bits & FloatProp::EXPONENT_MASK) == FloatProp::EXPONENT_MASK; } - static constexpr FPBits zero(bool sign = false) { + LIBC_INLINE static constexpr FPBits zero(bool sign = false) { return FPBits(sign ? FloatProp::SIGN_MASK : UIntType(0)); } - static constexpr FPBits neg_zero() { return zero(true); } + LIBC_INLINE static constexpr FPBits neg_zero() { return zero(true); } - static constexpr FPBits inf(bool sign = false) { + LIBC_INLINE static constexpr FPBits inf(bool sign = false) { FPBits bits(sign ? FloatProp::SIGN_MASK : UIntType(0)); bits.set_unbiased_exponent(MAX_EXPONENT); return bits; } - static constexpr FPBits neg_inf() { + LIBC_INLINE static constexpr FPBits neg_inf() { FPBits bits = inf(); bits.set_sign(1); return bits; } - static constexpr T build_nan(UIntType v) { + LIBC_INLINE static constexpr T build_nan(UIntType v) { FPBits bits = inf(); bits.set_mantissa(v); return T(bits); } - static constexpr T build_quiet_nan(UIntType v) { + LIBC_INLINE static constexpr T build_quiet_nan(UIntType v) { return build_nan(FloatProp::QUIET_NAN_MASK | v); } @@ -184,7 +188,7 @@ template struct FPBits { // 3) The function did not check exponent high limit. // 4) "number" zero value is not processed correctly. // 5) Number is unsigned, so the result can be only positive. - inline static constexpr FPBits make_value(UIntType number, int ep) { + LIBC_INLINE static constexpr FPBits make_value(UIntType number, int ep) { FPBits result; // offset: +1 for sign, but -1 for implicit first bit int lz = unsafe_clz(number) - FloatProp::EXPONENT_WIDTH; @@ -201,8 +205,8 @@ template struct FPBits { return result; } - inline static FPBits create_value(bool sign, UIntType unbiased_exp, - UIntType mantissa) { + LIBC_INLINE static FPBits create_value(bool sign, UIntType unbiased_exp, + UIntType mantissa) { FPBits result; result.set_sign(sign); result.set_unbiased_exponent(unbiased_exp); diff --git a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h index 57e1923795bfc..2c49a485073c0 100644 --- a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h +++ b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h @@ -12,6 +12,7 @@ #include "src/__support/CPP/bit.h" #include "src/__support/UInt128.h" #include "src/__support/architectures.h" +#include "src/__support/common.h" #if !defined(LLVM_LIBC_ARCH_X86) #error "Invalid include" @@ -52,19 +53,21 @@ template <> struct FPBits { UIntType bits; - void set_mantissa(UIntType mantVal) { + LIBC_INLINE void set_mantissa(UIntType mantVal) { mantVal &= (FloatProp::MANTISSA_MASK); bits &= ~(FloatProp::MANTISSA_MASK); bits |= mantVal; } - UIntType get_mantissa() const { return bits & FloatProp::MANTISSA_MASK; } + LIBC_INLINE UIntType get_mantissa() const { + return bits & FloatProp::MANTISSA_MASK; + } - UIntType get_explicit_mantissa() const { + LIBC_INLINE UIntType get_explicit_mantissa() const { return bits & (FloatProp::MANTISSA_MASK | FloatProp::EXPLICIT_BIT_MASK); } - void set_unbiased_exponent(UIntType expVal) { + LIBC_INLINE void set_unbiased_exponent(UIntType expVal) { expVal = (expVal << (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)) & FloatProp::EXPONENT_MASK; @@ -72,28 +75,28 @@ template <> struct FPBits { bits |= expVal; } - uint16_t get_unbiased_exponent() const { + LIBC_INLINE uint16_t get_unbiased_exponent() const { return uint16_t((bits & FloatProp::EXPONENT_MASK) >> (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)); } - void set_implicit_bit(bool implicitVal) { + LIBC_INLINE void set_implicit_bit(bool implicitVal) { bits &= ~(UIntType(1) << FloatProp::MANTISSA_WIDTH); bits |= (UIntType(implicitVal) << FloatProp::MANTISSA_WIDTH); } - bool get_implicit_bit() const { + LIBC_INLINE bool get_implicit_bit() const { return ((bits & (UIntType(1) << FloatProp::MANTISSA_WIDTH)) >> FloatProp::MANTISSA_WIDTH); } - void set_sign(bool signVal) { + LIBC_INLINE void set_sign(bool signVal) { bits &= ~(FloatProp::SIGN_MASK); UIntType sign1 = UIntType(signVal) << (FloatProp::BIT_WIDTH - 1); bits |= sign1; } - bool get_sign() const { + LIBC_INLINE bool get_sign() const { return ((bits & FloatProp::SIGN_MASK) >> (FloatProp::BIT_WIDTH - 1)); } @@ -111,9 +114,11 @@ template <> struct FPBits { cpp::enable_if_t, int> = 0> explicit FPBits(XType x) : bits(x) {} - operator long double() { return cpp::bit_cast(bits); } + LIBC_INLINE operator long double() { + return cpp::bit_cast(bits); + } - UIntType uintval() { + LIBC_INLINE UIntType uintval() { // We zero the padding bits as they can contain garbage. static constexpr UIntType MASK = (UIntType(1) << (sizeof(long double) * 8 - @@ -122,23 +127,23 @@ template <> struct FPBits { return bits & MASK; } - int get_exponent() const { + LIBC_INLINE int get_exponent() const { if (get_unbiased_exponent() == 0) return int(1) - EXPONENT_BIAS; return int(get_unbiased_exponent()) - EXPONENT_BIAS; } - bool is_zero() const { + LIBC_INLINE bool is_zero() const { return get_unbiased_exponent() == 0 && get_mantissa() == 0 && get_implicit_bit() == 0; } - bool is_inf() const { + LIBC_INLINE bool is_inf() const { return get_unbiased_exponent() == MAX_EXPONENT && get_mantissa() == 0 && get_implicit_bit() == 1; } - bool is_nan() const { + LIBC_INLINE bool is_nan() const { if (get_unbiased_exponent() == MAX_EXPONENT) { return (get_implicit_bit() == 0) || get_mantissa() != 0; } else if (get_unbiased_exponent() != 0) { @@ -147,29 +152,31 @@ template <> struct FPBits { return false; } - bool is_inf_or_nan() const { + LIBC_INLINE bool is_inf_or_nan() const { return (get_unbiased_exponent() == MAX_EXPONENT) || (get_unbiased_exponent() != 0 && get_implicit_bit() == 0); } // Methods below this are used by tests. - static FPBits zero() { return FPBits(0.0l); } + LIBC_INLINE static FPBits zero() { + return FPBits(0.0l); + } - static FPBits neg_zero() { + LIBC_INLINE static FPBits neg_zero() { FPBits bits(0.0l); bits.set_sign(1); return bits; } - static FPBits inf() { + LIBC_INLINE static FPBits inf() { FPBits bits(0.0l); bits.set_unbiased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); return bits; } - static FPBits neg_inf() { + LIBC_INLINE static FPBits neg_inf() { FPBits bits(0.0l); bits.set_unbiased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); @@ -177,7 +184,7 @@ template <> struct FPBits { return bits; } - static long double build_nan(UIntType v) { + LIBC_INLINE static long double build_nan(UIntType v) { FPBits bits(0.0l); bits.set_unbiased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); @@ -185,11 +192,11 @@ template <> struct FPBits { return bits; } - static long double build_quiet_nan(UIntType v) { + LIBC_INLINE static long double build_quiet_nan(UIntType v) { return build_nan(FloatProp::QUIET_NAN_MASK | v); } - inline static FPBits + LIBC_INLINE static FPBits create_value(bool sign, UIntType unbiased_exp, UIntType mantissa) { FPBits result; result.set_sign(sign); diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 512ed270200fb..dd4b07ed97b4f 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -56,6 +56,7 @@ add_header_library( libc.src.__support.FPUtil.fma libc.src.__support.FPUtil.multiply_add libc.src.__support.FPUtil.nearest_integer + libc.src.__support.common ) add_header_library( @@ -66,6 +67,7 @@ add_header_library( .range_reduction libc.src.__support.FPUtil.fp_bits libc.src.__support.FPUtil.polyeval + libc.src.__support.common ) add_entrypoint_object( @@ -1246,6 +1248,7 @@ add_object_library( libc.src.__support.FPUtil.multiply_add libc.src.__support.FPUtil.nearest_integer libc.src.__support.FPUtil.polyeval + libc.src.__support.common libc.include.errno libc.src.errno.errno libc.include.math @@ -1319,6 +1322,7 @@ add_object_library( libc.src.__support.FPUtil.nearest_integer libc.src.__support.FPUtil.nearest_integer_operations libc.src.__support.FPUtil.polyeval + libc.src.__support.common libc.include.errno libc.src.errno.errno libc.include.math diff --git a/libc/src/math/generic/explogxf.h b/libc/src/math/generic/explogxf.h index e60c0d5c34793..cce96172cf0ea 100644 --- a/libc/src/math/generic/explogxf.h +++ b/libc/src/math/generic/explogxf.h @@ -56,7 +56,7 @@ struct ExpBase { 0x1.ffffffffe5bc8p-2, 0x1.555555555cd67p-3, 0x1.5555c2a9b48b4p-5, 0x1.11112a0e34bdbp-7}; - static double powb_lo(double dx) { + LIBC_INLINE static double powb_lo(double dx) { using fputil::multiply_add; double dx2 = dx * dx; double c0 = 1.0 + dx; @@ -150,7 +150,7 @@ struct exp_b_reduc_t { // - EXP_2_MID : look up table for bit fields of 2^mid // Return: // { 2^(hi + mid), lo } -template static inline exp_b_reduc_t exp_b_range_reduc(float x) { +template LIBC_INLINE exp_b_reduc_t exp_b_range_reduc(float x) { double xd = static_cast(x); // kd = round((hi + mid) * log2(b) * 2^MID_BITS) double kd = fputil::nearest_integer(Base::LOG2_B * xd); @@ -207,7 +207,7 @@ template static inline exp_b_reduc_t exp_b_range_reduc(float x) { // The main point of these formulas is that the expensive part of calculating // the polynomials approximating lower parts of e^(x) and e^(-x) are shared // and only done once. -template static inline double exp_pm_eval(float x) { +template LIBC_INLINE double exp_pm_eval(float x) { double xd = static_cast(x); // round(x * log2(e) * 2^5) @@ -259,7 +259,7 @@ template static inline double exp_pm_eval(float x) { } // x should be positive, normal finite value -inline static double log2_eval(double x) { +LIBC_INLINE static double log2_eval(double x) { using FPB = fputil::FPBits; FPB bs(x); @@ -287,7 +287,7 @@ inline static double log2_eval(double x) { } // x should be positive, normal finite value -inline static double log_eval(double x) { +LIBC_INLINE static double log_eval(double x) { // ln(x) = log[2,x] * ln(2) return log2_eval(x) * 0x1.62e42fefa39efp-1; } diff --git a/libc/src/math/generic/inv_trigf_utils.h b/libc/src/math/generic/inv_trigf_utils.h index 202a4a6be5b16..a01db3ab8e7bd 100644 --- a/libc/src/math/generic/inv_trigf_utils.h +++ b/libc/src/math/generic/inv_trigf_utils.h @@ -37,7 +37,7 @@ extern const double ATAN_K[5]; // atan(u) + atan(v) = atan((u+v)/(1-uv)) // x should be positive, normal finite value -static inline double atan_eval(double x) { +LIBC_INLINE double atan_eval(double x) { using FPB = fputil::FPBits; // Added some small value to umin and umax mantissa to avoid possible rounding // errors. @@ -99,7 +99,7 @@ constexpr double ASIN_COEFFS[10] = {0x1.5555555540fa1p-3, 0x1.333333512edc2p-4, -0x1.df946fa875ddp-8, 0x1.02311ecf99c28p-5}; // Evaluate P(x^2) - 1, where P(x^2) ~ asin(x)/x -static inline double asin_eval(double xsq) { +LIBC_INLINE double asin_eval(double xsq) { double x4 = xsq * xsq; double r1 = fputil::polyeval(x4, ASIN_COEFFS[0], ASIN_COEFFS[2], ASIN_COEFFS[4], ASIN_COEFFS[6], ASIN_COEFFS[8]); diff --git a/libc/src/math/generic/math_utils.h b/libc/src/math/generic/math_utils.h index 61c84dc630cfc..550117f7818e6 100644 --- a/libc/src/math/generic/math_utils.h +++ b/libc/src/math/generic/math_utils.h @@ -19,21 +19,21 @@ namespace __llvm_libc { -static inline uint32_t as_uint32_bits(float x) { +LIBC_INLINE uint32_t as_uint32_bits(float x) { return cpp::bit_cast(x); } -static inline uint64_t as_uint64_bits(double x) { +LIBC_INLINE uint64_t as_uint64_bits(double x) { return cpp::bit_cast(x); } -static inline float as_float(uint32_t x) { return cpp::bit_cast(x); } +LIBC_INLINE float as_float(uint32_t x) { return cpp::bit_cast(x); } -static inline double as_double(uint64_t x) { return cpp::bit_cast(x); } +LIBC_INLINE double as_double(uint64_t x) { return cpp::bit_cast(x); } -static inline uint32_t top12_bits(float x) { return as_uint32_bits(x) >> 20; } +LIBC_INLINE uint32_t top12_bits(float x) { return as_uint32_bits(x) >> 20; } -static inline uint32_t top12_bits(double x) { return as_uint64_bits(x) >> 52; } +LIBC_INLINE uint32_t top12_bits(double x) { return as_uint64_bits(x) >> 52; } // Values to trigger underflow and overflow. template struct XFlowValues; @@ -50,17 +50,17 @@ template <> struct XFlowValues { static const double MAY_UNDERFLOW_VALUE; }; -template static inline T with_errno(T x, int err) { +template LIBC_INLINE T with_errno(T x, int err) { if (math_errhandling & MATH_ERRNO) errno = err; return x; } -template static inline void force_eval(T x) { +template LIBC_INLINE void force_eval(T x) { volatile T y UNUSED = x; } -template static inline T opt_barrier(T x) { +template LIBC_INLINE T opt_barrier(T x) { volatile T y = x; return y; } @@ -96,7 +96,7 @@ T may_underflow(uint32_t sign) { } template = 0> -static inline constexpr float invalid(T x) { +LIBC_INLINE constexpr float invalid(T x) { T y = (x - x) / (x - x); return isnan(x) ? y : with_errno(y, EDOM); } diff --git a/libc/src/math/generic/range_reduction.h b/libc/src/math/generic/range_reduction.h index 095226608ee79..f620bcda0948f 100644 --- a/libc/src/math/generic/range_reduction.h +++ b/libc/src/math/generic/range_reduction.h @@ -12,6 +12,7 @@ #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/multiply_add.h" #include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/common.h" namespace __llvm_libc { @@ -40,7 +41,7 @@ static constexpr int THIRTYTWO_OVER_PI_28_LSB_EXP[N_ENTRIES] = { // Return k and y, where // k = round(x * 16 / pi) and y = (x * 16 / pi) - k. -static inline int64_t small_range_reduction(double x, double &y) { +LIBC_INLINE int64_t small_range_reduction(double x, double &y) { double prod = x * THIRTYTWO_OVER_PI_28[0]; double kd = fputil::nearest_integer(prod); y = prod - kd; @@ -55,7 +56,7 @@ static inline int64_t small_range_reduction(double x, double &y) { // contributing to the lowest 6 binary digits (k & 63). If the least // significant bit of x * the least significant bit of THIRTYTWO_OVER_PI_28[i] // >= 64, we can completely ignore THIRTYTWO_OVER_PI_28[i]. -static inline int64_t large_range_reduction(double x, int x_exp, double &y) { +LIBC_INLINE int64_t large_range_reduction(double x, int x_exp, double &y) { int idx = 0; y = 0; int x_lsb_exp_m4 = x_exp - fputil::FloatProperties::MANTISSA_WIDTH; diff --git a/libc/src/math/generic/range_reduction_fma.h b/libc/src/math/generic/range_reduction_fma.h index 102c1d7a59254..79c076d265c67 100644 --- a/libc/src/math/generic/range_reduction_fma.h +++ b/libc/src/math/generic/range_reduction_fma.h @@ -12,6 +12,7 @@ #include "src/__support/FPUtil/FMA.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/common.h" namespace __llvm_libc { @@ -30,7 +31,7 @@ static constexpr double THIRTYTWO_OVER_PI[5] = { // Return k and y, where // k = round(x * 32 / pi) and y = (x * 32 / pi) - k. -static inline int64_t small_range_reduction(double x, double &y) { +LIBC_INLINE int64_t small_range_reduction(double x, double &y) { double kd = fputil::nearest_integer(x * THIRTYTWO_OVER_PI[0]); y = fputil::fma(x, THIRTYTWO_OVER_PI[0], -kd); y = fputil::fma(x, THIRTYTWO_OVER_PI[1], y); @@ -40,7 +41,7 @@ static inline int64_t small_range_reduction(double x, double &y) { // Return k and y, where // k = round(x * 32 / pi) and y = (x * 32 / pi) - k. // This is used for sinf, cosf, sincosf. -static inline int64_t large_range_reduction(double x, int x_exp, double &y) { +LIBC_INLINE int64_t large_range_reduction(double x, int x_exp, double &y) { // 2^45 <= |x| < 2^99 if (x_exp < 99) { // - When x < 2^99, the full exact product of x * THIRTYTWO_OVER_PI[0] diff --git a/libc/src/math/generic/sincosf_utils.h b/libc/src/math/generic/sincosf_utils.h index 38ee2784de5a9..a193216480b70 100644 --- a/libc/src/math/generic/sincosf_utils.h +++ b/libc/src/math/generic/sincosf_utils.h @@ -58,8 +58,8 @@ const double SIN_K_PI_OVER_32[64] = { -0x1.917a6bc29b42cp-4, }; -static inline void sincosf_eval(double xd, uint32_t x_abs, double &sin_k, - double &cos_k, double &sin_y, double &cosm1_y) { +LIBC_INLINE void sincosf_eval(double xd, uint32_t x_abs, double &sin_k, + double &cos_k, double &sin_y, double &cosm1_y) { int64_t k; double y; diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt index ca5f45efc809b..210de837189e8 100644 --- a/libc/src/pthread/CMakeLists.txt +++ b/libc/src/pthread/CMakeLists.txt @@ -103,6 +103,7 @@ add_header_library( HDRS pthread_mutexattr.h DEPENDS + libc.src.__support.common libc.include.pthread ) diff --git a/libc/src/pthread/pthread_mutexattr.h b/libc/src/pthread/pthread_mutexattr.h index 98f9ff9067e44..b65c672d34ce4 100644 --- a/libc/src/pthread/pthread_mutexattr.h +++ b/libc/src/pthread/pthread_mutexattr.h @@ -9,6 +9,8 @@ #ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_MUTEXATTR_H #define LLVM_LIBC_SRC_PTHREAD_PTHREAD_MUTEXATTR_H +#include "src/__support/common.h" + #include namespace __llvm_libc { @@ -31,12 +33,12 @@ constexpr pthread_mutexattr_t DEFAULT_MUTEXATTR = PTHREAD_MUTEX_STALLED << unsigned(PThreadMutexAttrPos::ROBUST_SHIFT) | PTHREAD_PROCESS_PRIVATE << unsigned(PThreadMutexAttrPos::PSHARED_SHIFT); -static inline int get_mutexattr_type(pthread_mutexattr_t attr) { +LIBC_INLINE int get_mutexattr_type(pthread_mutexattr_t attr) { return (attr & unsigned(PThreadMutexAttrPos::TYPE_MASK)) >> unsigned(PThreadMutexAttrPos::TYPE_SHIFT); } -static inline int get_mutexattr_robust(pthread_mutexattr_t attr) { +LIBC_INLINE int get_mutexattr_robust(pthread_mutexattr_t attr) { return (attr & unsigned(PThreadMutexAttrPos::ROBUST_MASK)) >> unsigned(PThreadMutexAttrPos::ROBUST_SHIFT); } diff --git a/libc/src/signal/linux/signal_utils.h b/libc/src/signal/linux/signal_utils.h index 7c45e0862d524..ce8a07a8a8b5f 100644 --- a/libc/src/signal/linux/signal_utils.h +++ b/libc/src/signal/linux/signal_utils.h @@ -11,6 +11,7 @@ #include "include/sys/syscall.h" // For syscall numbers. #include "src/__support/OSUtil/syscall.h" // For internal syscall function. +#include "src/__support/common.h" #include #include @@ -29,7 +30,7 @@ struct KernelSigaction { using HandlerType = void(int); using SiginfoHandlerType = void(int, siginfo_t *, void *); - KernelSigaction &operator=(const struct sigaction &sa) { + LIBC_INLINE KernelSigaction &operator=(const struct sigaction &sa) { sa_flags = sa.sa_flags; sa_restorer = sa.sa_restorer; sa_mask = sa.sa_mask; @@ -41,7 +42,7 @@ struct KernelSigaction { return *this; } - operator struct sigaction() const { + LIBC_INLINE operator struct sigaction() const { struct sigaction sa; sa.sa_flags = sa_flags; sa.sa_mask = sa_mask; @@ -63,14 +64,14 @@ struct KernelSigaction { static constexpr size_t BITS_PER_SIGWORD = sizeof(unsigned long) * 8; -constexpr sigset_t full_set() { return sigset_t{{-1UL}}; } +LIBC_INLINE constexpr sigset_t full_set() { return sigset_t{{-1UL}}; } -constexpr sigset_t empty_set() { return sigset_t{{0}}; } +LIBC_INLINE constexpr sigset_t empty_set() { return sigset_t{{0}}; } // Set the bit corresponding to |signal| in |set|. Return true on success // and false on failure. The function will fail if |signal| is greater than // NSIG or negative. -constexpr inline bool add_signal(sigset_t &set, int signal) { +LIBC_INLINE constexpr bool add_signal(sigset_t &set, int signal) { if (signal > NSIG || signal <= 0) return false; size_t n = size_t(signal) - 1; @@ -83,7 +84,7 @@ constexpr inline bool add_signal(sigset_t &set, int signal) { // Reset the bit corresponding to |signal| in |set|. Return true on success // and false on failure. The function will fail if |signal| is greater than // NSIG or negative. -constexpr inline bool delete_signal(sigset_t &set, int signal) { +LIBC_INLINE constexpr bool delete_signal(sigset_t &set, int signal) { if (signal > NSIG || signal <= 0) return false; size_t n = size_t(signal) - 1; @@ -93,13 +94,13 @@ constexpr inline bool delete_signal(sigset_t &set, int signal) { return true; } -static inline int block_all_signals(sigset_t &set) { +LIBC_INLINE int block_all_signals(sigset_t &set) { sigset_t full = full_set(); return __llvm_libc::syscall_impl(SYS_rt_sigprocmask, SIG_BLOCK, &full, &set, sizeof(sigset_t)); } -static inline int restore_signals(const sigset_t &set) { +LIBC_INLINE int restore_signals(const sigset_t &set) { return __llvm_libc::syscall_impl(SYS_rt_sigprocmask, SIG_SETMASK, &set, nullptr, sizeof(sigset_t)); } diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt index ec61735f41542..749cfab8d01f1 100644 --- a/libc/src/string/CMakeLists.txt +++ b/libc/src/string/CMakeLists.txt @@ -7,6 +7,7 @@ add_header_library( DEPENDS libc.include.stdlib libc.src.__support.CPP.bitset + libc.src.__support.common .memory_utils.memcpy_implementation .memory_utils.bzero_implementation ) diff --git a/libc/src/string/memory_utils/README.md b/libc/src/string/memory_utils/README.md index 83a2906675f69..4b2069caba065 100644 --- a/libc/src/string/memory_utils/README.md +++ b/libc/src/string/memory_utils/README.md @@ -37,20 +37,20 @@ template struct Memset { static constexpr size_t SIZE = Size; - static inline void block(Ptr dst, uint8_t value) { + LIBC_INLINE static void block(Ptr dst, uint8_t value) { // Implement me } - static inline void tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void tail(Ptr dst, uint8_t value, size_t count) { block(dst + count - SIZE, value); } - static inline void head_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void head_tail(Ptr dst, uint8_t value, size_t count) { block(dst, value); tail(dst, value, count); } - static inline void loop_and_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) { size_t offset = 0; do { block(dst + offset, value); diff --git a/libc/src/string/memory_utils/bzero_implementations.h b/libc/src/string/memory_utils/bzero_implementations.h index 550c910def885..02780d7859a73 100644 --- a/libc/src/string/memory_utils/bzero_implementations.h +++ b/libc/src/string/memory_utils/bzero_implementations.h @@ -9,17 +9,18 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BZERO_IMPLEMENTATIONS_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BZERO_IMPLEMENTATIONS_H +#include "src/__support/common.h" #include "src/string/memory_utils/memset_implementations.h" #include // size_t namespace __llvm_libc { -inline static void inline_bzero(Ptr dst, size_t count) { +LIBC_INLINE static void inline_bzero(Ptr dst, size_t count) { inline_memset(dst, 0, count); } -inline static void inline_bzero(void *dst, size_t count) { +LIBC_INLINE static void inline_bzero(void *dst, size_t count) { inline_bzero(reinterpret_cast(dst), count); } diff --git a/libc/src/string/memory_utils/memset_implementations.h b/libc/src/string/memory_utils/memset_implementations.h index dbcc356bcdf07..c9d047c01044c 100644 --- a/libc/src/string/memory_utils/memset_implementations.h +++ b/libc/src/string/memory_utils/memset_implementations.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H #include "src/__support/architectures.h" +#include "src/__support/common.h" #include "src/string/memory_utils/op_aarch64.h" #include "src/string/memory_utils/op_builtin.h" #include "src/string/memory_utils/op_generic.h" @@ -20,7 +21,7 @@ namespace __llvm_libc { -[[maybe_unused]] inline static void +[[maybe_unused]] LIBC_INLINE static void inline_memset_embedded_tiny(Ptr dst, uint8_t value, size_t count) { LLVM_LIBC_LOOP_NOUNROLL for (size_t offset = 0; offset < count; ++offset) @@ -29,8 +30,8 @@ inline_memset_embedded_tiny(Ptr dst, uint8_t value, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) template -[[maybe_unused]] inline static void inline_memset_x86(Ptr dst, uint8_t value, - size_t count) { +[[maybe_unused]] LIBC_INLINE static void +inline_memset_x86(Ptr dst, uint8_t value, size_t count) { if (count == 0) return; if (count == 1) @@ -58,7 +59,7 @@ template #if defined(LLVM_LIBC_ARCH_AARCH64) template -[[maybe_unused]] inline static void +[[maybe_unused]] LIBC_INLINE static void inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) { if (count == 0) return; @@ -94,7 +95,7 @@ inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) { } #endif // defined(LLVM_LIBC_ARCH_AARCH64) -inline static void inline_memset(Ptr dst, uint8_t value, size_t count) { +LIBC_INLINE static void inline_memset(Ptr dst, uint8_t value, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) static constexpr size_t kMaxSize = x86::kAvx512F ? 64 : x86::kAvx ? 32 @@ -113,7 +114,7 @@ inline static void inline_memset(Ptr dst, uint8_t value, size_t count) { #endif } -inline static void inline_memset(void *dst, uint8_t value, size_t count) { +LIBC_INLINE static void inline_memset(void *dst, uint8_t value, size_t count) { inline_memset(reinterpret_cast(dst), value, count); } diff --git a/libc/src/string/memory_utils/op_aarch64.h b/libc/src/string/memory_utils/op_aarch64.h index cd64905f36f15..3d625c20c7624 100644 --- a/libc/src/string/memory_utils/op_aarch64.h +++ b/libc/src/string/memory_utils/op_aarch64.h @@ -55,7 +55,7 @@ template struct BzeroCacheLine { } }; -inline static bool hasZva() { +LIBC_INLINE static bool hasZva() { uint64_t zva_val; asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val)); // DC ZVA is permitted if DZP, bit [4] is zero. diff --git a/libc/src/string/memory_utils/op_builtin.h b/libc/src/string/memory_utils/op_builtin.h index ce33de3e64678..cf90c02ef1dc1 100644 --- a/libc/src/string/memory_utils/op_builtin.h +++ b/libc/src/string/memory_utils/op_builtin.h @@ -23,7 +23,7 @@ namespace __llvm_libc::builtin { // Memcpy template struct Memcpy { static constexpr size_t SIZE = Size; - static inline void block(Ptr __restrict dst, CPtr __restrict src) { + LIBC_INLINE static void block(Ptr __restrict dst, CPtr __restrict src) { #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE return __builtin_memcpy_inline(dst, src, SIZE); #else @@ -33,19 +33,19 @@ template struct Memcpy { #endif } - static inline void tail(Ptr __restrict dst, CPtr __restrict src, - size_t count) { + LIBC_INLINE static void tail(Ptr __restrict dst, CPtr __restrict src, + size_t count) { block(dst + count - SIZE, src + count - SIZE); } - static inline void head_tail(Ptr __restrict dst, CPtr __restrict src, - size_t count) { + LIBC_INLINE static void head_tail(Ptr __restrict dst, CPtr __restrict src, + size_t count) { block(dst, src); tail(dst, src, count); } - static inline void loop_and_tail(Ptr __restrict dst, CPtr __restrict src, - size_t count) { + LIBC_INLINE static void loop_and_tail(Ptr __restrict dst, CPtr __restrict src, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -61,7 +61,7 @@ template struct Memcpy { template struct Memset { using ME = Memset; static constexpr size_t SIZE = Size; - static inline void block(Ptr dst, uint8_t value) { + LIBC_INLINE static void block(Ptr dst, uint8_t value) { #ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE __builtin_memset_inline(dst, value, Size); #else @@ -71,16 +71,16 @@ template struct Memset { #endif } - static inline void tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void tail(Ptr dst, uint8_t value, size_t count) { block(dst + count - SIZE, value); } - static inline void head_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void head_tail(Ptr dst, uint8_t value, size_t count) { block(dst, value); tail(dst, value, count); } - static inline void loop_and_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -96,22 +96,22 @@ template struct Memset { template struct Bcmp { using ME = Bcmp; static constexpr size_t SIZE = Size; - static inline BcmpReturnType block(CPtr, CPtr) { + LIBC_INLINE static BcmpReturnType block(CPtr, CPtr) { deferred_static_assert("Missing __builtin_memcmp_inline"); return BcmpReturnType::ZERO(); } - static inline BcmpReturnType tail(CPtr, CPtr, size_t) { + LIBC_INLINE static BcmpReturnType tail(CPtr, CPtr, size_t) { deferred_static_assert("Not implemented"); return BcmpReturnType::ZERO(); } - static inline BcmpReturnType head_tail(CPtr, CPtr, size_t) { + LIBC_INLINE static BcmpReturnType head_tail(CPtr, CPtr, size_t) { deferred_static_assert("Not implemented"); return BcmpReturnType::ZERO(); } - static inline BcmpReturnType loop_and_tail(CPtr, CPtr, size_t) { + LIBC_INLINE static BcmpReturnType loop_and_tail(CPtr, CPtr, size_t) { deferred_static_assert("Not implemented"); return BcmpReturnType::ZERO(); } @@ -122,22 +122,22 @@ template struct Bcmp { template struct Memcmp { using ME = Memcmp; static constexpr size_t SIZE = Size; - static inline MemcmpReturnType block(CPtr, CPtr) { + LIBC_INLINE static MemcmpReturnType block(CPtr, CPtr) { deferred_static_assert("Missing __builtin_memcmp_inline"); return MemcmpReturnType::ZERO(); } - static inline MemcmpReturnType tail(CPtr, CPtr, size_t) { + LIBC_INLINE static MemcmpReturnType tail(CPtr, CPtr, size_t) { deferred_static_assert("Not implemented"); return MemcmpReturnType::ZERO(); } - static inline MemcmpReturnType head_tail(CPtr, CPtr, size_t) { + LIBC_INLINE static MemcmpReturnType head_tail(CPtr, CPtr, size_t) { deferred_static_assert("Not implemented"); return MemcmpReturnType::ZERO(); } - static inline MemcmpReturnType loop_and_tail(CPtr, CPtr, size_t) { + LIBC_INLINE static MemcmpReturnType loop_and_tail(CPtr, CPtr, size_t) { deferred_static_assert("Not implemented"); return MemcmpReturnType::ZERO(); } diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h index 1603dbf4ebfd2..3d505fbd9d2c1 100644 --- a/libc/src/string/memory_utils/op_generic.h +++ b/libc/src/string/memory_utils/op_generic.h @@ -46,7 +46,9 @@ namespace __llvm_libc::generic { // using UInt8T = MyMap::find_type<1>; template struct CTPair { using type = T; - static CTPair get_pair(cpp::integral_constant) { return {}; } + LIBC_INLINE static CTPair get_pair(cpp::integral_constant) { + return {}; + } }; template struct CTMap : public Pairs... { using Pairs::get_pair...; @@ -63,11 +65,13 @@ template struct ScalarType { using Type = T; static_assert(cpp::is_integral_v && !cpp::is_signed_v); - static inline Type load(CPtr src) { return ::__llvm_libc::load(src); } - static inline void store(Ptr dst, Type value) { + LIBC_INLINE static Type load(CPtr src) { + return ::__llvm_libc::load(src); + } + LIBC_INLINE static void store(Ptr dst, Type value) { ::__llvm_libc::store(dst, value); } - static inline Type splat(uint8_t value) { + LIBC_INLINE static Type splat(uint8_t value) { return Type(~0) / Type(0xFF) * Type(value); } }; @@ -100,11 +104,13 @@ template <> struct VectorValueType<64> { // Implements load, store and splat for vector types. template struct VectorType { using Type = typename VectorValueType::type; - static inline Type load(CPtr src) { return ::__llvm_libc::load(src); } - static inline void store(Ptr dst, Type value) { + LIBC_INLINE static Type load(CPtr src) { + return ::__llvm_libc::load(src); + } + LIBC_INLINE static void store(Ptr dst, Type value) { ::__llvm_libc::store(dst, value); } - static inline Type splat(uint8_t value) { + LIBC_INLINE static Type splat(uint8_t value) { Type Out; // This for loop is optimized out for vector types. for (size_t i = 0; i < Size; ++i) @@ -140,17 +146,17 @@ using NativeTypeMap = template struct ArrayType { using Type = cpp::array; static constexpr size_t SizeOfElement = sizeof(typename SubType::Type); - static inline Type load(CPtr src) { + LIBC_INLINE static Type load(CPtr src) { Type Value; for (size_t I = 0; I < ArraySize; ++I) Value[I] = SubType::load(src + (I * SizeOfElement)); return Value; } - static inline void store(Ptr dst, Type Value) { + LIBC_INLINE static void store(Ptr dst, Type Value) { for (size_t I = 0; I < ArraySize; ++I) SubType::store(dst + (I * SizeOfElement), Value[I]); } - static inline Type splat(uint8_t value) { + LIBC_INLINE static Type splat(uint8_t value) { Type Out; for (size_t I = 0; I < ArraySize; ++I) Out[I] = SubType::splat(value); @@ -190,7 +196,7 @@ template struct Memset { static_assert(is_power2(MaxSize)); static constexpr size_t SIZE = Size; - static inline void block(Ptr dst, uint8_t value) { + LIBC_INLINE static void block(Ptr dst, uint8_t value) { if constexpr (Size == 3) { Memset<1, MaxSize>::block(dst + 2, value); Memset<2, MaxSize>::block(dst, value); @@ -204,16 +210,16 @@ template struct Memset { } } - static inline void tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void tail(Ptr dst, uint8_t value, size_t count) { block(dst + count - SIZE, value); } - static inline void head_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void head_tail(Ptr dst, uint8_t value, size_t count) { block(dst, value); tail(dst, value, count); } - static inline void loop_and_tail(Ptr dst, uint8_t value, size_t count) { + LIBC_INLINE static void loop_and_tail(Ptr dst, uint8_t value, size_t count) { static_assert(SIZE > 1); size_t offset = 0; do { @@ -233,16 +239,16 @@ template struct Bcmp { ? sizeof(uint64_t) : sizeof(uint32_t); - template static inline uint32_t load_xor(CPtr p1, CPtr p2) { + template LIBC_INLINE static uint32_t load_xor(CPtr p1, CPtr p2) { return load(p1) ^ load(p2); } template - static inline uint32_t load_not_equal(CPtr p1, CPtr p2) { + LIBC_INLINE static uint32_t load_not_equal(CPtr p1, CPtr p2) { return load(p1) != load(p2); } - static inline BcmpReturnType block(CPtr p1, CPtr p2) { + LIBC_INLINE static BcmpReturnType block(CPtr p1, CPtr p2) { if constexpr (Size == 1) { return load_xor(p1, p2); } else if constexpr (Size == 2) { @@ -261,15 +267,16 @@ template struct Bcmp { return BcmpReturnType::ZERO(); } - static inline BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { return block(p1 + count - SIZE, p2 + count - SIZE); } - static inline BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { return block(p1, p2) | tail(p1, p2, count); } - static inline BcmpReturnType loop_and_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType loop_and_tail(CPtr p1, CPtr p2, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -290,23 +297,23 @@ template struct Memcmp { ? sizeof(uint64_t) : sizeof(uint32_t); - template static inline T load_be(CPtr ptr) { + template LIBC_INLINE static T load_be(CPtr ptr) { return Endian::to_big_endian(load(ptr)); } template - static inline MemcmpReturnType load_be_diff(CPtr p1, CPtr p2) { + LIBC_INLINE static MemcmpReturnType load_be_diff(CPtr p1, CPtr p2) { return load_be(p1) - load_be(p2); } template - static inline MemcmpReturnType load_be_cmp(CPtr p1, CPtr p2) { + LIBC_INLINE static MemcmpReturnType load_be_cmp(CPtr p1, CPtr p2) { const auto la = load_be(p1); const auto lb = load_be(p2); return la > lb ? 1 : la < lb ? -1 : 0; } - static inline MemcmpReturnType block(CPtr p1, CPtr p2) { + LIBC_INLINE static MemcmpReturnType block(CPtr p1, CPtr p2) { if constexpr (Size == 1) { return load_be_diff(p1, p2); } else if constexpr (Size == 2) { @@ -329,17 +336,19 @@ template struct Memcmp { } } - static inline MemcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static MemcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { return block(p1 + count - SIZE, p2 + count - SIZE); } - static inline MemcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static MemcmpReturnType head_tail(CPtr p1, CPtr p2, + size_t count) { if (auto value = block(p1, p2)) return value; return tail(p1, p2, count); } - static inline MemcmpReturnType loop_and_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static MemcmpReturnType loop_and_tail(CPtr p1, CPtr p2, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -360,7 +369,7 @@ template struct Memmove { using T = getTypeFor; static constexpr size_t SIZE = Size; - static inline void block(Ptr dst, CPtr src) { + LIBC_INLINE static void block(Ptr dst, CPtr src) { if constexpr (is_void_v) { deferred_static_assert("Unimplemented Size"); } else { @@ -368,7 +377,7 @@ template struct Memmove { } } - static inline void head_tail(Ptr dst, CPtr src, size_t count) { + LIBC_INLINE static void head_tail(Ptr dst, CPtr src, size_t count) { const size_t offset = count - Size; if constexpr (is_void_v) { deferred_static_assert("Unimplemented Size"); @@ -402,7 +411,7 @@ template struct Memmove { // [_SSSSSSSS________________________] // [___SSSSSSSA______________________] template - static inline void align_forward(Ptr &dst, CPtr &src, size_t &count) { + LIBC_INLINE static void align_forward(Ptr &dst, CPtr &src, size_t &count) { Ptr prev_dst = dst; CPtr prev_src = src; size_t prev_count = count; @@ -430,7 +439,7 @@ template struct Memmove { // [__________________ASSSSSSS_______] // [______________________SSSSSSSS___] template - static inline void align_backward(Ptr &dst, CPtr &src, size_t &count) { + LIBC_INLINE static void align_backward(Ptr &dst, CPtr &src, size_t &count) { Ptr headtail_dst = dst + count; CPtr headtail_src = src + count; size_t headtail_size = 0; @@ -455,7 +464,8 @@ template struct Memmove { // [___________________LLLLLLLL_________] // [_________________SSSSSSSS___________] // [_______________________SSSSSSSS_____] - static inline void loop_and_tail_forward(Ptr dst, CPtr src, size_t count) { + LIBC_INLINE static void loop_and_tail_forward(Ptr dst, CPtr src, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); const size_t tail_offset = count - Size; const auto tail_value = T::load(src + tail_offset); @@ -482,7 +492,8 @@ template struct Memmove { // [_________LLLLLLLL___________________] // [___________SSSSSSSS_________________] // [_____SSSSSSSS_______________________] - static inline void loop_and_tail_backward(Ptr dst, CPtr src, size_t count) { + LIBC_INLINE static void loop_and_tail_backward(Ptr dst, CPtr src, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); const auto head_value = T::load(src); ptrdiff_t offset = count - Size; diff --git a/libc/src/string/memory_utils/op_x86.h b/libc/src/string/memory_utils/op_x86.h index b2355d9555eb5..dd96e9c9aec1a 100644 --- a/libc/src/string/memory_utils/op_x86.h +++ b/libc/src/string/memory_utils/op_x86.h @@ -64,7 +64,7 @@ struct Memcpy { // - BlockBcmp is the function that implements the bcmp logic. template struct BcmpImpl { static constexpr size_t SIZE = Size; - static inline BcmpReturnType block(CPtr p1, CPtr p2) { + LIBC_INLINE static BcmpReturnType block(CPtr p1, CPtr p2) { if constexpr (Size == BlockSize) { return BlockBcmp(p1, p2); } else if constexpr (Size % BlockSize == 0) { @@ -77,15 +77,16 @@ template struct BcmpImpl { return BcmpReturnType::ZERO(); } - static inline BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { return block(p1 + count - Size, p2 + count - Size); } - static inline BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { return block(p1, p2) | tail(p1, p2, count); } - static inline BcmpReturnType loop_and_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static BcmpReturnType loop_and_tail(CPtr p1, CPtr p2, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -98,7 +99,7 @@ template struct BcmpImpl { }; namespace sse2 { -static inline BcmpReturnType bcmp16(CPtr p1, CPtr p2) { +LIBC_INLINE BcmpReturnType bcmp16(CPtr p1, CPtr p2) { #if defined(__SSE2__) using T = char __attribute__((__vector_size__(16))); // A mask indicating which bytes differ after loading 16 bytes from p1 and p2. @@ -115,7 +116,7 @@ template using Bcmp = BcmpImpl; } // namespace sse2 namespace avx2 { -static inline BcmpReturnType bcmp32(CPtr p1, CPtr p2) { +LIBC_INLINE BcmpReturnType bcmp32(CPtr p1, CPtr p2) { #if defined(__AVX2__) using T = char __attribute__((__vector_size__(32))); // A mask indicating which bytes differ after loading 32 bytes from p1 and p2. @@ -134,7 +135,7 @@ template using Bcmp = BcmpImpl; } // namespace avx2 namespace avx512bw { -static inline BcmpReturnType bcmp64(CPtr p1, CPtr p2) { +LIBC_INLINE BcmpReturnType bcmp64(CPtr p1, CPtr p2) { #if defined(__AVX512BW__) using T = char __attribute__((__vector_size__(64))); // A mask indicating which bytes differ after loading 64 bytes from p1 and p2. @@ -154,8 +155,8 @@ template using Bcmp = BcmpImpl; // Assuming that the mask is non zero, the index of the first mismatching byte // is the number of trailing zeros in the mask. Trailing zeros and not leading // zeros because the x86 architecture is little endian. -static inline MemcmpReturnType char_diff_no_zero(CPtr p1, CPtr p2, - uint64_t mask) { +LIBC_INLINE MemcmpReturnType char_diff_no_zero(CPtr p1, CPtr p2, + uint64_t mask) { const size_t diff_index = __builtin_ctzll(mask); const int16_t ca = cpp::to_integer(p1[diff_index]); const int16_t cb = cpp::to_integer(p2[diff_index]); @@ -174,7 +175,7 @@ static inline MemcmpReturnType char_diff_no_zero(CPtr p1, CPtr p2, template struct MemcmpImpl { static constexpr size_t SIZE = Size; - static inline MemcmpReturnType block(CPtr p1, CPtr p2) { + LIBC_INLINE static MemcmpReturnType block(CPtr p1, CPtr p2) { if constexpr (Size == BlockSize) { return BlockMemcmp(p1, p2); } else if constexpr (Size % BlockSize == 0) { @@ -187,17 +188,19 @@ struct MemcmpImpl { return MemcmpReturnType::ZERO(); } - static inline MemcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static MemcmpReturnType tail(CPtr p1, CPtr p2, size_t count) { return block(p1 + count - Size, p2 + count - Size); } - static inline MemcmpReturnType head_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static MemcmpReturnType head_tail(CPtr p1, CPtr p2, + size_t count) { if (auto value = block(p1, p2)) return value; return tail(p1, p2, count); } - static inline MemcmpReturnType loop_and_tail(CPtr p1, CPtr p2, size_t count) { + LIBC_INLINE static MemcmpReturnType loop_and_tail(CPtr p1, CPtr p2, + size_t count) { static_assert(Size > 1, "a loop of size 1 does not need tail"); size_t offset = 0; do { @@ -210,7 +213,7 @@ struct MemcmpImpl { }; namespace sse2 { -static inline MemcmpReturnType memcmp16(CPtr p1, CPtr p2) { +LIBC_INLINE MemcmpReturnType memcmp16(CPtr p1, CPtr p2) { #if defined(__SSE2__) using T = char __attribute__((__vector_size__(16))); // A mask indicating which bytes differ after loading 16 bytes from p1 and p2. @@ -228,7 +231,7 @@ template using Memcmp = MemcmpImpl; } // namespace sse2 namespace avx2 { -static inline MemcmpReturnType memcmp32(CPtr p1, CPtr p2) { +LIBC_INLINE MemcmpReturnType memcmp32(CPtr p1, CPtr p2) { #if defined(__AVX2__) using T = char __attribute__((__vector_size__(32))); // A mask indicating which bytes differ after loading 32 bytes from p1 and p2. @@ -246,7 +249,7 @@ template using Memcmp = MemcmpImpl; } // namespace avx2 namespace avx512bw { -static inline MemcmpReturnType memcmp64(CPtr p1, CPtr p2) { +LIBC_INLINE MemcmpReturnType memcmp64(CPtr p1, CPtr p2) { #if defined(__AVX512BW__) using T = char __attribute__((__vector_size__(64))); // A mask indicating which bytes differ after loading 64 bytes from p1 and p2. diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h index 6abc85d390dd6..8e5ddd6723cea 100644 --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -12,6 +12,7 @@ #include "src/__support/CPP/bit.h" #include "src/__support/CPP/cstddef.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/common.h" #include "src/__support/compiler_features.h" #include // size_t @@ -91,8 +92,8 @@ template static T *assume_aligned(T *ptr) { // Performs a constant count copy. template -static inline void memcpy_inline(void *__restrict dst, - const void *__restrict src) { +LIBC_INLINE void memcpy_inline(void *__restrict dst, + const void *__restrict src) { #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE __builtin_memcpy_inline(dst, src, Size); #else @@ -130,7 +131,7 @@ template struct StrictIntegralType { } // Helper to get the zero value. - static inline constexpr StrictIntegralType ZERO() { return {T(0)}; } + LIBC_INLINE static constexpr StrictIntegralType ZERO() { return {T(0)}; } private: T value; @@ -141,22 +142,22 @@ using BcmpReturnType = StrictIntegralType; // Loads bytes from memory (possibly unaligned) and materializes them as // type. -template static inline T load(CPtr ptr) { +template LIBC_INLINE T load(CPtr ptr) { T Out; memcpy_inline(&Out, ptr); return Out; } // Stores a value of type T in memory (possibly unaligned). -template static inline void store(Ptr ptr, T value) { +template LIBC_INLINE void store(Ptr ptr, T value) { memcpy_inline(ptr, &value); } // Advances the pointers p1 and p2 by offset bytes and decrease count by the // same amount. template -static inline void adjust(ptrdiff_t offset, T1 *__restrict &p1, - T2 *__restrict &p2, size_t &count) { +LIBC_INLINE void adjust(ptrdiff_t offset, T1 *__restrict &p1, + T2 *__restrict &p2, size_t &count) { p1 += offset; p2 += offset; count -= offset; diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h index 24be2ab889b54..c6f0acb6949fe 100644 --- a/libc/src/string/string_utils.h +++ b/libc/src/string/string_utils.h @@ -58,7 +58,7 @@ template constexpr bool has_zeroes(Word block) { } template -static inline size_t string_length_wide_read(const char *src) { +LIBC_INLINE size_t string_length_wide_read(const char *src) { const char *char_ptr = src; // Step 1: read 1 byte at a time to align to block size for (; reinterpret_cast(char_ptr) % sizeof(Word) != 0; @@ -78,7 +78,7 @@ static inline size_t string_length_wide_read(const char *src) { return char_ptr - src; } -static inline size_t string_length_byte_read(const char *src) { +LIBC_INLINE size_t string_length_byte_read(const char *src) { size_t length; for (length = 0; *src; ++src, ++length) ; @@ -87,7 +87,7 @@ static inline size_t string_length_byte_read(const char *src) { // Returns the length of a string, denoted by the first occurrence // of a null terminator. -static inline size_t string_length(const char *src) { +LIBC_INLINE size_t string_length(const char *src) { #ifdef LIBC_UNSAFE_STRING_WIDE_READ // Unsigned int is the default size for most processors, and on x86-64 it // performs better than larger sizes when the src pointer can't be assumed to @@ -100,8 +100,8 @@ static inline size_t string_length(const char *src) { } template -static inline void *find_first_character_wide_read(const unsigned char *src, - unsigned char ch, size_t n) { +LIBC_INLINE void *find_first_character_wide_read(const unsigned char *src, + unsigned char ch, size_t n) { const unsigned char *char_ptr = src; size_t cur = 0; @@ -132,8 +132,8 @@ static inline void *find_first_character_wide_read(const unsigned char *src, return const_cast(char_ptr); } -static inline void *find_first_character_byte_read(const unsigned char *src, - unsigned char ch, size_t n) { +LIBC_INLINE void *find_first_character_byte_read(const unsigned char *src, + unsigned char ch, size_t n) { for (; n && *src != ch; --n, ++src) ; return n ? const_cast(src) : nullptr; @@ -141,8 +141,8 @@ static inline void *find_first_character_byte_read(const unsigned char *src, // Returns the first occurrence of 'ch' within the first 'n' characters of // 'src'. If 'ch' is not found, returns nullptr. -static inline void *find_first_character(const unsigned char *src, - unsigned char ch, size_t max_strlen) { +LIBC_INLINE void *find_first_character(const unsigned char *src, + unsigned char ch, size_t max_strlen) { #ifdef LIBC_UNSAFE_STRING_WIDE_READ // If the maximum size of the string is small, the overhead of aligning to a // word boundary and generating a bitmask of the appropriate size may be @@ -161,7 +161,7 @@ static inline void *find_first_character(const unsigned char *src, // Returns the maximum length span that contains only characters not found in // 'segment'. If no characters are found, returns the length of 'src'. -static inline size_t complementary_span(const char *src, const char *segment) { +LIBC_INLINE size_t complementary_span(const char *src, const char *segment) { const char *initial = src; cpp::bitset<256> bitset; @@ -181,9 +181,9 @@ static inline size_t complementary_span(const char *src, const char *segment) { // is found is then stored within 'context' for subsequent calls. Subsequent // calls will use 'context' when a nullptr is passed in for 'src'. Once the null // terminating character is reached, returns a nullptr. -static inline char *string_token(char *__restrict src, - const char *__restrict delimiter_string, - char **__restrict saveptr) { +LIBC_INLINE char *string_token(char *__restrict src, + const char *__restrict delimiter_string, + char **__restrict saveptr) { // Return nullptr immediately if both src AND saveptr are nullptr if (unlikely(src == nullptr && ((src = *saveptr) == nullptr))) return nullptr; @@ -210,8 +210,8 @@ static inline char *string_token(char *__restrict src, return token; } -static inline size_t strlcpy(char *__restrict dst, const char *__restrict src, - size_t size) { +LIBC_INLINE size_t strlcpy(char *__restrict dst, const char *__restrict src, + size_t size) { size_t len = internal::string_length(src); if (!size) return len; diff --git a/libc/src/time/time_utils.h b/libc/src/time/time_utils.h index 62a347f61e8ef..a4d51d8042db5 100644 --- a/libc/src/time/time_utils.h +++ b/libc/src/time/time_utils.h @@ -13,6 +13,7 @@ #include "include/errno.h" +#include "src/__support/common.h" #include "src/errno/llvmlibc_errno.h" #include "src/time/mktime.h" @@ -68,15 +69,15 @@ struct TimeConstants { extern int64_t update_from_seconds(int64_t total_seconds, struct tm *tm); // POSIX.1-2017 requires this. -static inline time_t out_of_range() { +LIBC_INLINE time_t out_of_range() { llvmlibc_errno = EOVERFLOW; return static_cast(-1); } -static inline void invalid_value() { llvmlibc_errno = EINVAL; } +LIBC_INLINE void invalid_value() { llvmlibc_errno = EINVAL; } -static inline char *asctime(const struct tm *timeptr, char *buffer, - size_t bufferLength) { +LIBC_INLINE char *asctime(const struct tm *timeptr, char *buffer, + size_t bufferLength) { if (timeptr == nullptr || buffer == nullptr) { invalid_value(); return nullptr; @@ -113,8 +114,7 @@ static inline char *asctime(const struct tm *timeptr, char *buffer, return buffer; } -static inline struct tm *gmtime_internal(const time_t *timer, - struct tm *result) { +LIBC_INLINE struct tm *gmtime_internal(const time_t *timer, struct tm *result) { int64_t seconds = *timer; // Update the tm structure's year, month, day, etc. from seconds. if (update_from_seconds(seconds, result) < 0) { diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 3d2a8d59eb542..be4dafdc3b8b5 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -699,6 +699,7 @@ libc_support_library( "src/math/generic/range_reduction_fma.h", ], deps = [ + ":__support_common", ":__support_fputil_fma", ":__support_fputil_fp_bits", ":__support_fputil_multiply_add",