diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake index f4e2a62d14b31..4e9a9b66a63a7 100644 --- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake +++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake @@ -81,8 +81,9 @@ function(_get_compile_options_from_config output_var) list(APPEND config_options "-DLIBC_QSORT_IMPL=${LIBC_CONF_QSORT_IMPL}") endif() - list(APPEND config_options "-DLIBC_COPT_STRING_LENGTH_IMPL=${LIBC_CONF_STRING_LENGTH_IMPL}") - list(APPEND config_options "-DLIBC_COPT_FIND_FIRST_CHARACTER_IMPL=${LIBC_CONF_FIND_FIRST_CHARACTER_IMPL}") + if(LIBC_CONF_STRING_UNSAFE_WIDE_READ) + list(APPEND config_options "-DLIBC_COPT_STRING_UNSAFE_WIDE_READ") + endif() if(LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING) list(APPEND config_options "-DLIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING") diff --git a/libc/config/config.json b/libc/config/config.json index f0ab3b9cce2e9..a7844e4fe2dd1 100644 --- a/libc/config/config.json +++ b/libc/config/config.json @@ -40,7 +40,6 @@ "value": false, "doc": "Use an alternative printf float implementation based on 320-bit floats" }, - "LIBC_CONF_PRINTF_DISABLE_FIXED_POINT": { "value": false, "doc": "Disable printing fixed point values in printf and friends." @@ -65,13 +64,9 @@ } }, "string": { - "LIBC_CONF_STRING_LENGTH_IMPL": { - "value": "element", - "doc": "Selects the implementation for string-length: 'element', 'word', 'clang_vector', or 'arch_vector'." - }, - "LIBC_CONF_FIND_FIRST_CHARACTER_IMPL": { - "value": "element", - "doc": "Selects the implementation for find-first-character-related functions: 'element', 'word', 'clang_vector', or 'arch_vector'." + "LIBC_CONF_STRING_UNSAFE_WIDE_READ": { + "value": false, + "doc": "Read more than a byte at a time to perform byte-string operations like strlen." }, "LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING": { "value": false, diff --git a/libc/config/linux/arm/config.json b/libc/config/linux/arm/config.json index caa16744d389f..e7ad4544b104d 100644 --- a/libc/config/linux/arm/config.json +++ b/libc/config/linux/arm/config.json @@ -1,10 +1,7 @@ { "string": { - "LIBC_CONF_STRING_LENGTH_IMPL": { - "value": "element" - } - "LIBC_CONF_FIND_FIRST_CHARACTER_IMPL": { - "value": "element" + "LIBC_CONF_STRING_UNSAFE_WIDE_READ": { + "value": false } } } diff --git a/libc/config/linux/config.json b/libc/config/linux/config.json index 8e7db248dc1bd..30e8b2cdadabe 100644 --- a/libc/config/linux/config.json +++ b/libc/config/linux/config.json @@ -1,10 +1,7 @@ { "string": { - "LIBC_CONF_STRING_LENGTH_IMPL": { - "value": "clang_vector", - }, - "LIBC_CONF_FIND_FIRST_CHARACTER_IMPL": { - "value": "word", + "LIBC_CONF_STRING_UNSAFE_WIDE_READ": { + "value": true } } } diff --git a/libc/config/linux/riscv/config.json b/libc/config/linux/riscv/config.json index caa16744d389f..e7ad4544b104d 100644 --- a/libc/config/linux/riscv/config.json +++ b/libc/config/linux/riscv/config.json @@ -1,10 +1,7 @@ { "string": { - "LIBC_CONF_STRING_LENGTH_IMPL": { - "value": "element" - } - "LIBC_CONF_FIND_FIRST_CHARACTER_IMPL": { - "value": "element" + "LIBC_CONF_STRING_UNSAFE_WIDE_READ": { + "value": false } } } diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst index 43d3c0ec06d3b..362e293a4b714 100644 --- a/libc/docs/configure.rst +++ b/libc/docs/configure.rst @@ -58,9 +58,8 @@ to learn about the defaults for your platform and target. * **"setjmp" options** - ``LIBC_CONF_SETJMP_AARCH64_RESTORE_PLATFORM_REGISTER``: Make setjmp save the value of x18, and longjmp restore it. The AArch64 ABI delegates this register to platform ABIs, which can choose whether to make it caller-saved. * **"string" options** - - ``LIBC_CONF_FIND_FIRST_CHARACTER_IMPL``: Selects the implementation for find-first-character-related functions: 'element', 'word', 'clang_vector', or 'arch_vector'. - ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled. - - ``LIBC_CONF_STRING_LENGTH_IMPL``: Selects the implementation for string-length: 'element', 'word', 'clang_vector', or 'arch_vector'. + - ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen. * **"threads" options** - ``LIBC_CONF_THREAD_MODE``: The implementation used for Mutex, acceptable values are LIBC_THREAD_MODE_PLATFORM, LIBC_THREAD_MODE_SINGLE, and LIBC_THREAD_MODE_EXTERNAL. * **"time" options** diff --git a/libc/src/string/memory_utils/aarch64/inline_strlen.h b/libc/src/string/memory_utils/aarch64/inline_strlen.h index 87f6cb8cf9bd5..eafaca9776a42 100644 --- a/libc/src/string/memory_utils/aarch64/inline_strlen.h +++ b/libc/src/string/memory_utils/aarch64/inline_strlen.h @@ -15,7 +15,7 @@ #include #include // size_t namespace LIBC_NAMESPACE_DECL { -namespace internal::neon { +namespace neon { [[maybe_unused]] LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE static size_t string_length(const char *src) { using Vector __attribute__((may_alias)) = uint8x8_t; @@ -43,7 +43,7 @@ string_length(const char *src) { (cpp::countr_zero(cmp) >> 3)); } } -} // namespace internal::neon +} // namespace neon } // namespace LIBC_NAMESPACE_DECL #endif // __ARM_NEON @@ -51,7 +51,7 @@ string_length(const char *src) { #include "src/__support/macros/optimization.h" #include namespace LIBC_NAMESPACE_DECL { -namespace internal::sve { +namespace sve { [[maybe_unused]] LIBC_INLINE static size_t string_length(const char *src) { const uint8_t *ptr = reinterpret_cast(src); // Initialize the first-fault register to all true @@ -92,19 +92,15 @@ namespace internal::sve { len += svcntp_b8(all_true, before_zero); return len; } -} // namespace internal::sve +} // namespace sve } // namespace LIBC_NAMESPACE_DECL #endif // LIBC_TARGET_CPU_HAS_SVE namespace LIBC_NAMESPACE_DECL { -namespace internal::arch_vector { -[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) { #ifdef LIBC_TARGET_CPU_HAS_SVE - return sve::string_length(src); +namespace string_length_impl = sve; #elif defined(__ARM_NEON) - return neon::string_length(src); +namespace string_length_impl = neon; #endif -} -} // namespace internal::arch_vector } // namespace LIBC_NAMESPACE_DECL #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H diff --git a/libc/src/string/memory_utils/generic/inline_strlen.h b/libc/src/string/memory_utils/generic/inline_strlen.h index 7a565b36617ed..69700e801bcea 100644 --- a/libc/src/string/memory_utils/generic/inline_strlen.h +++ b/libc/src/string/memory_utils/generic/inline_strlen.h @@ -14,7 +14,7 @@ #include "src/__support/common.h" namespace LIBC_NAMESPACE_DECL { -namespace clang_vector { +namespace internal { // Exploit the underlying integer representation to do a variable shift. LIBC_INLINE constexpr cpp::simd_mask shift_mask(cpp::simd_mask m, @@ -46,8 +46,9 @@ LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE size_t string_length(const char *src) { cpp::find_first_set(mask); } } -} // namespace clang_vector +} // namespace internal +namespace string_length_impl = internal; } // namespace LIBC_NAMESPACE_DECL #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_INLINE_STRLEN_H diff --git a/libc/src/string/memory_utils/x86_64/inline_strlen.h b/libc/src/string/memory_utils/x86_64/inline_strlen.h index 07b4a470f0d77..9e10d58363393 100644 --- a/libc/src/string/memory_utils/x86_64/inline_strlen.h +++ b/libc/src/string/memory_utils/x86_64/inline_strlen.h @@ -15,8 +15,7 @@ namespace LIBC_NAMESPACE_DECL { -namespace internal::arch_vector { - +namespace string_length_internal { // Return a bit-mask with the nth bit set if the nth-byte in block_ptr is zero. template LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE static Mask @@ -93,18 +92,15 @@ namespace avx512 { } } // namespace avx512 #endif +} // namespace string_length_internal -[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) { #if defined(__AVX512F__) - return avx512::string_length(src); +namespace string_length_impl = string_length_internal::avx512; #elif defined(__AVX2__) - return avx2::string_length(src); +namespace string_length_impl = string_length_internal::avx2; #else - return sse2::string_length(src); +namespace string_length_impl = string_length_internal::sse2; #endif -} - -} // namespace internal::arch_vector } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/string/string_length.h b/libc/src/string/string_length.h deleted file mode 100644 index 3d72dc606b82e..0000000000000 --- a/libc/src/string/string_length.h +++ /dev/null @@ -1,213 +0,0 @@ -//===-- String Length -------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Basic implementation and dispatch mechanism for performance-sensitive string- -// related code. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_STRING_STRING_LENGTH_H -#define LLVM_LIBC_SRC_STRING_STRING_LENGTH_H - -#include "hdr/limits_macros.h" -#include "hdr/stdint_proxy.h" // uintptr_t -#include "hdr/types/size_t.h" -#include "src/__support/CPP/type_traits.h" // cpp::is_same_v - -#if LIBC_HAS_VECTOR_TYPE -#include "src/string/memory_utils/generic/inline_strlen.h" -#endif -#if defined(LIBC_TARGET_ARCH_IS_X86) -#include "src/string/memory_utils/x86_64/inline_strlen.h" -#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) -#include "src/string/memory_utils/aarch64/inline_strlen.h" -#endif - -// Set sensible defaults -#ifndef LIBC_COPT_STRING_LENGTH_IMPL -#define LIBC_COPT_STRING_LENGTH_IMPL element -#endif -#ifndef LIBC_COPT_FIND_FIRST_CHARACTER_IMPL -#define LIBC_COPT_STRING_LENGTH_IMPL element -#endif - -namespace LIBC_NAMESPACE_DECL { -namespace internal { - -#if !LIBC_HAS_VECTOR_TYPE -// Forward any clang vector impls to architecture specific ones -namespace arch_vector {} -namespace clang_vector = arch_vector; -#endif - -namespace element { -// Element-by-element (usually a byte, but wider for wchar) implementations of -// functions that search for data. Slow, but easy to understand and analyze. - -// Returns the length of a string, denoted by the first occurrence -// of a null terminator. -LIBC_INLINE size_t string_length(const char *src) { - size_t length; - for (length = 0; *src; ++src, ++length) - ; - return length; -} - -template LIBC_INLINE size_t string_length_element(const T *src) { - size_t length; - for (length = 0; *src; ++src, ++length) - ; - return length; -} - -LIBC_INLINE void *find_first_character(const unsigned char *src, - unsigned char ch, size_t n) { - for (; n && *src != ch; --n, ++src) - ; - return n ? const_cast(src) : nullptr; -} -} // namespace element - -namespace word { -// Non-vector, implementations of functions that search for data by reading from -// memory word-by-word. - -template LIBC_INLINE constexpr Word repeat_byte(Word byte) { - static_assert(CHAR_BIT == 8, "repeat_byte assumes a byte is 8 bits."); - constexpr size_t BITS_IN_BYTE = CHAR_BIT; - constexpr size_t BYTE_MASK = 0xff; - Word result = 0; - byte = byte & BYTE_MASK; - for (size_t i = 0; i < sizeof(Word); ++i) - result = (result << BITS_IN_BYTE) | byte; - return result; -} - -// The goal of this function is to take in a block of arbitrary size and return -// if it has any bytes equal to zero without branching. This is done by -// transforming the block such that zero bytes become non-zero and non-zero -// bytes become zero. -// The first transformation relies on the properties of carrying in arithmetic -// subtraction. Specifically, if 0x01 is subtracted from a byte that is 0x00, -// then the result for that byte must be equal to 0xff (or 0xfe if the next byte -// needs a carry as well). -// The next transformation is a simple mask. All zero bytes will have the high -// bit set after the subtraction, so each byte is masked with 0x80. This narrows -// the set of bytes that result in a non-zero value to only zero bytes and bytes -// with the high bit and any other bit set. -// The final transformation masks the result of the previous transformations -// with the inverse of the original byte. This means that any byte that had the -// high bit set will no longer have it set, narrowing the list of bytes which -// result in non-zero values to just the zero byte. -template LIBC_INLINE constexpr bool has_zeroes(Word block) { - constexpr unsigned int LOW_BITS = repeat_byte(0x01); - constexpr Word HIGH_BITS = repeat_byte(0x80); - Word subtracted = block - LOW_BITS; - Word inverted = ~block; - return (subtracted & inverted & HIGH_BITS) != 0; -} - -// Unsigned int is the default size for most processors, and on x86-64 it -// performs better than larger sizes when the src pointer can't be assumed to -// be aligned to a word boundary, so it's the size we use for reading the -// string a block at a time. - -LIBC_INLINE size_t string_length(const char *src) { - using Word = unsigned int; - const char *char_ptr = src; - // Step 1: read 1 byte at a time to align to block size - for (; reinterpret_cast(char_ptr) % sizeof(Word) != 0; - ++char_ptr) { - if (*char_ptr == '\0') - return static_cast(char_ptr - src); - } - // Step 2: read blocks - for (const Word *block_ptr = reinterpret_cast(char_ptr); - !has_zeroes(*block_ptr); ++block_ptr) { - char_ptr = reinterpret_cast(block_ptr); - } - // Step 3: find the zero in the block - for (; *char_ptr != '\0'; ++char_ptr) { - ; - } - return static_cast(char_ptr - src); -} - -LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE void * -find_first_character(const unsigned char *src, unsigned char ch, - size_t max_strlen = cpp::numeric_limits::max()) { - using Word = unsigned int; - const unsigned char *char_ptr = src; - size_t cur = 0; - - // If the maximum size of the string is small, the overhead of aligning to a - // word boundary and generating a bitmask of the appropriate size may be - // greater than the gains from reading larger chunks. Based on some testing, - // the crossover point between when it's faster to just read bytewise and read - // blocks is somewhere between 16 and 32, so 4 times the size of the block - // should be in that range. - if (max_strlen < (sizeof(Word) * 4)) { - return element::find_first_character(src, ch, max_strlen); - } - size_t n = max_strlen; - // Step 1: read 1 byte at a time to align to block size - for (; cur < n && reinterpret_cast(char_ptr) % sizeof(Word) != 0; - ++cur, ++char_ptr) { - if (*char_ptr == ch) - return const_cast(char_ptr); - } - - const Word ch_mask = repeat_byte(ch); - - // Step 2: read blocks - const Word *block_ptr = reinterpret_cast(char_ptr); - for (; cur < n && !has_zeroes((*block_ptr) ^ ch_mask); - cur += sizeof(Word), ++block_ptr) - ; - char_ptr = reinterpret_cast(block_ptr); - - // Step 3: find the match in the block - for (; cur < n && *char_ptr != ch; ++cur, ++char_ptr) { - ; - } - - if (cur >= n || *char_ptr != ch) - return static_cast(nullptr); - - return const_cast(char_ptr); -} - -} // namespace word - -// Dispatch mechanism for implementations of performance-sensitive -// functions. Always measure, but generally from lower- to higher-performance -// order: -// -// 1. element - read char-by-char or wchar-by-wchar -// 3. word - read word-by-word -// 3. clang_vector - read using clang's internal vector types -// 4. arch_vector - hand-coded per architecture. Possibly in asm, or with -// intrinsics. -// -// The called implemenation is chosen at build-time by setting -// LIBC_CONF_{FUNC}_IMPL in config.json -static constexpr auto &string_length_impl = - LIBC_COPT_STRING_LENGTH_IMPL::string_length; -static constexpr auto &find_first_character_impl = - LIBC_COPT_FIND_FIRST_CHARACTER_IMPL::find_first_character; - -template LIBC_INLINE size_t string_length(const T *src) { - if constexpr (cpp::is_same_v) - return string_length_impl(src); - return element::string_length_element(src); -} - -} // namespace internal -} // namespace LIBC_NAMESPACE_DECL - -#endif // LLVM_LIBC_SRC_STRING_STRING_LENGTH_H diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h index b0144e01a9006..cbce62ead0328 100644 --- a/libc/src/string/string_utils.h +++ b/libc/src/string/string_utils.h @@ -14,17 +14,172 @@ #ifndef LLVM_LIBC_SRC_STRING_STRING_UTILS_H #define LLVM_LIBC_SRC_STRING_STRING_UTILS_H +#include "hdr/limits_macros.h" +#include "hdr/stdint_proxy.h" // uintptr_t #include "hdr/types/size_t.h" #include "src/__support/CPP/bitset.h" +#include "src/__support/CPP/type_traits.h" // cpp::is_same_v #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY #include "src/string/memory_utils/inline_memcpy.h" -#include "src/string/string_length.h" + +#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ) +#if LIBC_HAS_VECTOR_TYPE +#include "src/string/memory_utils/generic/inline_strlen.h" +#elif defined(LIBC_TARGET_ARCH_IS_X86) +#include "src/string/memory_utils/x86_64/inline_strlen.h" +#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_NEON) +#include "src/string/memory_utils/aarch64/inline_strlen.h" +#else +namespace string_length_impl = LIBC_NAMESPACE::wide_read; +#endif +#endif // defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ) namespace LIBC_NAMESPACE_DECL { namespace internal { +template LIBC_INLINE constexpr Word repeat_byte(Word byte) { + static_assert(CHAR_BIT == 8, "repeat_byte assumes a byte is 8 bits."); + constexpr size_t BITS_IN_BYTE = CHAR_BIT; + constexpr size_t BYTE_MASK = 0xff; + Word result = 0; + byte = byte & BYTE_MASK; + for (size_t i = 0; i < sizeof(Word); ++i) + result = (result << BITS_IN_BYTE) | byte; + return result; +} + +// The goal of this function is to take in a block of arbitrary size and return +// if it has any bytes equal to zero without branching. This is done by +// transforming the block such that zero bytes become non-zero and non-zero +// bytes become zero. +// The first transformation relies on the properties of carrying in arithmetic +// subtraction. Specifically, if 0x01 is subtracted from a byte that is 0x00, +// then the result for that byte must be equal to 0xff (or 0xfe if the next byte +// needs a carry as well). +// The next transformation is a simple mask. All zero bytes will have the high +// bit set after the subtraction, so each byte is masked with 0x80. This narrows +// the set of bytes that result in a non-zero value to only zero bytes and bytes +// with the high bit and any other bit set. +// The final transformation masks the result of the previous transformations +// with the inverse of the original byte. This means that any byte that had the +// high bit set will no longer have it set, narrowing the list of bytes which +// result in non-zero values to just the zero byte. +template LIBC_INLINE constexpr bool has_zeroes(Word block) { + constexpr unsigned int LOW_BITS = repeat_byte(0x01); + constexpr Word HIGH_BITS = repeat_byte(0x80); + Word subtracted = block - LOW_BITS; + Word inverted = ~block; + return (subtracted & inverted & HIGH_BITS) != 0; +} + +template +LIBC_INLINE size_t string_length_wide_read(const char *src) { + const char *char_ptr = src; + // Step 1: read 1 byte at a time to align to block size + for (; reinterpret_cast(char_ptr) % sizeof(Word) != 0; + ++char_ptr) { + if (*char_ptr == '\0') + return static_cast(char_ptr - src); + } + // Step 2: read blocks + for (const Word *block_ptr = reinterpret_cast(char_ptr); + !has_zeroes(*block_ptr); ++block_ptr) { + char_ptr = reinterpret_cast(block_ptr); + } + // Step 3: find the zero in the block + for (; *char_ptr != '\0'; ++char_ptr) { + ; + } + return static_cast(char_ptr - src); +} + +namespace wide_read { +LIBC_INLINE size_t string_length(const char *src) { + // Unsigned int is the default size for most processors, and on x86-64 it + // performs better than larger sizes when the src pointer can't be assumed to + // be aligned to a word boundary, so it's the size we use for reading the + // string a block at a time. + return string_length_wide_read(src); +} + +} // namespace wide_read + +// Returns the length of a string, denoted by the first occurrence +// of a null terminator. +template LIBC_INLINE size_t string_length(const T *src) { +#ifdef LIBC_COPT_STRING_UNSAFE_WIDE_READ + if constexpr (cpp::is_same_v) + return string_length_impl::string_length(src); +#endif + size_t length; + for (length = 0; *src; ++src, ++length) + ; + return length; +} + +template +LIBC_NO_SANITIZE_OOB_ACCESS LIBC_INLINE void * +find_first_character_wide_read(const unsigned char *src, unsigned char ch, + size_t n) { + const unsigned char *char_ptr = src; + size_t cur = 0; + + // Step 1: read 1 byte at a time to align to block size + for (; cur < n && reinterpret_cast(char_ptr) % sizeof(Word) != 0; + ++cur, ++char_ptr) { + if (*char_ptr == ch) + return const_cast(char_ptr); + } + + const Word ch_mask = repeat_byte(ch); + + // Step 2: read blocks + const Word *block_ptr = reinterpret_cast(char_ptr); + for (; cur < n && !has_zeroes((*block_ptr) ^ ch_mask); + cur += sizeof(Word), ++block_ptr) + ; + char_ptr = reinterpret_cast(block_ptr); + + // Step 3: find the match in the block + for (; cur < n && *char_ptr != ch; ++cur, ++char_ptr) { + ; + } + + if (cur >= n || *char_ptr != ch) + return static_cast(nullptr); + + return const_cast(char_ptr); +} + +LIBC_INLINE void *find_first_character_byte_read(const unsigned char *src, + unsigned char ch, size_t n) { + for (; n && *src != ch; --n, ++src) + ; + return n ? const_cast(src) : nullptr; +} + +// Returns the first occurrence of 'ch' within the first 'n' characters of +// 'src'. If 'ch' is not found, returns nullptr. +LIBC_INLINE void *find_first_character(const unsigned char *src, + unsigned char ch, size_t max_strlen) { +#ifdef LIBC_COPT_STRING_UNSAFE_WIDE_READ + // If the maximum size of the string is small, the overhead of aligning to a + // word boundary and generating a bitmask of the appropriate size may be + // greater than the gains from reading larger chunks. Based on some testing, + // the crossover point between when it's faster to just read bytewise and read + // blocks is somewhere between 16 and 32, so 4 times the size of the block + // should be in that range. + // Unsigned int is used for the same reason as in strlen. + using BlockType = unsigned int; + if (max_strlen > (sizeof(BlockType) * 4)) { + return find_first_character_wide_read(src, ch, max_strlen); + } +#endif + return find_first_character_byte_read(src, ch, max_strlen); +} + // Returns the maximum length span that contains only characters not found in // 'segment'. If no characters are found, returns the length of 'src'. LIBC_INLINE size_t complementary_span(const char *src, const char *segment) { @@ -117,13 +272,6 @@ LIBC_INLINE constexpr static char *strrchr_implementation(const char *src, } } -// Returns the first occurrence of 'ch' within the first 'n' characters of -// 'src'. If 'ch' is not found, returns nullptr. -LIBC_INLINE void *find_first_character(const unsigned char *src, - unsigned char ch, size_t max_strlen) { - return find_first_character_impl(src, ch, max_strlen); -} - } // namespace internal } // namespace LIBC_NAMESPACE_DECL diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 3f19a439d41f4..a3a0747a71003 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -5452,10 +5452,7 @@ libc_support_library( libc_support_library( name = "string_utils", - hdrs = [ - "src/string/string_length.h", - "src/string/string_utils.h", - ], + hdrs = ["src/string/string_utils.h"], deps = [ ":__support_common", ":__support_cpp_bitset", diff --git a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl index 1c2b65ea20113..259d4d292fcf4 100644 --- a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl +++ b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl @@ -39,8 +39,7 @@ LIBC_CONFIGURE_OPTIONS = [ # "LIBC_COPT_SCANF_DISABLE_FLOAT", # "LIBC_COPT_SCANF_DISABLE_INDEX_MODE", "LIBC_COPT_STDIO_USE_SYSTEM_FILE", - "LIBC_COPT_STRING_LENGTH_IMPL=clang_vector", - "LIBC_COPT_FIND_FIRST_CHARACTER_IMPL=word", + "LIBC_COPT_STRING_UNSAFE_WIDE_READ", # "LIBC_COPT_STRTOFLOAT_DISABLE_CLINGER_FAST_PATH", # "LIBC_COPT_STRTOFLOAT_DISABLE_EISEL_LEMIRE", # "LIBC_COPT_STRTOFLOAT_DISABLE_SIMPLE_DECIMAL_CONVERSION",