diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt index 371b4010a4dc8..5210958fe9baa 100644 --- a/libc/src/string/memory_utils/CMakeLists.txt +++ b/libc/src/string/memory_utils/CMakeLists.txt @@ -6,6 +6,8 @@ add_header_library( bzero_implementations.h memcmp_implementations.h memcpy_implementations.h + aarch64/memcpy_implementations.h + x86_64/memcpy_implementations.h memmove_implementations.h memset_implementations.h op_aarch64.h @@ -14,10 +16,10 @@ add_header_library( op_x86.h utils.h DEPS - libc.src.__support.common libc.src.__support.CPP.bit libc.src.__support.CPP.cstddef libc.src.__support.CPP.type_traits + libc.src.__support.macros.config libc.src.__support.macros.optimization ) diff --git a/libc/src/string/memory_utils/aarch64/memcpy_implementations.h b/libc/src/string/memory_utils/aarch64/memcpy_implementations.h new file mode 100644 index 0000000000000..e34eba27ece5b --- /dev/null +++ b/libc/src/string/memory_utils/aarch64/memcpy_implementations.h @@ -0,0 +1,48 @@ +//===-- Memcpy implementation for aarch64 -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H +#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H + +#include "src/__support/macros/config.h" // LIBC_INLINE +#include "src/string/memory_utils/op_builtin.h" +#include "src/string/memory_utils/utils.h" + +#include // size_t + +namespace __llvm_libc { + +[[maybe_unused]] LIBC_INLINE void +inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) { + if (count == 0) + return; + if (count == 1) + return builtin::Memcpy<1>::block(dst, src); + if (count == 2) + return builtin::Memcpy<2>::block(dst, src); + if (count == 3) + return builtin::Memcpy<3>::block(dst, src); + if (count == 4) + return builtin::Memcpy<4>::block(dst, src); + if (count < 8) + return builtin::Memcpy<4>::head_tail(dst, src, count); + if (count < 16) + return builtin::Memcpy<8>::head_tail(dst, src, count); + if (count < 32) + return builtin::Memcpy<16>::head_tail(dst, src, count); + if (count < 64) + return builtin::Memcpy<32>::head_tail(dst, src, count); + if (count < 128) + return builtin::Memcpy<64>::head_tail(dst, src, count); + builtin::Memcpy<16>::block(dst, src); + align_to_next_boundary<16, Arg::Src>(dst, src, count); + return builtin::Memcpy<64>::loop_and_tail(dst, src, count); +} + +} // namespace __llvm_libc + +#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H diff --git a/libc/src/string/memory_utils/memcpy_implementations.h b/libc/src/string/memory_utils/memcpy_implementations.h index 850e0256c7b1d..f0344b3319bb6 100644 --- a/libc/src/string/memory_utils/memcpy_implementations.h +++ b/libc/src/string/memory_utils/memcpy_implementations.h @@ -9,17 +9,20 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H -#include "src/__support/common.h" -#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY LIBC_LOOP_NOUNROLL +#include "src/__support/macros/config.h" // LIBC_INLINE +#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL #include "src/__support/macros/properties/architectures.h" -#include "src/string/memory_utils/op_aarch64.h" #include "src/string/memory_utils/op_builtin.h" -#include "src/string/memory_utils/op_generic.h" -#include "src/string/memory_utils/op_x86.h" #include "src/string/memory_utils/utils.h" #include // size_t +#if defined(LIBC_TARGET_ARCH_IS_X86) +#include "src/string/memory_utils/x86_64/memcpy_implementations.h" +#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) +#include "src/string/memory_utils/aarch64/memcpy_implementations.h" +#endif + namespace __llvm_libc { [[maybe_unused]] LIBC_INLINE void @@ -30,98 +33,6 @@ inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src, builtin::Memcpy<1>::block(dst + offset, src + offset); } -#if defined(LIBC_TARGET_ARCH_IS_X86) -[[maybe_unused]] LIBC_INLINE void -inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { - if (count == 0) - return; - if (count == 1) - return builtin::Memcpy<1>::block(dst, src); - if (count == 2) - return builtin::Memcpy<2>::block(dst, src); - if (count == 3) - return builtin::Memcpy<3>::block(dst, src); - if (count == 4) - return builtin::Memcpy<4>::block(dst, src); - if (count < 8) - return builtin::Memcpy<4>::head_tail(dst, src, count); - if (count < 16) - return builtin::Memcpy<8>::head_tail(dst, src, count); - if (count < 32) - return builtin::Memcpy<16>::head_tail(dst, src, count); - if (count < 64) - return builtin::Memcpy<32>::head_tail(dst, src, count); - if (count < 128) - return builtin::Memcpy<64>::head_tail(dst, src, count); - if (x86::kAvx && count < 256) - return builtin::Memcpy<128>::head_tail(dst, src, count); - builtin::Memcpy<32>::block(dst, src); - align_to_next_boundary<32, Arg::Dst>(dst, src, count); - static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32; - return builtin::Memcpy::loop_and_tail(dst, src, count); -} - -[[maybe_unused]] LIBC_INLINE void -inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst, - CPtr __restrict src, size_t count) { - // Whether to use rep;movsb exclusively, not at all, or only above a certain - // threshold. -#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1 -#endif - -#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB -#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead. -#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB - -#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead. -#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE - - static constexpr size_t kRepMovsbThreshold = - LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE; - if constexpr (kRepMovsbThreshold == 0) { - return x86::Memcpy::repmovsb(dst, src, count); - } else if constexpr (kRepMovsbThreshold == size_t(-1)) { - return inline_memcpy_x86(dst, src, count); - } else { - if (LIBC_UNLIKELY(count >= kRepMovsbThreshold)) - return x86::Memcpy::repmovsb(dst, src, count); - else - return inline_memcpy_x86(dst, src, count); - } -} -#endif // defined(LIBC_TARGET_ARCH_IS_X86) - -#if defined(LIBC_TARGET_ARCH_IS_AARCH64) -[[maybe_unused]] LIBC_INLINE void -inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) { - if (count == 0) - return; - if (count == 1) - return builtin::Memcpy<1>::block(dst, src); - if (count == 2) - return builtin::Memcpy<2>::block(dst, src); - if (count == 3) - return builtin::Memcpy<3>::block(dst, src); - if (count == 4) - return builtin::Memcpy<4>::block(dst, src); - if (count < 8) - return builtin::Memcpy<4>::head_tail(dst, src, count); - if (count < 16) - return builtin::Memcpy<8>::head_tail(dst, src, count); - if (count < 32) - return builtin::Memcpy<16>::head_tail(dst, src, count); - if (count < 64) - return builtin::Memcpy<32>::head_tail(dst, src, count); - if (count < 128) - return builtin::Memcpy<64>::head_tail(dst, src, count); - builtin::Memcpy<16>::block(dst, src); - align_to_next_boundary<16, Arg::Src>(dst, src, count); - return builtin::Memcpy<64>::loop_and_tail(dst, src, count); -} -#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64) - LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src, size_t count) { using namespace __llvm_libc::builtin; diff --git a/libc/src/string/memory_utils/x86_64/memcpy_implementations.h b/libc/src/string/memory_utils/x86_64/memcpy_implementations.h new file mode 100644 index 0000000000000..3844e06e01ab2 --- /dev/null +++ b/libc/src/string/memory_utils/x86_64/memcpy_implementations.h @@ -0,0 +1,84 @@ +//===-- Memcpy implementation for x86_64 ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H +#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H + +#include "src/__support/macros/config.h" // LIBC_INLINE +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#include "src/string/memory_utils/op_builtin.h" +#include "src/string/memory_utils/op_x86.h" +#include "src/string/memory_utils/utils.h" + +#include // size_t + +namespace __llvm_libc { + +[[maybe_unused]] LIBC_INLINE void +inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { + if (count == 0) + return; + if (count == 1) + return builtin::Memcpy<1>::block(dst, src); + if (count == 2) + return builtin::Memcpy<2>::block(dst, src); + if (count == 3) + return builtin::Memcpy<3>::block(dst, src); + if (count == 4) + return builtin::Memcpy<4>::block(dst, src); + if (count < 8) + return builtin::Memcpy<4>::head_tail(dst, src, count); + if (count < 16) + return builtin::Memcpy<8>::head_tail(dst, src, count); + if (count < 32) + return builtin::Memcpy<16>::head_tail(dst, src, count); + if (count < 64) + return builtin::Memcpy<32>::head_tail(dst, src, count); + if (count < 128) + return builtin::Memcpy<64>::head_tail(dst, src, count); + if (x86::kAvx && count < 256) + return builtin::Memcpy<128>::head_tail(dst, src, count); + builtin::Memcpy<32>::block(dst, src); + align_to_next_boundary<32, Arg::Dst>(dst, src, count); + static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32; + return builtin::Memcpy::loop_and_tail(dst, src, count); +} + +[[maybe_unused]] LIBC_INLINE void +inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst, + CPtr __restrict src, size_t count) { + // Whether to use rep;movsb exclusively, not at all, or only above a certain + // threshold. +#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE +#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1 +#endif + +#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB +#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead. +#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB + +#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE +#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead. +#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE + + static constexpr size_t kRepMovsbThreshold = + LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE; + if constexpr (kRepMovsbThreshold == 0) { + return x86::Memcpy::repmovsb(dst, src, count); + } else if constexpr (kRepMovsbThreshold == size_t(-1)) { + return inline_memcpy_x86(dst, src, count); + } else { + if (LIBC_UNLIKELY(count >= kRepMovsbThreshold)) + return x86::Memcpy::repmovsb(dst, src, count); + else + return inline_memcpy_x86(dst, src, count); + } +} + +} // namespace __llvm_libc + +#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index d5ba402e3dc43..c11f0be268fec 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1404,6 +1404,8 @@ libc_support_library( "src/string/memory_utils/bzero_implementations.h", "src/string/memory_utils/memcmp_implementations.h", "src/string/memory_utils/memcpy_implementations.h", + "src/string/memory_utils/aarch64/memcpy_implementations.h", + "src/string/memory_utils/x86_64/memcpy_implementations.h", "src/string/memory_utils/memmove_implementations.h", "src/string/memory_utils/memset_implementations.h", "src/string/memory_utils/strcmp_implementations.h", @@ -1418,6 +1420,7 @@ libc_support_library( ":__support_macros_attributes", ":__support_macros_config", ":__support_macros_optimization", + ":__support_macros_properties_architectures", ":__support_macros_properties_cpu_features", ":libc_root", ],