-
Notifications
You must be signed in to change notification settings - Fork 11.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[libc][NFC] Split memset implementations per platform
This is a follow up on D154800 and D154770 to make the code structure more principled and avoid too many nested #ifdef/#endif. Reviewed By: courbet Differential Revision: https://reviews.llvm.org/D155174
- Loading branch information
Showing
6 changed files
with
183 additions
and
109 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
61 changes: 61 additions & 0 deletions
61
libc/src/string/memory_utils/aarch64/memset_implementations.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| //===-- Memset implementation for aarch64 -----------------------*- C++ -*-===// | ||
| // | ||
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| // See https://llvm.org/LICENSE.txt for license information. | ||
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
| #ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMSET_IMPLEMENTATIONS_H | ||
| #define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMSET_IMPLEMENTATIONS_H | ||
|
|
||
| #include "src/__support/macros/attributes.h" // LIBC_INLINE | ||
| #include "src/string/memory_utils/op_aarch64.h" | ||
| #include "src/string/memory_utils/op_generic.h" | ||
| #include "src/string/memory_utils/utils.h" // Ptr, CPtr | ||
|
|
||
| #include <stddef.h> // size_t | ||
|
|
||
| namespace __llvm_libc { | ||
|
|
||
| [[maybe_unused]] LIBC_INLINE static void | ||
| inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) { | ||
| static_assert(aarch64::kNeon, "aarch64 supports vector types"); | ||
| using uint128_t = generic_v128; | ||
| using uint256_t = generic_v256; | ||
| using uint512_t = generic_v512; | ||
| if (count == 0) | ||
| return; | ||
| if (count <= 3) { | ||
| generic::Memset<uint8_t>::block(dst, value); | ||
| if (count > 1) | ||
| generic::Memset<uint16_t>::tail(dst, value, count); | ||
| return; | ||
| } | ||
| if (count <= 8) | ||
| return generic::Memset<uint32_t>::head_tail(dst, value, count); | ||
| if (count <= 16) | ||
| return generic::Memset<uint64_t>::head_tail(dst, value, count); | ||
| if (count <= 32) | ||
| return generic::Memset<uint128_t>::head_tail(dst, value, count); | ||
| if (count <= (32 + 64)) { | ||
| generic::Memset<uint256_t>::block(dst, value); | ||
| if (count <= 64) | ||
| return generic::Memset<uint256_t>::tail(dst, value, count); | ||
| generic::Memset<uint256_t>::block(dst + 32, value); | ||
| generic::Memset<uint256_t>::tail(dst, value, count); | ||
| return; | ||
| } | ||
| if (count >= 448 && value == 0 && aarch64::neon::hasZva()) { | ||
| generic::Memset<uint512_t>::block(dst, 0); | ||
| align_to_next_boundary<64>(dst, count); | ||
| return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count); | ||
| } else { | ||
| generic::Memset<uint128_t>::block(dst, value); | ||
| align_to_next_boundary<16>(dst, count); | ||
| return generic::Memset<uint512_t>::loop_and_tail(dst, value, count); | ||
| } | ||
| } | ||
|
|
||
| } // namespace __llvm_libc | ||
|
|
||
| #endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMSET_IMPLEMENTATIONS_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
33 changes: 33 additions & 0 deletions
33
libc/src/string/memory_utils/riscv/memset_implementations.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| //===-- Memset implementation for riscv -------------------------*- C++ -*-===// | ||
| // | ||
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| // See https://llvm.org/LICENSE.txt for license information. | ||
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
| #ifndef LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMSET_IMPLEMENTATIONS_H | ||
| #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMSET_IMPLEMENTATIONS_H | ||
|
|
||
| #include "src/__support/macros/attributes.h" // LIBC_INLINE | ||
| #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 | ||
| #include "src/string/memory_utils/generic/aligned_access.h" | ||
| #include "src/string/memory_utils/utils.h" // Ptr, CPtr | ||
|
|
||
| #include <stddef.h> // size_t | ||
|
|
||
| namespace __llvm_libc { | ||
|
|
||
| LIBC_INLINE static void inline_memset_riscv(Ptr dst, uint8_t value, | ||
| size_t count) { | ||
| #if defined(LIBC_TARGET_ARCH_IS_RISCV64) | ||
| return inline_memset_aligned_access_64bit(dst, value, count); | ||
| #elif defined(LIBC_TARGET_ARCH_IS_RISCV32) | ||
| return inline_memset_aligned_access_32bit(dst, value, count); | ||
| #else | ||
| #error "Unimplemented" | ||
| #endif | ||
| } | ||
|
|
||
| } // namespace __llvm_libc | ||
|
|
||
| #endif // LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMSET_IMPLEMENTATIONS_H |
65 changes: 65 additions & 0 deletions
65
libc/src/string/memory_utils/x86_64/memset_implementations.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| //===-- Memset implementation for x86_64 ------------------------*- C++ -*-===// | ||
| // | ||
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| // See https://llvm.org/LICENSE.txt for license information. | ||
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
| #ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMSET_IMPLEMENTATIONS_H | ||
| #define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMSET_IMPLEMENTATIONS_H | ||
|
|
||
| #include "src/__support/macros/attributes.h" // LIBC_INLINE | ||
| #include "src/string/memory_utils/op_generic.h" | ||
| #include "src/string/memory_utils/op_x86.h" | ||
| #include "src/string/memory_utils/utils.h" // Ptr, CPtr | ||
|
|
||
| #include <stddef.h> // size_t | ||
|
|
||
| namespace __llvm_libc { | ||
|
|
||
| [[maybe_unused]] LIBC_INLINE static void | ||
| inline_memset_x86(Ptr dst, uint8_t value, size_t count) { | ||
| #if defined(__AVX512F__) | ||
| using uint128_t = generic_v128; | ||
| using uint256_t = generic_v256; | ||
| using uint512_t = generic_v512; | ||
| #elif defined(__AVX__) | ||
| using uint128_t = generic_v128; | ||
| using uint256_t = generic_v256; | ||
| using uint512_t = cpp::array<generic_v256, 2>; | ||
| #elif defined(__SSE2__) | ||
| using uint128_t = generic_v128; | ||
| using uint256_t = cpp::array<generic_v128, 2>; | ||
| using uint512_t = cpp::array<generic_v128, 4>; | ||
| #else | ||
| using uint128_t = cpp::array<uint64_t, 2>; | ||
| using uint256_t = cpp::array<uint64_t, 4>; | ||
| using uint512_t = cpp::array<uint64_t, 8>; | ||
| #endif | ||
|
|
||
| if (count == 0) | ||
| return; | ||
| if (count == 1) | ||
| return generic::Memset<uint8_t>::block(dst, value); | ||
| if (count == 2) | ||
| return generic::Memset<uint16_t>::block(dst, value); | ||
| if (count == 3) | ||
| return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value); | ||
| if (count <= 8) | ||
| return generic::Memset<uint32_t>::head_tail(dst, value, count); | ||
| if (count <= 16) | ||
| return generic::Memset<uint64_t>::head_tail(dst, value, count); | ||
| if (count <= 32) | ||
| return generic::Memset<uint128_t>::head_tail(dst, value, count); | ||
| if (count <= 64) | ||
| return generic::Memset<uint256_t>::head_tail(dst, value, count); | ||
| if (count <= 128) | ||
| return generic::Memset<uint512_t>::head_tail(dst, value, count); | ||
| // Aligned loop | ||
| generic::Memset<uint256_t>::block(dst, value); | ||
| align_to_next_boundary<32>(dst, count); | ||
| return generic::Memset<uint256_t>::loop_and_tail(dst, value, count); | ||
| } | ||
| } // namespace __llvm_libc | ||
|
|
||
| #endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMSET_IMPLEMENTATIONS_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters