| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| //===-- Elementary operations for x86 -------------------------------------===// | ||
| // | ||
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| // See https://llvm.org/LICENSE.txt for license information. | ||
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
|
|
||
| #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H | ||
| #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H | ||
|
|
||
| #include <stddef.h> // size_t | ||
| #include <stdint.h> // uint8_t, uint16_t, uint32_t, uint64_t | ||
|
|
||
| #ifdef __SSE2__ | ||
| #include <immintrin.h> | ||
| #endif // __SSE2__ | ||
|
|
||
| #include "src/string/memory_utils/elements.h" // __llvm_libc::scalar | ||
|
|
||
| // Fixed-size Vector Operations | ||
| // ---------------------------- | ||
|
|
||
| namespace __llvm_libc { | ||
| namespace x86 { | ||
|
|
||
| #ifdef __SSE2__ | ||
| template <typename Base> struct Vector : public Base { | ||
| static void Copy(char *dst, const char *src) { | ||
| Base::Store(dst, Base::Load(src)); | ||
| } | ||
|
|
||
| static bool Equals(const char *a, const char *b) { | ||
| return Base::NotEqualMask(Base::Load(a), Base::Load(b)) == 0; | ||
| } | ||
|
|
||
| static int ThreeWayCompare(const char *a, const char *b) { | ||
| const auto mask = Base::NotEqualMask(Base::Load(a), Base::Load(b)); | ||
| if (!mask) | ||
| return 0; | ||
| return CharDiff(a, b, mask); | ||
| } | ||
|
|
||
| static void SplatSet(char *dst, const unsigned char value) { | ||
| Base::Store(dst, Base::GetSplattedValue(value)); | ||
| } | ||
|
|
||
| static int CharDiff(const char *a, const char *b, uint64_t mask) { | ||
| const size_t diff_index = __builtin_ctzl(mask); | ||
| const int ca = (unsigned char)a[diff_index]; | ||
| const int cb = (unsigned char)b[diff_index]; | ||
| return ca - cb; | ||
| } | ||
| }; | ||
|
|
||
| struct M128 { | ||
| static constexpr size_t kSize = 16; | ||
| using T = char __attribute__((__vector_size__(kSize))); | ||
| static uint16_t mask(T value) { return _mm_movemask_epi8(value); } | ||
| static uint16_t NotEqualMask(T a, T b) { return mask(a != b); } | ||
| static T Load(const char *ptr) { | ||
| return _mm_loadu_si128(reinterpret_cast<__m128i_u const *>(ptr)); | ||
| } | ||
| static void Store(char *ptr, T value) { | ||
| return _mm_storeu_si128(reinterpret_cast<__m128i_u *>(ptr), value); | ||
| } | ||
| static T GetSplattedValue(const char v) { | ||
| const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v}; | ||
| return splatted; | ||
| } | ||
| }; | ||
|
|
||
| using Vector128 = Vector<M128>; // 16 Bytes | ||
|
|
||
| #ifdef __AVX2__ | ||
| struct M256 { | ||
| static constexpr size_t kSize = 32; | ||
| using T = char __attribute__((__vector_size__(kSize))); | ||
| static uint32_t mask(T value) { return _mm256_movemask_epi8(value); } | ||
| static uint32_t NotEqualMask(T a, T b) { return mask(a != b); } | ||
| static T Load(const char *ptr) { | ||
| return _mm256_loadu_si256(reinterpret_cast<__m256i const *>(ptr)); | ||
| } | ||
| static void Store(char *ptr, T value) { | ||
| return _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), value); | ||
| } | ||
| static T GetSplattedValue(const char v) { | ||
| const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, | ||
| v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v}; | ||
| return splatted; | ||
| } | ||
| }; | ||
|
|
||
| using Vector256 = Vector<M256>; // 32 Bytes | ||
|
|
||
| #if defined(__AVX512F__) and defined(__AVX512BW__) | ||
| struct M512 { | ||
| static constexpr size_t kSize = 64; | ||
| using T = char __attribute__((__vector_size__(kSize))); | ||
| static uint64_t NotEqualMask(T a, T b) { | ||
| return _mm512_cmpneq_epi8_mask(a, b); | ||
| } | ||
| static T Load(const char *ptr) { return _mm512_loadu_epi8(ptr); } | ||
| static void Store(char *ptr, T value) { | ||
| return _mm512_storeu_epi8(ptr, value); | ||
| } | ||
| static T GetSplattedValue(const char v) { | ||
| const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, | ||
| v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, | ||
| v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, | ||
| v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v}; | ||
| return splatted; | ||
| } | ||
| }; | ||
| using Vector512 = Vector<M512>; | ||
|
|
||
| #endif // defined(__AVX512F__) and defined(__AVX512BW__) | ||
| #endif // __AVX2__ | ||
| #endif // __SSE2__ | ||
|
|
||
| using _1 = __llvm_libc::scalar::_1; | ||
| using _2 = __llvm_libc::scalar::_2; | ||
| using _3 = __llvm_libc::scalar::_3; | ||
| using _4 = __llvm_libc::scalar::_4; | ||
| using _8 = __llvm_libc::scalar::_8; | ||
| #if defined(__AVX512F__) && defined(__AVX512BW__) | ||
| using _16 = __llvm_libc::x86::Vector128; | ||
| using _32 = __llvm_libc::x86::Vector256; | ||
| using _64 = __llvm_libc::x86::Vector512; | ||
| using _128 = __llvm_libc::Repeated<_64, 2>; | ||
| #elif defined(__AVX2__) | ||
| using _16 = __llvm_libc::x86::Vector128; | ||
| using _32 = __llvm_libc::x86::Vector256; | ||
| using _64 = __llvm_libc::Repeated<_32, 2>; | ||
| using _128 = __llvm_libc::Repeated<_32, 4>; | ||
| #elif defined(__SSE2__) | ||
| using _16 = __llvm_libc::x86::Vector128; | ||
| using _32 = __llvm_libc::Repeated<_16, 2>; | ||
| using _64 = __llvm_libc::Repeated<_16, 4>; | ||
| using _128 = __llvm_libc::Repeated<_16, 8>; | ||
| #else | ||
| using _16 = __llvm_libc::Repeated<_8, 2>; | ||
| using _32 = __llvm_libc::Repeated<_8, 4>; | ||
| using _64 = __llvm_libc::Repeated<_8, 8>; | ||
| using _128 = __llvm_libc::Repeated<_8, 16>; | ||
| #endif | ||
|
|
||
| } // namespace x86 | ||
| } // namespace __llvm_libc | ||
|
|
||
| #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| //===-- Unittests for memory_utils ----------------------------------------===// | ||
| // | ||
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| // See https://llvm.org/LICENSE.txt for license information. | ||
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
|
|
||
| #include "src/string/memory_utils/elements.h" | ||
| #include "utils/CPP/Array.h" | ||
| #include "utils/UnitTest/Test.h" | ||
|
|
||
| namespace __llvm_libc { | ||
|
|
||
| // Registering Types | ||
| using FixedSizeTypes = testing::TypeList< | ||
| #ifdef __SSE2__ | ||
| x86::Vector128, // | ||
| #endif // __SSE2__ | ||
| #ifdef __AVX2__ | ||
| x86::Vector256, // | ||
| #endif // __AVX2__ | ||
| #if defined(__AVX512F__) and defined(__AVX512BW__) | ||
| x86::Vector512, // | ||
| #endif // defined(__AVX512F__) and defined(__AVX512BW__) | ||
| scalar::UINT8, // | ||
| scalar::UINT16, // | ||
| scalar::UINT32, // | ||
| scalar::UINT64, // | ||
| Repeated<scalar::UINT64, 2>, // | ||
| Repeated<scalar::UINT64, 4>, // | ||
| Repeated<scalar::UINT64, 8>, // | ||
| Repeated<scalar::UINT64, 16>, // | ||
| Repeated<scalar::UINT64, 32>, // | ||
| Chained<scalar::UINT16, scalar::UINT8>, // | ||
| Chained<scalar::UINT32, scalar::UINT16, scalar::UINT8>, // | ||
| builtin::_1, // | ||
| builtin::_2, // | ||
| builtin::_3, // | ||
| builtin::_4, // | ||
| builtin::_8 // | ||
| >; | ||
|
|
||
| char GetRandomChar() { | ||
| static constexpr const uint64_t a = 1103515245; | ||
| static constexpr const uint64_t c = 12345; | ||
| static constexpr const uint64_t m = 1ULL << 31; | ||
| static uint64_t seed = 123456789; | ||
| seed = (a * seed + c) % m; | ||
| return seed; | ||
| } | ||
|
|
||
| template <typename Element> using Buffer = cpp::Array<char, Element::kSize>; | ||
| template <typename Element> Buffer<Element> GetRandomBuffer() { | ||
| Buffer<Element> buffer; | ||
| for (auto ¤t : buffer) | ||
| current = GetRandomChar(); | ||
| return buffer; | ||
| } | ||
|
|
||
| TYPED_TEST(LlvmLibcMemoryElements, Copy, FixedSizeTypes) { | ||
| Buffer<ParamType> Dst; | ||
| const auto buffer = GetRandomBuffer<ParamType>(); | ||
| Copy<ParamType>(Dst.data(), buffer.data()); | ||
| for (size_t i = 0; i < ParamType::kSize; ++i) | ||
| EXPECT_EQ(Dst[i], buffer[i]); | ||
| } | ||
|
|
||
| TYPED_TEST(LlvmLibcMemoryElements, Equals, FixedSizeTypes) { | ||
| const auto buffer = GetRandomBuffer<ParamType>(); | ||
| EXPECT_TRUE(Equals<ParamType>(buffer.data(), buffer.data())); | ||
| } | ||
|
|
||
| TYPED_TEST(LlvmLibcMemoryElements, ThreeWayCompare, FixedSizeTypes) { | ||
| Buffer<ParamType> initial; | ||
| for (auto &c : initial) | ||
| c = 5; | ||
|
|
||
| // Testing equality | ||
| EXPECT_EQ(ThreeWayCompare<ParamType>(initial.data(), initial.data()), 0); | ||
|
|
||
| // Testing all mismatching positions | ||
| for (size_t i = 0; i < ParamType::kSize; ++i) { | ||
| auto copy = initial; | ||
| ++copy[i]; // Copy is now lexicographycally greated than initial | ||
| const auto *less = initial.data(); | ||
| const auto *greater = copy.data(); | ||
| EXPECT_LT(ThreeWayCompare<ParamType>(less, greater), 0); | ||
| EXPECT_GT(ThreeWayCompare<ParamType>(greater, less), 0); | ||
| } | ||
| } | ||
|
|
||
| TYPED_TEST(LlvmLibcMemoryElements, Splat, FixedSizeTypes) { | ||
| Buffer<ParamType> Dst; | ||
| const cpp::Array<char, 3> values = {char(0x00), char(0x7F), char(0xFF)}; | ||
| for (char value : values) { | ||
| SplatSet<ParamType>(Dst.data(), value); | ||
| for (size_t i = 0; i < ParamType::kSize; ++i) | ||
| EXPECT_EQ(Dst[i], value); | ||
| } | ||
| } | ||
|
|
||
| } // namespace __llvm_libc |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,231 @@ | ||
| //===-- Unittests for memory_utils ----------------------------------------===// | ||
| // | ||
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| // See https://llvm.org/LICENSE.txt for license information. | ||
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
|
|
||
| #define LLVM_LIBC_UNITTEST_OBSERVE 1 | ||
|
|
||
| #include "src/string/memory_utils/elements.h" | ||
| #include "utils/CPP/Array.h" | ||
| #include "utils/CPP/ArrayRef.h" | ||
| #include "utils/UnitTest/Test.h" | ||
|
|
||
| #include <stdio.h> | ||
| #include <string.h> | ||
|
|
||
| namespace __llvm_libc { | ||
|
|
||
| static constexpr const size_t kMaxBuffer = 32; | ||
|
|
||
| struct BufferAccess : cpp::Array<char, kMaxBuffer + 1> { | ||
| BufferAccess() { Reset(); } | ||
| void Reset() { | ||
| for (auto &value : *this) | ||
| value = '0'; | ||
| this->operator[](kMaxBuffer) = '\0'; | ||
| } | ||
| void Touch(ptrdiff_t offset, size_t size) { | ||
| if (offset < 0) | ||
| return; | ||
| for (size_t i = 0; i < size; ++i) | ||
| ++(*this)[offset + i]; | ||
| } | ||
| operator const char *() const { return this->data(); } | ||
| }; | ||
|
|
||
| struct Buffer { | ||
| ptrdiff_t Offset(const char *ptr) const { | ||
| const bool contained = ptr >= data.begin() && ptr < data.end(); | ||
| return contained ? ptr - data.begin() : -1; | ||
| } | ||
| void Reset() { | ||
| reads.Reset(); | ||
| writes.Reset(); | ||
| } | ||
| cpp::Array<char, kMaxBuffer> data; | ||
| BufferAccess __attribute__((aligned(64))) reads; | ||
| BufferAccess __attribute__((aligned(64))) writes; | ||
| }; | ||
|
|
||
| struct MemoryAccessObserver { | ||
| void ObserveRead(const char *ptr, size_t size) { | ||
| Buffer1.reads.Touch(Buffer1.Offset(ptr), size); | ||
| Buffer2.reads.Touch(Buffer2.Offset(ptr), size); | ||
| } | ||
|
|
||
| void ObserveWrite(const char *ptr, size_t size) { | ||
| Buffer1.writes.Touch(Buffer1.Offset(ptr), size); | ||
| Buffer2.writes.Touch(Buffer2.Offset(ptr), size); | ||
| } | ||
|
|
||
| void Reset() { | ||
| Buffer1.Reset(); | ||
| Buffer2.Reset(); | ||
| } | ||
|
|
||
| Buffer Buffer1; | ||
| Buffer Buffer2; | ||
| }; | ||
|
|
||
| MemoryAccessObserver Observer; | ||
|
|
||
| template <size_t Size> struct TestingElement { | ||
| static constexpr size_t kSize = Size; | ||
|
|
||
| static void Copy(char *__restrict dst, const char *__restrict src) { | ||
| Observer.ObserveRead(src, kSize); | ||
| Observer.ObserveWrite(dst, kSize); | ||
| } | ||
|
|
||
| static bool Equals(const char *lhs, const char *rhs) { | ||
| Observer.ObserveRead(lhs, kSize); | ||
| Observer.ObserveRead(rhs, kSize); | ||
| return true; | ||
| } | ||
|
|
||
| static int ThreeWayCompare(const char *lhs, const char *rhs) { | ||
| Observer.ObserveRead(lhs, kSize); | ||
| Observer.ObserveRead(rhs, kSize); | ||
| return 0; | ||
| } | ||
|
|
||
| static void SplatSet(char *dst, const unsigned char value) { | ||
| Observer.ObserveWrite(dst, kSize); | ||
| } | ||
| }; | ||
|
|
||
| using Types = testing::TypeList< | ||
| TestingElement<1>, // 1 Byte | ||
| TestingElement<2>, // 2 Bytes | ||
| TestingElement<4>, // 4 Bytes | ||
| Repeated<TestingElement<2>, 3>, // 6 Bytes | ||
| Chained<TestingElement<4>, TestingElement<2>, TestingElement<1>> // 7 Bytes | ||
| >; | ||
|
|
||
| struct LlvmLibcTestAccessBase : public testing::Test { | ||
|
|
||
| template <typename HigherOrder, size_t Size, size_t Offset = 0> | ||
| void checkOperations(const BufferAccess &expected) { | ||
| static const BufferAccess untouched; | ||
|
|
||
| Observer.Reset(); | ||
| HigherOrder::Copy(dst_ptr() + Offset, src_ptr() + Offset, Size); | ||
| ASSERT_STREQ(src().writes, untouched); | ||
| ASSERT_STREQ(dst().reads, untouched); | ||
| ASSERT_STREQ(src().reads, expected); | ||
| ASSERT_STREQ(dst().writes, expected); | ||
| Observer.Reset(); | ||
| HigherOrder::Equals(lhs_ptr() + Offset, rhs_ptr() + Offset, Size); | ||
| ASSERT_STREQ(lhs().writes, untouched); | ||
| ASSERT_STREQ(rhs().writes, untouched); | ||
| ASSERT_STREQ(lhs().reads, expected); | ||
| ASSERT_STREQ(rhs().reads, expected); | ||
| Observer.Reset(); | ||
| HigherOrder::ThreeWayCompare(lhs_ptr() + Offset, rhs_ptr() + Offset, Size); | ||
| ASSERT_STREQ(lhs().writes, untouched); | ||
| ASSERT_STREQ(rhs().writes, untouched); | ||
| ASSERT_STREQ(lhs().reads, expected); | ||
| ASSERT_STREQ(rhs().reads, expected); | ||
| Observer.Reset(); | ||
| HigherOrder::SplatSet(dst_ptr() + Offset, 5, Size); | ||
| ASSERT_STREQ(src().reads, untouched); | ||
| ASSERT_STREQ(src().writes, untouched); | ||
| ASSERT_STREQ(dst().reads, untouched); | ||
| ASSERT_STREQ(dst().writes, expected); | ||
| } | ||
|
|
||
| void checkMaxAccess(const BufferAccess &expected, int max) { | ||
| for (size_t i = 0; i < kMaxBuffer; ++i) { | ||
| int value = (int)expected[i] - '0'; | ||
| if (value < 0 || value > max) { | ||
| printf("expected no more than %d access, was '%s'\n", max, | ||
| (const char *)expected); | ||
| ASSERT_LE(value, max); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private: | ||
| const Buffer &lhs() const { return Observer.Buffer1; } | ||
| const Buffer &rhs() const { return Observer.Buffer2; } | ||
| const Buffer &src() const { return Observer.Buffer2; } | ||
| const Buffer &dst() const { return Observer.Buffer1; } | ||
| Buffer &dst() { return Observer.Buffer1; } | ||
|
|
||
| char *dst_ptr() { return dst().data.begin(); } | ||
| const char *src_ptr() { return src().data.begin(); } | ||
| const char *lhs_ptr() { return lhs().data.begin(); } | ||
| const char *rhs_ptr() { return rhs().data.begin(); } | ||
| }; | ||
|
|
||
| template <typename ParamType> | ||
| struct LlvmLibcTestAccessTail : public LlvmLibcTestAccessBase { | ||
|
|
||
| void TearDown() override { | ||
| static constexpr size_t Size = 10; | ||
|
|
||
| BufferAccess expected; | ||
| expected.Touch(Size - ParamType::kSize, ParamType::kSize); | ||
|
|
||
| checkMaxAccess(expected, 1); | ||
| checkOperations<Tail<ParamType>, Size>(expected); | ||
| } | ||
| }; | ||
| TYPED_TEST_F(LlvmLibcTestAccessTail, Operations, Types) {} | ||
|
|
||
| template <typename ParamType> | ||
| struct LlvmLibcTestAccessHeadTail : public LlvmLibcTestAccessBase { | ||
| void TearDown() override { | ||
| static constexpr size_t Size = 10; | ||
|
|
||
| BufferAccess expected; | ||
| expected.Touch(0, ParamType::kSize); | ||
| expected.Touch(Size - ParamType::kSize, ParamType::kSize); | ||
|
|
||
| checkMaxAccess(expected, 2); | ||
| checkOperations<HeadTail<ParamType>, Size>(expected); | ||
| } | ||
| }; | ||
| TYPED_TEST_F(LlvmLibcTestAccessHeadTail, Operations, Types) {} | ||
|
|
||
| template <typename ParamType> | ||
| struct LlvmLibcTestAccessLoop : public LlvmLibcTestAccessBase { | ||
| void TearDown() override { | ||
| static constexpr size_t Size = 20; | ||
|
|
||
| BufferAccess expected; | ||
| for (size_t i = 0; i < Size - ParamType::kSize; i += ParamType::kSize) | ||
| expected.Touch(i, ParamType::kSize); | ||
| expected.Touch(Size - ParamType::kSize, ParamType::kSize); | ||
|
|
||
| checkMaxAccess(expected, 2); | ||
| checkOperations<Loop<ParamType>, Size>(expected); | ||
| } | ||
| }; | ||
| TYPED_TEST_F(LlvmLibcTestAccessLoop, Operations, Types) {} | ||
|
|
||
| template <typename ParamType> | ||
| struct LlvmLibcTestAccessAlignedAccess : public LlvmLibcTestAccessBase { | ||
| void TearDown() override { | ||
| static constexpr size_t Size = 10; | ||
| static constexpr size_t Offset = 2; | ||
| using AlignmentT = TestingElement<4>; | ||
|
|
||
| BufferAccess expected; | ||
| expected.Touch(Offset, AlignmentT::kSize); | ||
| expected.Touch(AlignmentT::kSize, ParamType::kSize); | ||
| expected.Touch(Offset + Size - ParamType::kSize, ParamType::kSize); | ||
|
|
||
| checkMaxAccess(expected, 3); | ||
| checkOperations<Align<AlignmentT, Arg::_1>::Then<HeadTail<ParamType>>, Size, | ||
| Offset>(expected); | ||
| checkOperations<Align<AlignmentT, Arg::_2>::Then<HeadTail<ParamType>>, Size, | ||
| Offset>(expected); | ||
| } | ||
| }; | ||
| TYPED_TEST_F(LlvmLibcTestAccessAlignedAccess, Operations, Types) {} | ||
|
|
||
| } // namespace __llvm_libc |