From e688df7fcc54750bf162b93a460d0238170f3a2e Mon Sep 17 00:00:00 2001 From: Nathaniel Brough Date: Tue, 24 Oct 2023 04:32:01 +0800 Subject: [PATCH 1/3] bug: Fix out of bounds access for haswell simd op (#340) NUM_CHUNKS can only be 2, but we are accessing indexes outside of this range i.e. 2,3. --- src/simdutf/haswell/simd.h | 4 +--- src/simdutf/haswell/simd16-inl.h | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/simdutf/haswell/simd.h b/src/simdutf/haswell/simd.h index 64f8411b6..d1834b5ac 100644 --- a/src/simdutf/haswell/simd.h +++ b/src/simdutf/haswell/simd.h @@ -348,9 +348,7 @@ namespace simd { return simd8x64( (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), - (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), - (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), - (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low) ).to_bitmask(); } simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { diff --git a/src/simdutf/haswell/simd16-inl.h b/src/simdutf/haswell/simd16-inl.h index fb2a8dccf..04c1b7fe0 100644 --- a/src/simdutf/haswell/simd16-inl.h +++ b/src/simdutf/haswell/simd16-inl.h @@ -243,9 +243,7 @@ struct simd16: base16_numeric { return simd16x32( (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low), - (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low), - (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low), - (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low) + (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low) ).to_bitmask(); } simdutf_really_inline uint64_t not_in_range(const T low, const T high) const { From fe9ac10431bd338b86645a1d184d37965d1d642d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 24 Oct 2023 09:25:53 -0400 Subject: [PATCH 2/3] fix alignment issue for bele tests (#342) * fix alignment issue for bele tests * typo --- tests/bele_tests.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/bele_tests.cpp b/tests/bele_tests.cpp index 10932bdc5..26fdc4bfe 100644 --- a/tests/bele_tests.cpp +++ b/tests/bele_tests.cpp @@ -5,19 +5,16 @@ #include // We use explicit arrays so that no funny business is possible. - -// -// s = "@\u00A7\u2208\U0001D4AA" const unsigned char utf8_string[] = {0x40,0xc2,0xa7,0xe2,0x88,0x88,0xf0,0x9d,0x92,0xaa}; const char *utf8 = reinterpret_cast(utf8_string); const size_t utf8_size = sizeof(utf8_string)/sizeof(uint8_t); -const unsigned char utf16le_string[] = {0x40,0x00,0xa7,0x00,0x08,0x22,0x35,0xd8,0xaa,0xdc}; +alignas(char16_t) const unsigned char utf16le_string[] = {0x40,0x00,0xa7,0x00,0x08,0x22,0x35,0xd8,0xaa,0xdc}; const char16_t *utf16le = reinterpret_cast(utf16le_string); const size_t utf16_size = sizeof(utf16le_string)/sizeof(uint16_t); -const unsigned char utf16be_string[] = {0x00,0x40,0x00,0xa7,0x22,0x08,0xd8,0x35,0xdc,0xaa}; +alignas(char16_t) const unsigned char utf16be_string[] = {0x00,0x40,0x00,0xa7,0x22,0x08,0xd8,0x35,0xdc,0xaa}; const char16_t *utf16be = reinterpret_cast(utf16be_string); #if SIMDUTF_IS_BIG_ENDIAN const char16_t *utf16 = utf16be; @@ -27,11 +24,11 @@ const char16_t *utf16 = utf16le; // Native order #if SIMDUTF_IS_BIG_ENDIAN -const unsigned char utf32_string[] = {0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xa7,0x00,0x00,0x22,0x08,0x00,0x01,0xd4,0xaa}; -const char32_t *utf32 = reinterpret_cast(utf32_string); // Technically undefined behavior. +alignas(char32_t) const unsigned char utf32_string[] = {0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xa7,0x00,0x00,0x22,0x08,0x00,0x01,0xd4,0xaa}; +const char32_t *utf32 = reinterpret_cast(utf32_string); #else -const unsigned char utf32_string[] = {0x40,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x08,0x22,0x00,0x00,0xaa,0xd4,0x01,0x00}; -const char32_t *utf32 = reinterpret_cast(utf32_string); // Technically undefined behavior. +alignas(char32_t) const unsigned char utf32_string[] = {0x40,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x08,0x22,0x00,0x00,0xaa,0xd4,0x01,0x00}; +const char32_t *utf32 = reinterpret_cast(utf32_string); #endif const size_t utf32_size = sizeof(utf32_string)/sizeof(char32_t); const size_t number_of_code_points = utf32_size; From af4858e5ba6e8893aae8c60cc2c88dc19b307567 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 24 Oct 2023 09:26:06 -0400 Subject: [PATCH 3/3] The basic_fuzzer test times out too often when running the tests in debug mode under windows. (#343) --- .github/workflows/vs16-ci.yml | 2 +- .github/workflows/vs17-ci-cxx20.yml | 2 +- .github/workflows/vs17-ci.yml | 2 +- .github/workflows/vs17-clang-ci.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/vs16-ci.yml b/.github/workflows/vs16-ci.yml index 96182b994..85d46ab88 100644 --- a/.github/workflows/vs16-ci.yml +++ b/.github/workflows/vs16-ci.yml @@ -31,7 +31,7 @@ jobs: - name: Run Debug tests run: | cd build - ctest -C Debug --output-on-failure + ctest -C Debug --output-on-failure -E basic_fuzzer - name: Install run: | cmake --install build --config Release diff --git a/.github/workflows/vs17-ci-cxx20.yml b/.github/workflows/vs17-ci-cxx20.yml index 2e674b557..9e6d05235 100644 --- a/.github/workflows/vs17-ci-cxx20.yml +++ b/.github/workflows/vs17-ci-cxx20.yml @@ -31,7 +31,7 @@ jobs: - name: Run Debug tests run: | cd build - ctest -C Debug --output-on-failure + ctest -C Debug --output-on-failure -E basic_fuzzer - name: Install run: | cmake --install build --config Release diff --git a/.github/workflows/vs17-ci.yml b/.github/workflows/vs17-ci.yml index 9f5dc49ba..c0d6fa562 100644 --- a/.github/workflows/vs17-ci.yml +++ b/.github/workflows/vs17-ci.yml @@ -31,7 +31,7 @@ jobs: - name: Run Debug tests run: | cd build - ctest -C Debug --output-on-failure + ctest -C Debug --output-on-failure -E basic_fuzzer - name: Install run: | cmake --install build --config Release diff --git a/.github/workflows/vs17-clang-ci.yml b/.github/workflows/vs17-clang-ci.yml index 5b51fa9b4..a3ef11ad6 100644 --- a/.github/workflows/vs17-clang-ci.yml +++ b/.github/workflows/vs17-clang-ci.yml @@ -31,7 +31,7 @@ jobs: - name: Run Debug tests run: | cd build - ctest -C Debug --output-on-failure + ctest -C Debug --output-on-failure -E basic_fuzzer - name: Install run: | cmake --install build --config Release