From 1c885b0df6a94f7e927b443cdb9e46fe8560bd63 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 14 Sep 2025 00:11:15 -0700 Subject: [PATCH 1/3] [ADT] Define countr_zero in terms of popcount (NFC) We can express the fallback mechanism of llvm::countr_zero a lot more concisely with llvm::popcount. Since llvm::countr_zero now requires llvm::popcount, this patch moves llvm::popcount earlier. --- llvm/include/llvm/ADT/bit.h | 74 ++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h index d6e33c3e6133a..73aa84504d6e9 100644 --- a/llvm/include/llvm/ADT/bit.h +++ b/llvm/include/llvm/ADT/bit.h @@ -148,6 +148,35 @@ template >> return (Value != 0) && ((Value & (Value - 1)) == 0); } +/// Count the number of set bits in a value. +/// Ex. popcount(0xF000F000) = 8 +/// Returns 0 if the word is zero. +template >> +[[nodiscard]] inline int popcount(T Value) noexcept { + if constexpr (sizeof(T) <= 4) { +#if defined(__GNUC__) + return (int)__builtin_popcount(Value); +#else + uint32_t v = Value; + v = v - ((v >> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + return int(((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24); +#endif + } else if constexpr (sizeof(T) <= 8) { +#if defined(__GNUC__) + return (int)__builtin_popcountll(Value); +#else + uint64_t v = Value; + v = v - ((v >> 1) & 0x5555555555555555ULL); + v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); + v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; + return int((uint64_t)(v * 0x0101010101010101ULL) >> 56); +#endif + } else { + static_assert(sizeof(T) == 0, "T must be 8 bytes or less"); + } +} + /// Count number of 0's from the least significant bit to the most /// stopping at the first 1. /// @@ -179,19 +208,9 @@ template [[nodiscard]] int countr_zero(T Val) { #endif } - // Fall back to the bisection method. - unsigned ZeroBits = 0; - T Shift = std::numeric_limits::digits >> 1; - T Mask = std::numeric_limits::max() >> Shift; - while (Shift) { - if ((Val & Mask) == 0) { - Val >>= Shift; - ZeroBits |= Shift; - } - Shift >>= 1; - Mask >>= Shift; - } - return ZeroBits; + // Fallback to popcount. "(Val & -Val) - 1" is a bitmask with all bits below + // the least significant 1 set. + return llvm::popcount(static_cast>((Val & -Val) - 1)); } /// Count number of 0's from the most significant bit to the least @@ -300,35 +319,6 @@ template [[nodiscard]] T bit_ceil(T Value) { return T(1) << llvm::bit_width(Value - 1u); } -/// Count the number of set bits in a value. -/// Ex. popcount(0xF000F000) = 8 -/// Returns 0 if the word is zero. -template >> -[[nodiscard]] inline int popcount(T Value) noexcept { - if constexpr (sizeof(T) <= 4) { -#if defined(__GNUC__) - return (int)__builtin_popcount(Value); -#else - uint32_t v = Value; - v = v - ((v >> 1) & 0x55555555); - v = (v & 0x33333333) + ((v >> 2) & 0x33333333); - return int(((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24); -#endif - } else if constexpr (sizeof(T) <= 8) { -#if defined(__GNUC__) - return (int)__builtin_popcountll(Value); -#else - uint64_t v = Value; - v = v - ((v >> 1) & 0x5555555555555555ULL); - v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); - v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; - return int((uint64_t)(v * 0x0101010101010101ULL) >> 56); -#endif - } else { - static_assert(sizeof(T) == 0, "T must be 8 bytes or less"); - } -} - // Forward-declare rotr so that rotl can use it. template >> [[nodiscard]] constexpr T rotr(T V, int R); From 6ca2e58d3d43ce5d31d1ee150da459644d462073 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 14 Sep 2025 20:10:41 -0700 Subject: [PATCH 2/3] Address comments. --- llvm/include/llvm/ADT/bit.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h index 73aa84504d6e9..fec8340f9689a 100644 --- a/llvm/include/llvm/ADT/bit.h +++ b/llvm/include/llvm/ADT/bit.h @@ -150,30 +150,31 @@ template >> /// Count the number of set bits in a value. /// Ex. popcount(0xF000F000) = 8 -/// Returns 0 if the word is zero. -template >> +/// Returns 0 if Value is zero. +template [[nodiscard]] inline int popcount(T Value) noexcept { + static_assert(std::is_unsigned_v, "T must be an unsigned integer type"); + static_assert(sizeof(T) <= 8, "T must be 8 bytes or less"); + if constexpr (sizeof(T) <= 4) { #if defined(__GNUC__) return (int)__builtin_popcount(Value); #else - uint32_t v = Value; - v = v - ((v >> 1) & 0x55555555); - v = (v & 0x33333333) + ((v >> 2) & 0x33333333); - return int(((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24); + uint32_t V = Value; + V = V - ((V >> 1) & 0x55555555); + V = (V & 0x33333333) + ((V >> 2) & 0x33333333); + return int(((V + (V >> 4) & 0xF0F0F0F) * 0x1010101) >> 24); #endif - } else if constexpr (sizeof(T) <= 8) { + } else { #if defined(__GNUC__) return (int)__builtin_popcountll(Value); #else - uint64_t v = Value; - v = v - ((v >> 1) & 0x5555555555555555ULL); - v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); - v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; - return int((uint64_t)(v * 0x0101010101010101ULL) >> 56); + uint64_t V = Value; + V = V - ((V >> 1) & 0x5555555555555555ULL); + V = (V & 0x3333333333333333ULL) + ((V >> 2) & 0x3333333333333333ULL); + V = (V + (V >> 4)) & 0x0F0F0F0F0F0F0F0FULL; + return int((uint64_t)(V * 0x0101010101010101ULL) >> 56); #endif - } else { - static_assert(sizeof(T) == 0, "T must be 8 bytes or less"); } } From 2ce4e6c7dc195eb6c3ec83b748efb9f50aa72c33 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Sun, 14 Sep 2025 21:26:56 -0700 Subject: [PATCH 3/3] Fix formatting. --- llvm/include/llvm/ADT/bit.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h index fec8340f9689a..671d9bfa78bb2 100644 --- a/llvm/include/llvm/ADT/bit.h +++ b/llvm/include/llvm/ADT/bit.h @@ -151,8 +151,7 @@ template >> /// Count the number of set bits in a value. /// Ex. popcount(0xF000F000) = 8 /// Returns 0 if Value is zero. -template -[[nodiscard]] inline int popcount(T Value) noexcept { +template [[nodiscard]] inline int popcount(T Value) noexcept { static_assert(std::is_unsigned_v, "T must be an unsigned integer type"); static_assert(sizeof(T) <= 8, "T must be 8 bytes or less");