Skip to content

Commit

Permalink
[WTF] Introduce SIMDHelpers
Browse files Browse the repository at this point in the history 
https://bugs.webkit.org/show_bug.cgi?id=273927
rdar://127795191

Reviewed by Keith Miller.

This patch adds wtf/SIMDHelpers.h, which offers various SIMD abstraction on the top of simde.
The main motivation is writing SIMD code for LChar / UChar in the same code.
Also, we add some x64 specific optimizations for some of particular primitive operations.

* Source/JavaScriptCore/runtime/JSONObject.cpp:
(JSC::FastStringifier<CharType>::append):
* Source/JavaScriptCore/runtime/LiteralParser.cpp:
(JSC::LiteralParser<CharType>::Lexer::lexString):
* Source/WTF/WTF.xcodeproj/project.pbxproj:
* Source/WTF/wtf/CMakeLists.txt:
* Source/WTF/wtf/SIMDHelpers.h: Added.
(WTF::SIMD::splat):
(WTF::SIMD::load):
(WTF::SIMD::store):
(WTF::SIMD::merge):
(WTF::SIMD::isNonZero):
(WTF::SIMD::findFirstNonZeroIndex):
(WTF::SIMD::equal):
(WTF::SIMD::lessThan):
* Source/WTF/wtf/text/StringCommon.h:

Canonical link: https://commits.webkit.org/278572@main
  • Loading branch information
@Constellation
Constellation committed May 9, 2024
1 parent 0d5648d commit d99e74d
Show file tree
Hide file tree
Showing 6 changed files with 234 additions and 167 deletions.
78 changes: 39 additions & 39 deletions Source/JavaScriptCore/runtime/JSONObject.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1066,41 +1066,41 @@ void FastStringifier<CharType>::append(JSValue value)
constexpr size_t stride = 16 / sizeof(CharType);
if (span.size() >= stride) {
using UnsignedType = std::make_unsigned_t<CharType>;
using BulkType = decltype(WTF::loadBulk(static_cast<const UnsignedType*>(nullptr)));
const auto quoteMask = WTF::splatBulk(static_cast<UnsignedType>('"'));
const auto escapeMask = WTF::splatBulk(static_cast<UnsignedType>('\\'));
const auto controlMask = WTF::splatBulk(static_cast<UnsignedType>(' '));
using BulkType = decltype(SIMD::load(static_cast<const UnsignedType*>(nullptr)));
const auto quoteMask = SIMD::splat(static_cast<UnsignedType>('"'));
const auto escapeMask = SIMD::splat(static_cast<UnsignedType>('\\'));
const auto controlMask = SIMD::splat(static_cast<UnsignedType>(' '));
const auto* ptr = span.data();
const auto* end = ptr + span.size();
auto* cursorEnd = cursor + span.size();
BulkType accumulated { };
for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) {
auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(ptr));
WTF::storeBulk(input, bitwise_cast<UnsignedType*>(cursor));
auto quotes = WTF::equalBulk(input, quoteMask);
auto escapes = WTF::equalBulk(input, escapeMask);
auto controls = WTF::lessThanBulk(input, controlMask);
accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
auto input = SIMD::load(bitwise_cast<const UnsignedType*>(ptr));
SIMD::store(input, bitwise_cast<UnsignedType*>(cursor));
auto quotes = SIMD::equal(input, quoteMask);
auto escapes = SIMD::equal(input, escapeMask);
auto controls = SIMD::lessThan(input, controlMask);
accumulated = SIMD::merge(accumulated, SIMD::merge(quotes, SIMD::merge(escapes, controls)));
if constexpr (sizeof(CharType) != 1) {
const auto surrogateMask = WTF::splatBulk(static_cast<UnsignedType>(0xf800));
const auto surrogateCheckMask = WTF::splatBulk(static_cast<UnsignedType>(0xd800));
accumulated = WTF::mergeBulk(accumulated, WTF::equalBulk(simde_vandq_u16(input, surrogateMask), surrogateCheckMask));
const auto surrogateMask = SIMD::splat(static_cast<UnsignedType>(0xf800));
const auto surrogateCheckMask = SIMD::splat(static_cast<UnsignedType>(0xd800));
accumulated = SIMD::merge(accumulated, SIMD::equal(simde_vandq_u16(input, surrogateMask), surrogateCheckMask));
}
}
if (ptr < end) {
auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(end - stride));
WTF::storeBulk(input, bitwise_cast<UnsignedType*>(cursorEnd - stride));
auto quotes = WTF::equalBulk(input, quoteMask);
auto escapes = WTF::equalBulk(input, escapeMask);
auto controls = WTF::lessThanBulk(input, controlMask);
accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
auto input = SIMD::load(bitwise_cast<const UnsignedType*>(end - stride));
SIMD::store(input, bitwise_cast<UnsignedType*>(cursorEnd - stride));
auto quotes = SIMD::equal(input, quoteMask);
auto escapes = SIMD::equal(input, escapeMask);
auto controls = SIMD::lessThan(input, controlMask);
accumulated = SIMD::merge(accumulated, SIMD::merge(quotes, SIMD::merge(escapes, controls)));
if constexpr (sizeof(CharType) != 1) {
const auto surrogateMask = WTF::splatBulk(static_cast<UnsignedType>(0xf800));
const auto surrogateCheckMask = WTF::splatBulk(static_cast<UnsignedType>(0xd800));
accumulated = WTF::mergeBulk(accumulated, WTF::equalBulk(simde_vandq_u16(input, surrogateMask), surrogateCheckMask));
const auto surrogateMask = SIMD::splat(static_cast<UnsignedType>(0xf800));
const auto surrogateCheckMask = SIMD::splat(static_cast<UnsignedType>(0xd800));
accumulated = SIMD::merge(accumulated, SIMD::equal(simde_vandq_u16(input, surrogateMask), surrogateCheckMask));
}
}
return WTF::isNonZeroBulk(accumulated);
return SIMD::isNonZero(accumulated);
}
#endif
for (auto character : span) {
Expand All @@ -1120,32 +1120,32 @@ void FastStringifier<CharType>::append(JSValue value)
constexpr size_t stride = 16 / sizeof(LChar);
if (span.size() >= stride) {
using UnsignedType = std::make_unsigned_t<LChar>;
using BulkType = decltype(WTF::loadBulk(static_cast<const UnsignedType*>(nullptr)));
const auto quoteMask = WTF::splatBulk(static_cast<UnsignedType>('"'));
const auto escapeMask = WTF::splatBulk(static_cast<UnsignedType>('\\'));
const auto controlMask = WTF::splatBulk(static_cast<UnsignedType>(' '));
const auto zeros = WTF::splatBulk(static_cast<UnsignedType>(0));
using BulkType = decltype(SIMD::load(static_cast<const UnsignedType*>(nullptr)));
const auto quoteMask = SIMD::splat(static_cast<UnsignedType>('"'));
const auto escapeMask = SIMD::splat(static_cast<UnsignedType>('\\'));
const auto controlMask = SIMD::splat(static_cast<UnsignedType>(' '));
const auto zeros = SIMD::splat(static_cast<UnsignedType>(0));
const auto* ptr = span.data();
const auto* end = ptr + span.size();
auto* cursorEnd = cursor + span.size();
BulkType accumulated { };
for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) {
auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(ptr));
auto input = SIMD::load(bitwise_cast<const UnsignedType*>(ptr));
simde_vst2q_u8(bitwise_cast<UnsignedType*>(cursor), (simde_uint8x16x2_t { input, zeros }));
auto quotes = WTF::equalBulk(input, quoteMask);
auto escapes = WTF::equalBulk(input, escapeMask);
auto controls = WTF::lessThanBulk(input, controlMask);
accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
auto quotes = SIMD::equal(input, quoteMask);
auto escapes = SIMD::equal(input, escapeMask);
auto controls = SIMD::lessThan(input, controlMask);
accumulated = SIMD::merge(accumulated, SIMD::merge(quotes, SIMD::merge(escapes, controls)));
}
if (ptr < end) {
auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(end - stride));
auto input = SIMD::load(bitwise_cast<const UnsignedType*>(end - stride));
simde_vst2q_u8(bitwise_cast<UnsignedType*>(cursorEnd - stride), (simde_uint8x16x2_t { input, zeros }));
auto quotes = WTF::equalBulk(input, quoteMask);
auto escapes = WTF::equalBulk(input, escapeMask);
auto controls = WTF::lessThanBulk(input, controlMask);
accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
auto quotes = SIMD::equal(input, quoteMask);
auto escapes = SIMD::equal(input, escapeMask);
auto controls = SIMD::lessThan(input, controlMask);
accumulated = SIMD::merge(accumulated, SIMD::merge(quotes, SIMD::merge(escapes, controls)));
}
return WTF::isNonZeroBulk(accumulated);
return SIMD::isNonZero(accumulated);
}
#endif
for (auto character : span) {
Expand Down
34 changes: 17 additions & 17 deletions Source/JavaScriptCore/runtime/LiteralParser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -872,28 +872,28 @@ ALWAYS_INLINE TokenType LiteralParser<CharType>::Lexer::lexString(LiteralParserT
constexpr size_t stride = 16 / sizeof(CharType);
using UnsignedType = std::make_unsigned_t<CharType>;
if (static_cast<size_t>(m_end - m_ptr) >= stride) {
constexpr auto quoteMask = WTF::splatBulk(static_cast<UnsignedType>('"'));
constexpr auto escapeMask = WTF::splatBulk(static_cast<UnsignedType>('\\'));
constexpr auto controlMask = WTF::splatBulk(static_cast<UnsignedType>(' '));
constexpr auto quoteMask = SIMD::splat(static_cast<UnsignedType>('"'));
constexpr auto escapeMask = SIMD::splat(static_cast<UnsignedType>('\\'));
constexpr auto controlMask = SIMD::splat(static_cast<UnsignedType>(' '));
for (; m_ptr + (stride - 1) < m_end; m_ptr += stride) {
auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(m_ptr));
auto quotes = WTF::equalBulk(input, quoteMask);
auto escapes = WTF::equalBulk(input, escapeMask);
auto controls = WTF::lessThanBulk(input, controlMask);
auto mask = WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls));
if (WTF::isNonZeroBulk(mask)) {
m_ptr += WTF::findFirstNonZeroIndexBulk(mask);
auto input = SIMD::load(bitwise_cast<const UnsignedType*>(m_ptr));
auto quotes = SIMD::equal(input, quoteMask);
auto escapes = SIMD::equal(input, escapeMask);
auto controls = SIMD::lessThan(input, controlMask);
auto mask = SIMD::merge(quotes, SIMD::merge(escapes, controls));
if (auto index = SIMD::findFirstNonZeroIndex(mask)) {
m_ptr += index.value();
return;
}
}
if (m_ptr < m_end) {
auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(m_end - stride));
auto quotes = WTF::equalBulk(input, quoteMask);
auto escapes = WTF::equalBulk(input, escapeMask);
auto controls = WTF::lessThanBulk(input, controlMask);
auto mask = WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls));
if (WTF::isNonZeroBulk(mask)) {
m_ptr = m_end - stride + WTF::findFirstNonZeroIndexBulk(mask);
auto input = SIMD::load(bitwise_cast<const UnsignedType*>(m_end - stride));
auto quotes = SIMD::equal(input, quoteMask);
auto escapes = SIMD::equal(input, escapeMask);
auto controls = SIMD::lessThan(input, controlMask);
auto mask = SIMD::merge(quotes, SIMD::merge(escapes, controls));
if (auto index = SIMD::findFirstNonZeroIndex(mask)) {
m_ptr = m_end - stride + index.value();
return;
}
m_ptr = m_end;
Expand Down
4 changes: 4 additions & 0 deletions Source/WTF/WTF.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,7 @@
E383AAC92B6C730B00058E60 /* AdaptiveStringSearcher.h in Headers */ = {isa = PBXBuildFile; fileRef = E383AAC82B6C730B00058E60 /* AdaptiveStringSearcher.h */; settings = {ATTRIBUTES = (Private, ); }; };
E388886F20C9095100E632BC /* WorkerPool.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E388886D20C9095100E632BC /* WorkerPool.cpp */; };
E38C41281EB4E0680042957D /* CPUTime.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E38C41261EB4E0680042957D /* CPUTime.cpp */; };
E38C82692BECAE9D0071EF52 /* SIMDHelpers.h in Headers */ = {isa = PBXBuildFile; fileRef = E38C82682BECAE9D0071EF52 /* SIMDHelpers.h */; settings = {ATTRIBUTES = (Private, ); }; };
E38D6E271F5522E300A75CC4 /* StringBuilderJSON.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E38D6E261F5522E300A75CC4 /* StringBuilderJSON.cpp */; };
E392FA2722E92BFF00ECDC73 /* ResourceUsageCocoa.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E392FA2622E92BFF00ECDC73 /* ResourceUsageCocoa.cpp */; };
E396C11D2BE885D9000CBAE1 /* neon.h in Headers */ = {isa = PBXBuildFile; fileRef = E396C1192BE885D9000CBAE1 /* neon.h */; settings = {ATTRIBUTES = (Private, ); }; };
Expand Down Expand Up @@ -1789,6 +1790,7 @@
E388886E20C9095100E632BC /* WorkerPool.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WorkerPool.h; sourceTree = "<group>"; };
E38C41261EB4E0680042957D /* CPUTime.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CPUTime.cpp; sourceTree = "<group>"; };
E38C41271EB4E0680042957D /* CPUTime.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CPUTime.h; sourceTree = "<group>"; };
E38C82682BECAE9D0071EF52 /* SIMDHelpers.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SIMDHelpers.h; sourceTree = "<group>"; };
E38D6E261F5522E300A75CC4 /* StringBuilderJSON.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = StringBuilderJSON.cpp; sourceTree = "<group>"; };
E392FA2622E92BFF00ECDC73 /* ResourceUsageCocoa.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ResourceUsageCocoa.cpp; sourceTree = "<group>"; };
E396C1192BE885D9000CBAE1 /* neon.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = neon.h; sourceTree = "<group>"; };
Expand Down Expand Up @@ -2371,6 +2373,7 @@
A8A47309151A825B004123FF /* SHA1.h */,
0FEB3DCE1BB5D684009D7AAD /* SharedTask.h */,
862A8D32278DE74A0014120C /* SignedPtr.h */,
E38C82682BECAE9D0071EF52 /* SIMDHelpers.h */,
A8A4730A151A825B004123FF /* SimpleStats.h */,
795212021F42588800BD6421 /* SingleRootGraph.h */,
463CCFBB2B251D77009AB04E /* SingleThreadIntegralWrapper.h */,
Expand Down Expand Up @@ -3416,6 +3419,7 @@
DDF307F227C086EA006A526F /* Signals.h in Headers */,
DD03059327B5DA0D00344002 /* SignedPtr.h in Headers */,
E3ABBABE2BE88FBE00D84916 /* simde.h in Headers */,
E38C82692BECAE9D0071EF52 /* SIMDHelpers.h in Headers */,
E361DB63289115D000B2A2B8 /* simple_decimal_conversion.h in Headers */,
DD3DC8F727A4BF8E007E5B61 /* SimpleStats.h in Headers */,
DD3DC97627A4BF8E007E5B61 /* SingleRootGraph.h in Headers */,
Expand Down
1 change: 1 addition & 0 deletions Source/WTF/wtf/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ set(WTF_PUBLIC_HEADERS
RobinHoodHashTable.h
RunLoop.h
SHA1.h
SIMDHelpers.h
SafeStrerror.h
SaturatedArithmetic.h
SchedulePair.h
Expand Down
Loading

0 comments on commit d99e74d

Please sign in to comment.