Optimize lexing on ARM with NEON

Use ARM NEON SIMD intrinsics to speed up lexing.

NEON doesn't have an equivalent to SSE's _mm_movemask_epi8, so we
implemented a different algorithm for NEON's find_first_false based on
ARM's reference memchr implementation.

Lexing comments with SIMD is still not implemented for NEON.

Before:

    Benchmark                                       Time             CPU   Iterations UserCounters...
    benchmark_lex/long_identifier_1_mean         76.9 ns         76.9 ns            3 byte=1.47969ns bytes=675.818M/s

After:

    Benchmark                                       Time             CPU   Iterations UserCounters...
    benchmark_lex/long_identifier_1_mean         53.2 ns         53.2 ns            3 byte=1023ps bytes=977.682M/s

Author: strager

src/lex.cpp

@@ -1359,7 +1359,9 @@ lexer::parsed_identifier lexer::parse_identifier(const char8* input) {
   // TODO(strager): Is the check for '\\' correct?
   QLJS_SLOW_ASSERT(this->is_identifier_byte(*input) || *input == u8'\\');
 
-#if QLJS_HAVE_X86_SSE2
+#if QLJS_HAVE_ARM_NEON
+  using char_vector = char_vector_16_neon;
+#elif QLJS_HAVE_X86_SSE2
   using char_vector = char_vector_16_sse2;
 #else
   using char_vector = char_vector_1;
@@ -1382,7 +1384,9 @@ lexer::parsed_identifier lexer::parse_identifier(const char8* input) {
                         _SIDD_CMP_RANGES | _SIDD_LEAST_SIGNIFICANT |
                             _SIDD_NEGATIVE_POLARITY | _SIDD_UBYTE_OPS);
 #else
-#if QLJS_HAVE_X86_SSE2
+#if QLJS_HAVE_ARM_NEON
+    using bool_vector = bool_vector_16_neon;
+#elif QLJS_HAVE_X86_SSE2
     using bool_vector = bool_vector_16_sse2;
 #else
     using bool_vector = bool_vector_1;

src/quick-lint-js/have.h

@@ -228,6 +228,14 @@
 #define QLJS_HAVE_CHARCONV_HEADER 0
 #endif
 
+#if !defined(QLJS_HAVE_ARM_NEON)
+#if defined(__ARM_NEON)
+#define QLJS_HAVE_ARM_NEON 1
+#else
+#define QLJS_HAVE_ARM_NEON 0
+#endif
+#endif
+
 #if !defined(QLJS_HAVE_X86_SSE2)
 #if defined(_M_AMD64) || defined(_M_X64) || \
     (defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__)

src/quick-lint-js/simd-neon-arm.h

@@ -0,0 +1,115 @@
+// Copyright (C) 2020  Matthew "strager" Glazar
+// Copyright (c) 2014-2020, Arm Limited.
+// See end of file for extended copyright information.
+
+#ifndef QUICK_LINT_JS_SIMD_NEON_ARM_H
+#define QUICK_LINT_JS_SIMD_NEON_ARM_H
+
+#include <cstdint>
+#include <quick-lint-js/bit.h>
+#include <quick-lint-js/force-inline.h>
+#include <quick-lint-js/simd.h>
+
+#if QLJS_HAVE_ARM_NEON
+#include <arm_neon.h>
+#endif
+
+// Some routines have a different copyright than the rest of quick-lint-js, thus
+// are in this separate file.
+
+namespace quick_lint_js {
+#if QLJS_HAVE_ARM_NEON
+QLJS_FORCE_INLINE inline int bool_vector_16_neon::find_first_false() const
+    noexcept {
+  // You might expect a magic pattern to look like the following:
+  //
+  //   { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x08, [repeat] }
+  //
+  // However, the above magic pattern requires mixing cells 3 times
+  // (16x8 -> 8x16 -> 4x32 -> 2x64). Our magic pattern requires mixing cells
+  // only 2 times, but creates an unusual mask (see
+  // NOTE[find_first_false NEON mask]).
+  ::uint8x16_t magic = {
+      0x01, 0x04, 0x10, 0x40, 0x01, 0x04, 0x10, 0x40,  //
+      0x01, 0x04, 0x10, 0x40, 0x01, 0x04, 0x10, 0x40,  //
+  };
+
+  // It doesn't matter what 'garbage' is. Could be zeros or ones or anything. If
+  // we ever extend this algorithm to uint8x32_t inputs, garbage would be the
+  // upper 128 bits.
+  ::uint8x16_t garbage = this->data_;
+
+  // We invert the input so that we can use countr_zero instead of countr_one.
+  // countr_one can't be used because of the zero bits in our mask (see
+  // NOTE[find_first_false NEON mask]).
+  ::uint8x16_t mixed_0 = ::vbicq_u8(magic, this->data_);
+
+  // Mix bits to create a mask. Note that arithmetic ADD is effectively
+  // bitwise OR.
+  //
+  // mixed_0: { a b c d  e f g h  i j k l  m n o p }
+  // mixed_1: { a+b c+d  e+f g+h  i+j k+l  m+n o+p  (64 bits unused...) }
+  // mixed_2: { a+b+c+d  e+f+g+h  i+j+k+l  m+n+o+p  (96 bits unused...) }
+  ::uint8x16_t mixed_1 = ::vpaddq_u8(mixed_0, garbage);
+  ::uint8x16_t mixed_2 = ::vpaddq_u8(mixed_1, mixed_1);
+  std::uint32_t mask = vgetq_lane_u32(::vreinterpretq_u32_u8(mixed_2), 0);
+
+  // NOTE[find_first_false NEON mask]: After mixing bits, an ideal mask looks
+  // like this:
+  //
+  //   0b0000000000000000ABCDEFGHIJKLMNOP
+  //
+  // But our mask looks like this:
+  //
+  //   0b0A0B0C0D0E0F0G0H0I0J0K0L0M0N0O0P
+  //
+  // To deal with the extra zeros, we to divide our countr_zero result by 2.
+  return countr_zero(mask) / 2;
+}
+#endif
+}
+
+#endif
+
+// quick-lint-js finds bugs in JavaScript programs.
+// Copyright (C) 2020  Matthew "strager" Glazar
+//
+// This file is part of quick-lint-js.
+//
+// quick-lint-js is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// quick-lint-js is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with quick-lint-js.  If not, see <https://www.gnu.org/licenses/>.
+//
+// ---
+//
+// Portions are this file are
+// Copyright (c) 2014-2020, Arm Limited.
+// Source:
+// https://github.com/ARM-software/optimized-routines/blob/7a9fd1603e1179b044406fb9b6cc5770d736cde7/string/aarch64/memchr.S
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.

src/quick-lint-js/simd.h

@@ -4,13 +4,18 @@
 #ifndef QUICK_LINT_JS_SIMD_H
 #define QUICK_LINT_JS_SIMD_H
 
+#include <cstdint>
 #include <quick-lint-js/bit.h>
 #include <quick-lint-js/char8.h>
 #include <quick-lint-js/force-inline.h>
 #include <quick-lint-js/have.h>
 #include <quick-lint-js/narrow-cast.h>
 #include <quick-lint-js/unreachable.h>
 
+#if QLJS_HAVE_ARM_NEON
+#include <arm_neon.h>
+#endif
+
 #if QLJS_HAVE_X86_SSE2
 #include <emmintrin.h>
 #endif
@@ -96,6 +101,76 @@ class alignas(__m128i) char_vector_16_sse2 {
 };
 #endif
 
+#if QLJS_HAVE_ARM_NEON
+class alignas(::uint8x16_t) bool_vector_16_neon {
+ public:
+  static constexpr int size = 16;
+
+  QLJS_FORCE_INLINE explicit bool_vector_16_neon(::uint8x16_t data) noexcept
+      : data_(data) {}
+
+  QLJS_FORCE_INLINE friend bool_vector_16_neon operator|(
+      bool_vector_16_neon x, bool_vector_16_neon y) noexcept {
+    return bool_vector_16_neon(::vorrq_u8(x.data_, y.data_));
+  }
+
+  QLJS_FORCE_INLINE friend bool_vector_16_neon operator&(
+      bool_vector_16_neon x, bool_vector_16_neon y) noexcept {
+    return bool_vector_16_neon(::vandq_u8(x.data_, y.data_));
+  }
+
+  QLJS_FORCE_INLINE int find_first_false() const noexcept;
+
+ private:
+  ::uint8x16_t data_;
+};
+
+class alignas(::uint8x16_t) char_vector_16_neon {
+ public:
+  static constexpr int size = 16;
+
+  QLJS_FORCE_INLINE explicit char_vector_16_neon(::uint8x16_t data) noexcept
+      : data_(data) {}
+
+  QLJS_FORCE_INLINE static char_vector_16_neon load(const char8* data) {
+    ::uint8x16_t vector;
+    std::memcpy(&vector, data, sizeof(vector));
+    return char_vector_16_neon(vector);
+  }
+
+  QLJS_FORCE_INLINE static char_vector_16_neon repeated(std::uint8_t c) {
+    return char_vector_16_neon(::vdupq_n_u8(c));
+  }
+
+  QLJS_FORCE_INLINE friend char_vector_16_neon operator|(
+      char_vector_16_neon x, char_vector_16_neon y) noexcept {
+    return char_vector_16_neon(::vorrq_u8(x.data_, y.data_));
+  }
+
+  QLJS_FORCE_INLINE friend bool_vector_16_neon operator==(
+      char_vector_16_neon x, char_vector_16_neon y) noexcept {
+    return bool_vector_16_neon(::vceqq_u8(x.data_, y.data_));
+  }
+
+  QLJS_FORCE_INLINE friend bool_vector_16_neon operator<(
+      char_vector_16_neon x, char_vector_16_neon y) noexcept {
+    return bool_vector_16_neon(::vcltq_u8(x.data_, y.data_));
+  }
+
+  QLJS_FORCE_INLINE friend bool_vector_16_neon operator>(
+      char_vector_16_neon x, char_vector_16_neon y) noexcept {
+    return bool_vector_16_neon(::vcgtq_u8(x.data_, y.data_));
+  }
+
+  QLJS_FORCE_INLINE ::uint8x16_t uint8x16() const noexcept {
+    return this->data_;
+  }
+
+ private:
+  ::uint8x16_t data_;
+};
+#endif
+
 class bool_vector_1 {
  public:
   static constexpr int size = 1;
@@ -164,6 +239,9 @@ class char_vector_1 {
 };
 }
 
+// Some routines have a different copyright, thus are in a separate file.
+#include <quick-lint-js/simd-neon-arm.h>
+
 #endif
 
 // quick-lint-js finds bugs in JavaScript programs.

test/test-monotonic-allocator.cpp

@@ -12,6 +12,10 @@
 
 QLJS_WARNING_IGNORE_GCC("-Wsuggest-override")
 
+#if QLJS_HAVE_ARM_NEON
+#include <arm_neon.h>
+#endif
+
 #if QLJS_HAVE_X86_SSE2
 #include <emmintrin.h>
 #endif
@@ -23,6 +27,10 @@ QLJS_WARNING_IGNORE_GCC("-Wsuggest-override")
 #define QLJS_HAVE_OVERALIGNED 0
 #endif
 
+// TODO(strager): For some reason, allocating a ::uint8x16_t crashes with
+// std::bad_alloc.
+#define QLJS_CAN_ALIGN_NEON 0
+
 #if defined(_MSC_VER)
 // TODO(strager): For some reason, allocating a __m128 crashes with
 // std::bad_alloc.
@@ -101,6 +109,13 @@ TEST(test_monotonic_allocator, filling_first_chunk_allocates_second_chunk) {
 struct alignas(std::max_align_t) overaligned_64 {};
 #endif
 
+#if QLJS_CAN_ALIGN_NEON
+struct neon_aligned {
+  ::uint8x16_t uint8x16;
+  ::uint64x2_t uint64x2;
+};
+#endif
+
 #if QLJS_CAN_ALIGN_SSE
 struct sse_aligned {
   __m128 m128;
@@ -112,6 +127,9 @@ using test_monotonic_allocator_typed_types = ::testing::Types<
 #if QLJS_HAVE_OVERALIGNED
     overaligned_64,
 #endif
+#if QLJS_CAN_ALIGN_NEON
+    neon_aligned,
+#endif
 #if QLJS_CAN_ALIGN_SSE
     sse_aligned,
 #endif

test/test-simd.cpp

@@ -9,6 +9,10 @@
 #include <quick-lint-js/have.h>
 #include <quick-lint-js/simd.h>
 
+#if QLJS_HAVE_ARM_NEON
+#include <arm_neon.h>
+#endif
+
 #if QLJS_HAVE_X86_SSE2
 #include <emmintrin.h>
 #endif
@@ -43,6 +47,72 @@ TEST(test_char_vector_16_sse2, bitwise_or) {
   EXPECT_EQ(std::memcmp(&actual, expected, sizeof(actual)), 0);
 }
 #endif
+
+#if QLJS_HAVE_ARM_NEON
+TEST(test_char_vector_16_neon, repeated) {
+  char8 expected[16];
+  std::fill(std::begin(expected), std::end(expected), u8'x');
+  ::uint8x16_t actual = char_vector_16_neon::repeated('x').uint8x16();
+  EXPECT_EQ(std::memcmp(&actual, expected, sizeof(actual)), 0);
+}
+
+TEST(test_char_vector_16_neon, bitwise_or) {
+  constexpr std::uint8_t lhs[16] = {
+      0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,  //
+      0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,  //
+  };
+  constexpr std::uint8_t rhs[16] = {
+      0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80,  //
+      0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80,  //
+  };
+  ::uint8x16_t actual =
+      (char_vector_16_neon::load(reinterpret_cast<const char8*>(lhs)) |
+       char_vector_16_neon::load(reinterpret_cast<const char8*>(rhs)))
+          .uint8x16();
+  constexpr std::uint8_t expected[16] = {
+      0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,  //
+      0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,  //
+  };
+  EXPECT_EQ(std::memcmp(&actual, expected, sizeof(actual)), 0);
+}
+
+TEST(test_bool_vector_16_neon, first_false_of_all_false) {
+  ::uint8x16_t bools_data = {
+      0, 0, 0, 0, 0, 0, 0, 0,  //
+      0, 0, 0, 0, 0, 0, 0, 0,  //
+  };
+  bool_vector_16_neon bools(bools_data);
+  EXPECT_EQ(bools.find_first_false(), 0);
+}
+
+TEST(test_bool_vector_16_neon, first_false_of_all_true) {
+  constexpr std::uint8_t t = 0xff;
+  ::uint8x16_t bools_data = {
+      t, t, t, t, t, t, t, t,  //
+      t, t, t, t, t, t, t, t,  //
+  };
+  bool_vector_16_neon bools(bools_data);
+  EXPECT_EQ(bools.find_first_false(), 16);
+}
+
+TEST(test_bool_vector_16_neon, find_first_false_exhaustive) {
+  for (std::uint32_t i = 0; i <= 0xffff; ++i) {
+    SCOPED_TRACE(i);
+    ::uint8x16_t bools_data;
+    int first_false = 16;
+    for (int bit = 0; bit < 16; ++bit) {
+      bool bit_on = (i >> bit) & 1;
+      bools_data[bit] = bit_on ? 0xff : 0x00;
+      if (!bit_on) {
+        first_false = std::min(first_false, bit);
+      }
+    }
+
+    bool_vector_16_neon bools(bools_data);
+    EXPECT_EQ(bools.find_first_false(), first_false);
+  }
+}
+#endif
 }
 }