Solve midgames and on locally via bare WebAssembly

The pentago javascript client now solves boards with 18 or more stones
locally via bare WebAssembly code running in a WebWorker.  Bare
WebAssembly means clang-only, without Emscripten.  That in turn means no
standard library, no malloc, etc.

The midengine was already pretty light on memory allocation, so the main
hardship was getting a stripped version of libc++ (for type_traits and
similar niceties) that works in -ffreestanding mode without malloc and
such.  I did this by copying the libc++ headers into
third_party/freestanding and adding a very large number of ifdefs.

A wrinkle: the midengine is memory heavy, and WebAssembly instances
can't ever shrink their overall memory size.  To get around this, the
client makes a fresh WebAssembly instance for each request and throws it
away when compile.  Thankfully the API separates compilation from
instance creation, so this seems fine.

Timings for an 18 stone board (https://perfect-pentago.net/#274440791932540184):

  MacBook Pro, 13-inch, 2019, 2.5 GHz Quad-Core Intel Core i5:
    Chrome 79 = 12.381 s
    Firefox 71 = 10.587 s
    Safari 13.0.4 = 12.572 s
  iPhone 11 Pro, iOS 13.3:
    9.0 s (!)

Other improvements along the way:

1. Remove the weird "boards" argument to midengine routines.  All
   available boards are now computed inside the routines.
2. Reduce excessive templatization of midengine to help code size.
   This results in a small but acceptable 10-15% slowdown.
3. Make midengine work without SSE.
4. Make high_board_t smaller (16 -> 8 bytes) and leaner (no memory
   allocation).
5. Upgrade server to node 13.5.0.  Extensive and painful binding tweaks
   required.
6. Use fetch instead of XMLHttpRequest in client.

Posts which gave me the idea that bare WebAssembly was possible:

* https://dassur.ma/things/c-to-webassembly
* https://medium.com/@dougschaefer/going-straight-to-clang-for-webassembly-928df1484430

pentago/base/board.cc

@@ -4,17 +4,15 @@
 #include "pentago/utility/debug.h"
 #include "pentago/utility/array.h"
 #include "pentago/utility/popcount.h"
-#include "pentago/utility/random.h"
 #include "pentago/utility/range.h"
 #include "pentago/utility/format.h"
-#include <cmath>
-#include <vector>
+#ifndef __wasm__
+#include "pentago/utility/random.h"
+#endif
 namespace pentago {
 
 using std::min;
-using std::pow;
 using std::swap;
-using std::vector;
 
 bool black_to_move(board_t board) {
   check_board(board);
@@ -28,11 +26,47 @@ bool black_to_move(board_t board) {
 
 void check_board(board_t board) {
   #define CHECK(q) \
-    if (!(quadrant(board,q)<(int)pow(3.,9.))) \
+    if (!(quadrant(board,q) < 19683)) \
       THROW(ValueError,"quadrant %d has invalid value %d",q,quadrant(board,q));
   CHECK(0) CHECK(1) CHECK(2) CHECK(3)
 }
 
+static inline board_t pack(const Vector<Vector<quadrant_t,2>,4>& sides) {
+    return quadrants(pack(sides[0][0],sides[0][1]),
+                     pack(sides[1][0],sides[1][1]),
+                     pack(sides[2][0],sides[2][1]),
+                     pack(sides[3][0],sides[3][1]));
+}
+
+// Rotate and reflect a board to minimize its value
+board_t standardize(board_t board) {
+  Vector<Vector<quadrant_t,2>,4> sides;
+  for (int q=0;q<4;q++) for (int s=0;s<2;s++)
+    sides[q][s] = unpack(quadrant(board,q),s);
+  board_t transformed[8];
+  for (int rotation=0;rotation<4;rotation++) {
+    for (int reflection=0;reflection<2;reflection++) {
+      transformed[2*rotation+reflection] = pack(sides);
+      // Reflect about x = y line
+      for (int q=0;q<4;q++)
+        for (int s=0;s<2;s++)
+          sides[q][s] = reflections[sides[q][s]];
+      swap(sides[0],sides[3]);
+    }
+    // Rotate left
+    for (int q=0;q<4;q++)
+      for (int s=0;s<2;s++)
+        sides[q][s] = rotations[sides[q][s]][0];
+    Vector<Vector<quadrant_t,2>,4> prev = sides;
+    sides[0] = prev[1];
+    sides[1] = prev[3];
+    sides[2] = prev[0];
+    sides[3] = prev[2];
+  }
+  return RawArray<board_t>(8,transformed).min();
+}
+
+#ifndef __wasm__
 Array<int,2> to_table(const board_t board) {
   check_board(board);
   Array<int,2> table(6, 6, uninit);
@@ -70,41 +104,6 @@ board_t from_table(RawArray<const int,2> table) {
   return quadrants(q[0], q[1], q[2], q[3]);
 }
 
-static inline board_t pack(const Vector<Vector<quadrant_t,2>,4>& sides) {
-    return quadrants(pack(sides[0][0],sides[0][1]),
-                     pack(sides[1][0],sides[1][1]),
-                     pack(sides[2][0],sides[2][1]),
-                     pack(sides[3][0],sides[3][1]));
-}
-
-// Rotate and reflect a board to minimize its value
-board_t standardize(board_t board) {
-  Vector<Vector<quadrant_t,2>,4> sides;
-  for (int q=0;q<4;q++) for (int s=0;s<2;s++)
-    sides[q][s] = unpack(quadrant(board,q),s);
-  board_t transformed[8];
-  for (int rotation=0;rotation<4;rotation++) {
-    for (int reflection=0;reflection<2;reflection++) {
-      transformed[2*rotation+reflection] = pack(sides);
-      // Reflect about x = y line
-      for (int q=0;q<4;q++)
-        for (int s=0;s<2;s++)
-          sides[q][s] = reflections[sides[q][s]];
-      swap(sides[0],sides[3]);
-    }
-    // Rotate left
-    for (int q=0;q<4;q++)
-      for (int s=0;s<2;s++)
-        sides[q][s] = rotations[sides[q][s]][0];
-    Vector<Vector<quadrant_t,2>,4> prev = sides;
-    sides[0] = prev[1];
-    sides[1] = prev[3];
-    sides[2] = prev[0];
-    sides[3] = prev[2];
-  }
-  return RawArray<board_t>(8,transformed).min();
-}
-
 side_t random_side(Random& random) {
   return random.bits<uint64_t>()&side_mask;
 }
@@ -146,7 +145,6 @@ board_t random_board(Random& random, int n) {
   return pack(black,white);
 }
 
-#ifndef __EMSCRIPTEN__
 string str_board(board_t board) {
   string s;
   s += format("counts: 0s = %d, 1s = %d\n\n",popcount(unpack(board,0)),popcount(unpack(board,1)));
@@ -161,6 +159,6 @@ string str_board(board_t board) {
   }
   return s+"\n   123456";
 }
-#endif
+#endif  // !__wasm__
 
 }

pentago/base/board.h

@@ -8,15 +8,15 @@
 #pragma once
 
 #include <cassert>
-#include <string>
 #include "pentago/utility/array.h"
 #include "pentago/utility/popcount.h"
-#include "pentago/utility/random.h"
 #include "pentago/base/gen/tables.h"
+#ifndef __wasm__
+#include "pentago/utility/random.h"
+#include <string>
+#endif
 namespace pentago {
 
-using std::string;
-
 // Each board is divided into 4 quadrants, and each quadrant is stored
 // in one of the 16-bit quarters of a 64-bit int.  Within a quadrant,
 // the state is packed in radix 3, which works since 3**9 < 2**16.
@@ -94,24 +94,24 @@ void check_board(board_t board);
 
 board_t standardize(board_t board);
 
+// Maybe swap sides
+static inline board_t flip_board(board_t board, bool turn = true) {
+  return pack(unpack(board,turn),unpack(board,1-turn));
+}
+
+#ifndef __wasm__
 // Random board and side generation
 side_t random_side(Random& random);
 board_t random_board(Random& random);
 
 // Generate a random board with n stones
 board_t random_board(Random& random, int n);
 
-// Maybe swap sides
-static inline board_t flip_board(board_t board, bool turn = true) {
-  return pack(unpack(board,turn),unpack(board,1-turn));
-}
-
-#ifndef __EMSCRIPTEN__
-string str_board(board_t board);
-#endif
+std::string str_board(board_t board);
 
 // Turn a board into a 6x6 grid: x-y major order, 0,0 is lower left, value 0 for empty or 2^k for player k
 Array<int,2> to_table(const board_t boards);
 board_t from_table(RawArray<const int,2> tables);
+#endif  // !__wasm__
 
 }

pentago/base/superscore.cc

@@ -4,13 +4,14 @@
 #include "pentago/base/score.h"
 #include "pentago/utility/array.h"
 #include "pentago/utility/debug.h"
-#include "pentago/utility/random.h"
 #include "pentago/utility/range.h"
-#include <cmath>
 #include <numeric>
+#ifndef __wasm__
+#include "pentago/utility/random.h"
+#include <cmath>
+#endif  // !__wasm__
 namespace pentago {
 
-using std::abs;
 using std::min;
 using std::numeric_limits;
 using std::swap;
@@ -101,6 +102,14 @@ const Vector<int,4> single_rotations[8] = {
     vec(0,0,1,0),vec(0,0,-1,0),vec(0,0,0,1),vec(0,0,0,-1)
 };
 
+uint8_t first(super_t s) {
+  for (int r=0;r<256;r++)
+    if (s(r))
+      return r;
+  THROW(ValueError,"zero passed to super_t first");
+}
+
+#ifndef __wasm__
 super_t random_super(Random& random) {
   const uint64_t r0 = random.bits<uint64_t>(),
                  r1 = random.bits<uint64_t>(),
@@ -143,13 +152,6 @@ ostream& operator<<(ostream& output, super_t s) {
   return output<<i;
 }
 
-uint8_t first(super_t s) {
-  for (int r=0;r<256;r++)
-    if (s(r))
-      return r;
-  THROW(ValueError,"zero passed to super_t first");
-}
-
 uint64_t super_popcount(NdArray<const super_t> data) {
   uint64_t sum = 0;
   for (auto& s : data.flat())
@@ -163,5 +165,6 @@ NdArray<int> super_popcounts(NdArray<const super_t> data) {
     counts.flat()[i] = popcount(data.flat()[i]);
   return counts;
 }
+#endif  // !__wasm__
 
 }

pentago/base/superscore.h

@@ -35,8 +35,6 @@ namespace pentago {
 #endif
 #endif
 
-using std::ostream;
-
 struct zero {};
 
 // A subset of the rotation group Z_4^4 represented as a 256 bit mask.
@@ -266,11 +264,6 @@ extern const Vector<int,4> single_rotations[8];
 
 uint8_t first(super_t s);
 
-super_t random_super(Random& random);
-
-ostream& operator<<(ostream& output, super_t s);
-ostream& operator<<(ostream& output, superinfo_t s);
-
 int popcount(super_t s);
 
 #if PENTAGO_SSE // SSE version of rmax
@@ -334,12 +327,19 @@ static inline super_t endian_reverse(const super_t s) {
 }
 #endif
 
+#ifndef __wasm__
+super_t random_super(Random& random);
+
+std::ostream& operator<<(std::ostream& output, super_t s);
+std::ostream& operator<<(std::ostream& output, superinfo_t s);
+
 uint64_t super_popcount(NdArray<const super_t> data);
 NdArray<int> super_popcounts(NdArray<const super_t> data);
 Array<super_t> random_supers(const uint128_t key, const int size);
 
 template<int d> Array<super_t,d> random_supers(const uint128_t key, const Vector<int,d> shape) {
   return random_supers(key, shape.product()).reshape_own(shape);
 }
+#endif  // !__wasm__
 
 }  // namespace pentago

pentago/base/symmetry.cc

@@ -4,30 +4,20 @@
 #include "pentago/base/hash.h"
 #include "pentago/utility/array.h"
 #include "pentago/utility/integer_log.h"
-#include "pentago/utility/random.h"
 #include "pentago/utility/range.h"
-#include <unordered_set>
+#ifndef __wasm__
+#include "pentago/utility/random.h"
+#endif
 namespace pentago {
 
 using std::get;
 using std::make_tuple;
 using std::min;
 using std::swap;
 using std::tuple;
-using std::unordered_set;
 
 const symmetries_t symmetries;
 
-#ifndef __EMSCRIPTEN__
-ostream& operator<<(ostream& output, symmetry_t s) {
-  return output<<format("(%d,%d,%d=%d%d%d%d)",s.global>>2,s.global&3,s.local,s.local&3,s.local>>2&3,s.local>>4&3,s.local>>6);
-}
-
-ostream& operator<<(ostream& output, local_symmetry_t s) {
-  return output<<format("(%d=%d%d%d%d)",s.local,s.local&3,s.local>>2&3,s.local>>4&3,s.local>>6);
-}
-#endif
-
 // rotate_quadrants[r][q] is the quadrant moved to q under rotation r
 static const uint8_t rotate_quadrants[4][4] = {{0,1,2,3},{1,3,0,2},{3,2,1,0},{2,0,3,1}};
 
@@ -87,11 +77,6 @@ board_t transform_board(symmetry_t s, board_t board) {
   return quadrants(pack(qr[0][0],qr[0][1]),pack(qr[1][0],qr[1][1]),pack(qr[2][0],qr[2][1]),pack(qr[3][0],qr[3][1]));
 }
 
-symmetry_t random_symmetry(Random& random) {
-  const int g = random.uniform<int>(0,symmetries.size());
-  return symmetry_t(g>>8,g&255);
-}
-
 tuple<board_t,symmetry_t> superstandardize(board_t board) {
   return superstandardize(unpack(board,0),unpack(board,1));
 }
@@ -333,4 +318,19 @@ super_t super_meaningless(const board_t board, const uint64_t salt) {
   return result;
 }
 
+#ifndef __wasm__
+symmetry_t random_symmetry(Random& random) {
+  const int g = random.uniform<int>(0,symmetries.size());
+  return symmetry_t(g>>8,g&255);
+}
+
+ostream& operator<<(ostream& output, symmetry_t s) {
+  return output<<format("(%d,%d,%d=%d%d%d%d)",s.global>>2,s.global&3,s.local,s.local&3,s.local>>2&3,s.local>>4&3,s.local>>6);
+}
+
+ostream& operator<<(ostream& output, local_symmetry_t s) {
+  return output<<format("(%d=%d%d%d%d)",s.local,s.local&3,s.local>>2&3,s.local>>4&3,s.local>>6);
+}
+#endif  // !__wasm__
+
 }

pentago/base/symmetry.h

@@ -12,7 +12,6 @@
 #include "pentago/base/superscore.h"
 namespace pentago {
 
-using std::ostream;
 using std::tuple;
 
 // A purely local symmetry
@@ -121,9 +120,11 @@ tuple<board_t,symmetry_t> superstandardize(side_t side0, side_t side1) __attribu
 bool meaningless(board_t board, uint64_t salt=0) __attribute__((const));
 super_t super_meaningless(board_t board, uint64_t salt=0) __attribute__((const));
 
-ostream& operator<<(ostream& output, symmetry_t s);
-ostream& operator<<(ostream& output, local_symmetry_t s);
+#ifndef __wasm__
+std::ostream& operator<<(std::ostream& output, symmetry_t s);
+std::ostream& operator<<(std::ostream& output, local_symmetry_t s);
 symmetry_t random_symmetry(Random& random);
+#endif  // !__wasm__
 
 // Convenient enumeration of all symmetries
 struct symmetries_t {

pentago/data/async_block_cache.cc

@@ -13,10 +13,10 @@ async_block_cache_t::async_block_cache_t(const uint64_t memory_limit)
 
 async_block_cache_t::~async_block_cache_t() {}
 
-block_t async_block_cache_t::board_block(const high_board_t& board) {
-  GEODE_ASSERT(!board.middle);
+block_t async_block_cache_t::board_block(const high_board_t board) {
+  GEODE_ASSERT(!board.middle());
   // Account for global symmetries
-  const auto flip_board = board.board; // Unlike block_cache_t::lookup, we don't need to flip the board
+  const auto flip_board = board.board();  // Unlike block_cache_t::lookup, we don't need to flip the board
   const auto section = count(flip_board).standardize<8>();
 
   // More global symmetries

pentago/data/async_block_cache.h

@@ -31,7 +31,7 @@ struct async_block_cache_t : public block_cache_t {
   // Look up the section and block containing a given board.
   // Code copied from block_cache_t::lookup due to laziness.
   // The only difference is that the board isn't flipped.
-  static block_t board_block(const high_board_t& board);
+  static block_t board_block(const high_board_t board);
 
   bool contains(const block_t block) const;
   unit_t set(const block_t block, RawArray<const uint8_t> compressed);