Make benchmark more like Stockfish bench (#1069)

* Make benchmark more like Stockfish bench

Add some of the SF bench positions, iterate over them while collecting nodes and times, report time and node count summary at the end.

* Fix whitespace

Spaces instead of tabs.

* Properly handle nodes per second reporting

Handle division by zero and proper rounding.

* Fix C4596 error

Fix illegal qualified name in member declaration error by removing redundant Benchmark::.

* Add support for old bench of only starting position

Use the --starting-position flag to change the position set to only be the starting position so that scripts that use benchmark can run as they always have.

* Update the check scripts to use only the starting position in bench

Bench only the starting position to save time.

* Fix assignment

Allow assignment for starting position.

* Fix variable names

Fix the variables names to conform to the contributor style.

* Remove positions which inflate nps

Remove TB endgame positions which inflate nps greatly.

* Update the dx check script to use only the starting position in bench

Bench only the starting position to save time.

* Change how positions are chosen

Allow for choosing more than just the starting position. Allow choice of the first number of positions in the list of positions to allow for faster benching.

* Update the check scripts to use the new benchmark option

Use --num-positions=1 instead of --starting-position.

* Update our PGO AppVeyor build to use the new benchmark

For now only bench the starting position.

* Re-add support for benchmarking a FEN string

If provided a FEN string, only test that one position irrespective of --num-positions which is forced to one.

* Revert "Update our PGO AppVeyor build to use the new benchmark"

This reverts commit dd48294.
We no longer do PGO builds and no longer do benching here.

* Update our PGO AppVeyor build to use the new benchmark

For now only bench the starting position.

* Adjust header ordering to conform to style

Conform to the style of other code.

scripts/appveyor_win_build.cmd

@@ -10,7 +10,7 @@ IF %PGO%==true (
   IF %OPENCL%==true copy C:\cache\opencl-nug.0.777.77\build\native\bin\OpenCL.dll
   IF %CUDA%==true copy "%CUDA_PATH%"\bin\*.dll
   IF %CUDA%==true copy %PKG_FOLDER%\cuda\bin\cudnn64_7.dll
-  lc0 benchmark --weights=c:\cache\591226.pb.gz --backend=random --movetime=10000
+  lc0 benchmark --num-positions=1 --weights=c:\cache\591226.pb.gz --backend=random --movetime=10000
 )
 cd ..
 IF %PGO%==true msbuild "C:\projects\lc0\build\lc0.sln" /m /p:WholeProgramOptimization=PGOptimize /p:DebugInformationFormat=ProgramDatabase /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"

scripts/check_dx.bat

@@ -1,5 +1,5 @@
 @ECHO OFF
 ECHO Sanity checking the dx12 driver.
-lc0 benchmark --backend=check --backend-opts=mode=check,freq=1.0,atol=5e-1,dx12 %*
+lc0 benchmark --num-positions=1 --backend=check --backend-opts=mode=check,freq=1.0,atol=5e-1,dx12 %*
 PAUSE
 

scripts/check_opencl.bat

@@ -1,5 +1,5 @@
 @ECHO OFF
 ECHO Sanity checking the opencl driver.
-lc0 benchmark --backend=check --backend-opts=mode=check,freq=1.0,opencl %*
+lc0 benchmark --num-positions=1 --backend=check --backend-opts=mode=check,freq=1.0,opencl %*
 PAUSE
 

src/benchmark/benchmark.cc

@@ -27,6 +27,8 @@
 
 #include "benchmark/benchmark.h"
 
+#include <numeric>
+
 #include "mcts/search.h"
 #include "mcts/stoppers/factory.h"
 #include "mcts/stoppers/stoppers.h"
@@ -40,7 +42,9 @@ const OptionId kThreadsOptionId{"threads", "Threads",
 const OptionId kNodesId{"nodes", "", "Number of nodes to run as a benchmark."};
 const OptionId kMovetimeId{"movetime", "",
                            "Benchmark time allocation, in milliseconds."};
-const OptionId kFenId{"fen", "", "Benchmark initial position FEN."};
+const OptionId kFenId{"fen", "", "Benchmark position FEN."};
+const OptionId kNumPositionsId{"num-positions", "",
+                               "The number of benchmark positions to test."};
 }  // namespace
 
 void Benchmark::Run() {
@@ -52,7 +56,8 @@ void Benchmark::Run() {
 
   options.Add<IntOption>(kNodesId, -1, 999999999) = -1;
   options.Add<IntOption>(kMovetimeId, -1, 999999999) = 10000;
-  options.Add<StringOption>(kFenId) = ChessBoard::kStartposFen;
+  options.Add<StringOption>(kFenId) = "";
+  options.Add<IntOption>(kNumPositionsId, 1, 34) = 34;
 
   if (!options.ProcessAllFlags()) return;
 
@@ -61,41 +66,67 @@ void Benchmark::Run() {
 
     auto network = NetworkFactory::LoadNetwork(option_dict);
 
-    NodeTree tree;
-    tree.ResetToPosition(option_dict.Get<std::string>(kFenId), {});
-
-    NNCache cache;
-    cache.SetCapacity(option_dict.Get<int>(kNNCacheSizeId));
-
-    int visits = option_dict.Get<int>(kNodesId);
+    const int visits = option_dict.Get<int>(kNodesId);
     const int movetime = option_dict.Get<int>(kMovetimeId);
+    const std::string fen = option_dict.Get<std::string>(kFenId);
+    int num_positions = option_dict.Get<int>(kNumPositionsId);
+
+    std::vector<std::double_t> times;
+    std::vector<std::int64_t> playouts;
+    std::uint64_t cnt = 1;
 
-    auto stopper = std::make_unique<ChainedSearchStopper>();
-    if (movetime > -1) {
-      stopper->AddStopper(std::make_unique<TimeLimitStopper>(movetime));
+    if (fen.length() > 0) {
+      positions = {fen};
+      num_positions = 1;
     }
-    if (visits > -1) {
-      stopper->AddStopper(std::make_unique<VisitsStopper>(visits));
+    std::vector<std::string> testing_positions(
+        positions.cbegin(), positions.cbegin() + num_positions);
+
+    for (std::string position : testing_positions) {
+      std::cout << "\nPosition: " << cnt++ << "/" << testing_positions.size()
+                << " " << position << std::endl;
+
+      auto stopper = std::make_unique<ChainedSearchStopper>();
+      if (movetime > -1) {
+        stopper->AddStopper(std::make_unique<TimeLimitStopper>(movetime));
+      }
+      if (visits > -1) {
+        stopper->AddStopper(std::make_unique<VisitsStopper>(visits));
+      }
+
+      NNCache cache;
+      cache.SetCapacity(option_dict.Get<int>(kNNCacheSizeId));
+
+      NodeTree tree;
+      tree.ResetToPosition(position, {});
+
+      const auto start = std::chrono::steady_clock::now();
+      auto search = std::make_unique<Search>(
+          tree, network.get(),
+          std::make_unique<CallbackUciResponder>(
+              std::bind(&Benchmark::OnBestMove, this, std::placeholders::_1),
+              std::bind(&Benchmark::OnInfo, this, std::placeholders::_1)),
+          MoveList(), start, std::move(stopper), false, option_dict, &cache,
+          nullptr);
+      search->StartThreads(option_dict.Get<int>(kThreadsOptionId));
+      search->Wait();
+      const auto end = std::chrono::steady_clock::now();
+
+      const auto time =
+          std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+      times.push_back(time.count());
+      playouts.push_back(search->GetTotalPlayouts());
     }
 
-    const auto start = std::chrono::steady_clock::now();
-    auto search = std::make_unique<Search>(
-        tree, network.get(),
-        std::make_unique<CallbackUciResponder>(
-            std::bind(&Benchmark::OnBestMove, this, std::placeholders::_1),
-            std::bind(&Benchmark::OnInfo, this, std::placeholders::_1)),
-        MoveList(), start, std::move(stopper), false, option_dict, &cache,
-        nullptr);
-
-    search->StartThreads(option_dict.Get<int>(kThreadsOptionId));
-
-    search->Wait();
-
-    const auto end = std::chrono::steady_clock::now();
-    std::chrono::duration<double> time = end - start;
-    std::cout << "Benchmark final time " << time.count() << "s calculating "
-              << search->GetTotalPlayouts() / time.count()
-              << " nodes per second." << std::endl;
+    const auto total_playouts =
+        std::accumulate(playouts.begin(), playouts.end(), 0);
+    const auto total_time = std::accumulate(times.begin(), times.end(), 0);
+    std::cout << "\n==========================="
+              << "\nTotal time (ms) : " << total_time
+              << "\nNodes searched  : " << total_playouts
+              << "\nNodes/second    : "
+              << std::lround(1000.0 * total_playouts / (total_time + 1))
+              << std::endl;
   } catch (Exception& ex) {
     std::cerr << ex.what() << std::endl;
   }

src/benchmark/benchmark.h

@@ -38,6 +38,46 @@ class Benchmark{
  public:
   Benchmark() = default;
 
+  // Same positions as Stockfish uses.
+  std::vector<std::string> positions = {
+      "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1",
+      "r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq - 0 10",
+      "8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - - 0 11",
+      "4rrk1/pp1n3p/3q2pQ/2p1pb2/2PP4/2P3N1/P2B2PP/4RRK1 b - - 7 19",
+      "rq3rk1/ppp2ppp/1bnpb3/3N2B1/3NP3/7P/PPPQ1PP1/2KR3R w - - 7 14 moves "
+      "d4e6",
+      "r1bq1r1k/1pp1n1pp/1p1p4/4p2Q/4Pp2/1BNP4/PPP2PPP/3R1RK1 w - - 2 14 moves "
+      "g2g4",
+      "r3r1k1/2p2ppp/p1p1bn2/8/1q2P3/2NPQN2/PPP3PP/R4RK1 b - - 2 15",
+      "r1bbk1nr/pp3p1p/2n5/1N4p1/2Np1B2/8/PPP2PPP/2KR1B1R w kq - 0 13",
+      "r1bq1rk1/ppp1nppp/4n3/3p3Q/3P4/1BP1B3/PP1N2PP/R4RK1 w - - 1 16",
+      "4r1k1/r1q2ppp/ppp2n2/4P3/5Rb1/1N1BQ3/PPP3PP/R5K1 w - - 1 17",
+      "2rqkb1r/ppp2p2/2npb1p1/1N1Nn2p/2P1PP2/8/PP2B1PP/R1BQK2R b KQ - 0 11",
+      "r1bq1r1k/b1p1npp1/p2p3p/1p6/3PP3/1B2NN2/PP3PPP/R2Q1RK1 w - - 1 16",
+      "3r1rk1/p5pp/bpp1pp2/8/q1PP1P2/b3P3/P2NQRPP/1R2B1K1 b - - 6 22",
+      "r1q2rk1/2p1bppp/2Pp4/p6b/Q1PNp3/4B3/PP1R1PPP/2K4R w - - 2 18",
+      "4k2r/1pb2ppp/1p2p3/1R1p4/3P4/2r1PN2/P4PPP/1R4K1 b - - 3 22",
+      "3q2k1/pb3p1p/4pbp1/2r5/PpN2N2/1P2P2P/5PP1/Q2R2K1 b - - 4 26",
+      "6k1/6p1/6Pp/ppp5/3pn2P/1P3K2/1PP2P2/3N4 b - - 0 1",
+      "3b4/5kp1/1p1p1p1p/pP1PpP1P/P1P1P3/3KN3/8/8 w - - 0 1",
+      "2K5/p7/7P/5pR1/8/5k2/r7/8 w - - 0 1 moves g5g6 f3e3 g6g5 e3f3",
+      "8/6pk/1p6/8/PP3p1p/5P2/4KP1q/3Q4 w - - 0 1",
+      "7k/3p2pp/4q3/8/4Q3/5Kp1/P6b/8 w - - 0 1",
+      "8/2p5/8/2kPKp1p/2p4P/2P5/3P4/8 w - - 0 1",
+      "8/1p3pp1/7p/5P1P/2k3P1/8/2K2P2/8 w - - 0 1",
+      "8/pp2r1k1/2p1p3/3pP2p/1P1P1P1P/P5KR/8/8 w - - 0 1",
+      "8/3p4/p1bk3p/Pp6/1Kp1PpPp/2P2P1P/2P5/5B2 b - - 0 1",
+      "5k2/7R/4P2p/5K2/p1r2P1p/8/8/8 b - - 0 1",
+      "6k1/6p1/P6p/r1N5/5p2/7P/1b3PP1/4R1K1 w - - 0 1",
+      "1r3k2/4q3/2Pp3b/3Bp3/2Q2p2/1p1P2P1/1P2KP2/3N4 w - - 0 1",
+      "6k1/4pp1p/3p2p1/P1pPb3/R7/1r2P1PP/3B1P2/6K1 w - - 0 1",
+      "8/3p3B/5p2/5P2/p7/PP5b/k7/6K1 w - - 0 1",
+      "5rk1/q6p/2p3bR/1pPp1rP1/1P1Pp3/P3B1Q1/1K3P2/R7 w - - 93 90",
+      "4rrk1/1p1nq3/p7/2p1P1pp/3P2bp/3Q1Bn1/PPPB4/1K2R1NR w - - 40 21",
+      "r3k2r/3nnpbp/q2pp1p1/p7/Pp1PPPP1/4BNN1/1P5P/R2Q1RK1 w kq - 0 16",
+      "3Qb1k1/1r2ppb1/pN1n2q1/Pp1Pp1Pr/4P2p/4BP2/4B1R1/1R5K b - - 11 40"
+  };
+
   void Run();
   void OnBestMove(const BestMoveInfo& move);
   void OnInfo(const std::vector<ThinkingInfo>& infos);