Add OpenBench support (I) (#339)

* Add Threads UCI option * Add `bench` support for both UCI and as executable parameter * Add EXE and CC fields to makefile * Follow most accepted standard for node count: after legal move
lynx-chess · Aug 14, 2023 · 7feb991 · 7feb991
1 parent 6b3196f
commit 7feb991
Show file tree

Hide file tree

Showing 15 changed files with 214 additions and 47 deletions.
diff --git a/Makefile b/Makefile
@@ -1,5 +1,8 @@
 .DEFAULT_GOAL := publish
 
+EXE = Lynx.Cli
+CC = dotnet
+
 RUNTIME=
 OUTPUT_DIR:=artifacts/Lynx/
 
@@ -39,7 +42,7 @@ test:
 	dotnet test -c Release & dotnet test -c Release --filter "TestCategory=LongRunning" & dotnet test -c Release --filter "TestCategory=Perft"
 
 publish:
-	dotnet publish src/Lynx.Cli/Lynx.Cli.csproj -c Release --runtime ${RUNTIME} --self-contained /p:Optimized=true -o ${OUTPUT_DIR}
+	dotnet publish src/Lynx.Cli/Lynx.Cli.csproj -c Release --runtime ${RUNTIME} --self-contained /p:Optimized=true /p:DeterministicBuild=true -o ${OUTPUT_DIR}
 
 run:
 	dotnet run --project src/Lynx.Cli/Lynx.Cli.csproj -c Release --runtime ${RUNTIME}
diff --git a/src/Lynx.Cli/Program.cs b/src/Lynx.Cli/Program.cs
@@ -1,10 +1,13 @@
 using Lynx;
 using Lynx.Cli;
+using Lynx.UCI.Commands.Engine;
 using Microsoft.Extensions.Configuration;
 using NLog;
 using NLog.Extensions.Logging;
 using System.Threading.Channels;
 
+Console.WriteLine($"{IdCommand.EngineName} {IdCommand.GetVersion()} by {IdCommand.EngineAuthor}");
+
 #if DEBUG
 Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development");
 #endif
@@ -34,14 +37,18 @@
 var tasks = new List<Task>
 {
     Task.Run(() => new Writer(engineChannel).Run(cancellationToken)),
-    Task.Run(() => new LinxDriver(uciChannel, engineChannel, new Engine(engineChannel)).Run(cancellationToken)),
+    Task.Run(() => new LynxDriver(uciChannel, engineChannel, new Engine(engineChannel)).Run(cancellationToken)),
     Task.Run(() => new Listener(uciChannel).Run(cancellationToken)),
     uciChannel.Reader.Completion,
     engineChannel.Reader.Completion
 };
 
 try
 {
+    if (args[0] == "bench")
+    {
+        await uciChannel.Writer.WriteAsync("bench");
+    }
     await Task.WhenAny(tasks);
 }
 catch (AggregateException ae)

diff --git a/src/Lynx.Cli/appsettings.json b/src/Lynx.Cli/appsettings.json
@@ -53,7 +53,8 @@
     "TranspositionTableEnabled": true,
     "TranspositionTableSize": "0x10000000", // 256 MB
     "UseOnlineTablebaseInRootPositions": false,
-    "UseOnlineTablebaseInSearch": false
+    "UseOnlineTablebaseInSearch": false,
+    "BenchDepth": 5
   },
 
   // Logging settings

diff --git a/src/Lynx.Dev/Program.cs b/src/Lynx.Dev/Program.cs
@@ -527,17 +527,19 @@ static void _42_Perft()
     {
         var sw = new Stopwatch();
         sw.Start();
-        var nodes = Perft.Results(pos, depth);
+        var result = Perft.Results(pos, depth);
         sw.Stop();
 
-        Console.WriteLine($"Depth {depth}\tNodes: {nodes}\tTime: {sw.ElapsedMilliseconds}ms");
+        Perft.PrintPerftResult(depth, result, Console.WriteLine);
     }
 }
 
 static void _43_Perft()
 {
-    Perft.Divide(new Position(Constants.InitialPositionFEN), 5);
-    Perft.Divide(new Position(TrickyPosition), 5);
+    var result = Perft.Divide(new Position(Constants.InitialPositionFEN), 5);
+    Perft.PrintPerftResult(5, result, Console.WriteLine);
+    result = Perft.Divide(new Position(TrickyPosition), 5);
+    Perft.PrintPerftResult(5, result, Console.WriteLine);
 }
 
 static void _44_ParseUCI()

diff --git a/src/Lynx/Configuration.cs b/src/Lynx/Configuration.cs
@@ -205,4 +205,9 @@ public sealed class EngineSettings
     /// This can also de used to reduce online probing
     /// </summary>
     public int OnlineTablebaseMaxSupportedPieces { get; set; } = 7;
+
+    /// <summary>
+    /// Depth for bench command
+    /// </summary>
+    public int BenchDepth { get; set; } = 3;
 }
diff --git a/src/Lynx/Engine.cs b/src/Lynx/Engine.cs
@@ -165,7 +165,7 @@ private SearchResult SearchBestMove(int minDepth, int? maxDepth, int? decisionTi
 
         if (searchResult is not null)
         {
-            _logger.Info("Search evaluation result - eval: {0}, mate: {1}, depth: {2}, refutation: {3}",
+            _logger.Info("Search evaluation result - eval: {0}, mate: {1}, depth: {2}, pv: {3}",
                 searchResult.Evaluation, searchResult.Mate, searchResult.TargetDepth, string.Join(", ", searchResult.Moves.Select(m => m.ToMoveString())));
         }
 

diff --git a/src/Lynx/LinxDriver.cs → src/Lynx/LynxDriver.cs b/src/Lynx/LinxDriver.cs → src/Lynx/LynxDriver.cs
@@ -6,15 +6,15 @@
 
 namespace Lynx;
 
-public sealed class LinxDriver
+public sealed class LynxDriver
 {
     private readonly ChannelReader<string> _uciReader;
     private readonly Channel<string> _engineWriter;
     private readonly Engine _engine;
     private readonly Logger _logger;
     private static readonly string[] _none = new[] { "none" };
 
-    public LinxDriver(ChannelReader<string> uciReader, Channel<string> engineWriter, Engine engine)
+    public LynxDriver(ChannelReader<string> uciReader, Channel<string> engineWriter, Engine engine)
     {
         _uciReader = uciReader;
         _engineWriter = engineWriter;
@@ -81,12 +81,14 @@ public async Task Run(CancellationToken cancellationToken)
                                 HandleNewGame();
                                 break;
                             case "perft":
-                                HandlePerft(rawCommand);
+                                await HandlePerft(rawCommand);
                                 break;
                             case "divide":
-                                HandleDivide(rawCommand);
+                                await HandleDivide(rawCommand);
+                                break;
+                            case "bench":
+                                await HandleBench(rawCommand);
                                 break;
-
                             default:
                                 _logger.Warn("Unknown command received: {0}", rawCommand);
                                 break;
@@ -105,7 +107,7 @@ public async Task Run(CancellationToken cancellationToken)
         }
         finally
         {
-            _logger.Info("Finishing {0}", nameof(LinxDriver));
+            _logger.Info("Finishing {0}", nameof(LynxDriver));
         }
     }
 
@@ -232,6 +234,17 @@ private void HandleSetOption(string command, string[] commandItems)
                     }
                     break;
                 }
+            case "threads":
+                {
+                    if (commandItems.Length > 4 && int.TryParse(commandItems[4], out var value))
+                    {
+                        if (value != 1)
+                        {
+                            _logger.Warn("Unsopported threads value: {0}", value);
+                        }
+                    }
+                    break;
+                }
             default:
                 _logger.Warn("Unsupported option: {0}", command);
                 break;
@@ -260,26 +273,34 @@ private void HandleQuit()
 
     private void HandleRegister(string rawCommand) => _engine.Registration = new RegisterCommand(rawCommand);
 
-    private void HandlePerft(string rawCommand)
+    private async Task HandlePerft(string rawCommand)
     {
         var items = rawCommand.Split(' ', StringSplitOptions.RemoveEmptyEntries);
 
         if (items.Length >= 2 && int.TryParse(items[1], out int depth) && depth >= 1)
         {
-            Perft.Results(_engine.Game.CurrentPosition, depth);
+            var results = Perft.Results(_engine.Game.CurrentPosition, depth);
+            await Perft.PrintPerftResult(depth, results, str => _engineWriter.Writer.WriteAsync(str));
         }
     }
 
-    private void HandleDivide(string rawCommand)
+    private async ValueTask HandleDivide(string rawCommand)
     {
         var items = rawCommand.Split(' ', StringSplitOptions.RemoveEmptyEntries);
 
         if (items.Length >= 2 && int.TryParse(items[1], out int depth) && depth >= 1)
         {
-            Perft.Divide(_engine.Game.CurrentPosition, depth);
+            var results = Perft.Divide(_engine.Game.CurrentPosition, depth);
+            await Perft.PrintPerftResult(depth, results, str => _engineWriter.Writer.WriteAsync(str));
         }
     }
 
+    private async ValueTask HandleBench(string rawCommand)
+    {
+        var results = OpenBench.Bench();
+        await OpenBench.PrintBenchResults(results, str => _engineWriter.Writer.WriteAsync(str));
+    }
+
     #endregion
 
     private async Task SendCommand(string command, CancellationToken cancellationToken)

diff --git a/src/Lynx/Model/Position.cs b/src/Lynx/Model/Position.cs
@@ -45,6 +45,8 @@ public Position(string fen) : this(FENParser.ParseFEN(fen))
         EnPassant = parsedFEN.EnPassant;
 
         UniqueIdentifier = ZobristTable.PositionHash(this);
+
+        // TODO: half move and full move counters
     }
 
     /// <summary>

diff --git a/src/Lynx/OpenBench.cs b/src/Lynx/OpenBench.cs
@@ -0,0 +1,106 @@
+using System.Diagnostics;
+using System.Threading.Channels;
+
+namespace Lynx;
+
+public static class OpenBench
+{
+    private static readonly string[] _benchmarkFens = new string[]
+    {
+        "r3k2r/2pb1ppp/2pp1q2/p7/1nP1B3/1P2P3/P2N1PPP/R2QK2R w KQkq a6 0 14",
+        "4rrk1/2p1b1p1/p1p3q1/4p3/2P2n1p/1P1NR2P/PB3PP1/3R1QK1 b - - 2 24",
+        "r3qbrk/6p1/2b2pPp/p3pP1Q/PpPpP2P/3P1B2/2PB3K/R5R1 w - - 16 42",
+        "6k1/1R3p2/6p1/2Bp3p/3P2q1/P7/1P2rQ1K/5R2 b - - 4 44",
+        "8/8/1p2k1p1/3p3p/1p1P1P1P/1P2PK2/8/8 w - - 3 54",
+        "7r/2p3k1/1p1p1qp1/1P1Bp3/p1P2r1P/P7/4R3/Q4RK1 w - - 0 36",
+        "r1bq1rk1/pp2b1pp/n1pp1n2/3P1p2/2P1p3/2N1P2N/PP2BPPP/R1BQ1RK1 b - - 2 10",
+        "3r3k/2r4p/1p1b3q/p4P2/P2Pp3/1B2P3/3BQ1RP/6K1 w - - 3 87",
+        "2r4r/1p4k1/1Pnp4/3Qb1pq/8/4BpPp/5P2/2RR1BK1 w - - 0 42",
+        "4q1bk/6b1/7p/p1p4p/PNPpP2P/KN4P1/3Q4/4R3 b - - 0 37",
+        "2q3r1/1r2pk2/pp3pp1/2pP3p/P1Pb1BbP/1P4Q1/R3NPP1/4R1K1 w - - 2 34",
+        "1r2r2k/1b4q1/pp5p/2pPp1p1/P3Pn2/1P1B1Q1P/2R3P1/4BR1K b - - 1 37",
+        "r3kbbr/pp1n1p1P/3ppnp1/q5N1/1P1pP3/P1N1B3/2P1QP2/R3KB1R b KQkq b3 0 17",
+        "8/6pk/2b1Rp2/3r4/1R1B2PP/P5K1/8/2r5 b - - 16 42",
+        "1r4k1/4ppb1/2n1b1qp/pB4p1/1n1BP1P1/7P/2PNQPK1/3RN3 w - - 8 29",
+        "8/p2B4/PkP5/4p1pK/4Pb1p/5P2/8/8 w - - 29 68",
+        "3r4/ppq1ppkp/4bnp1/2pN4/2P1P3/1P4P1/PQ3PBP/R4K2 b - - 2 20",
+        "5rr1/4n2k/4q2P/P1P2n2/3B1p2/4pP2/2N1P3/1RR1K2Q w - - 1 49",
+        "1r5k/2pq2p1/3p3p/p1pP4/4QP2/PP1R3P/6PK/8 w - - 1 51",
+        "q5k1/5ppp/1r3bn1/1B6/P1N2P2/BQ2P1P1/5K1P/8 b - - 2 34",
+        "r1b2k1r/5n2/p4q2/1ppn1Pp1/3pp1p1/NP2P3/P1PPBK2/1RQN2R1 w - - 0 22",
+        "r1bqk2r/pppp1ppp/5n2/4b3/4P3/P1N5/1PP2PPP/R1BQKB1R w KQkq - 0 5",
+        "r1bqr1k1/pp1p1ppp/2p5/8/3N1Q2/P2BB3/1PP2PPP/R3K2n b Q - 1 12",
+        "r1bq2k1/p4r1p/1pp2pp1/3p4/1P1B3Q/P2B1N2/2P3PP/4R1K1 b - - 2 19",
+        "r4qk1/6r1/1p4p1/2ppBbN1/1p5Q/P7/2P3PP/5RK1 w - - 2 25",
+        "r7/6k1/1p6/2pp1p2/7Q/8/p1P2K1P/8 w - - 0 32",
+        "r3k2r/ppp1pp1p/2nqb1pn/3p4/4P3/2PP4/PP1NBPPP/R2QK1NR w KQkq - 1 5",
+        "3r1rk1/1pp1pn1p/p1n1q1p1/3p4/Q3P3/2P5/PP1NBPPP/4RRK1 w - - 0 12",
+        "5rk1/1pp1pn1p/p3Brp1/8/1n6/5N2/PP3PPP/2R2RK1 w - - 2 20",
+        "8/1p2pk1p/p1p1r1p1/3n4/8/5R2/PP3PPP/4R1K1 b - - 3 27",
+        "8/4pk2/1p1r2p1/p1p4p/Pn5P/3R4/1P3PP1/4RK2 w - - 1 33",
+        "8/5k2/1pnrp1p1/p1p4p/P6P/4R1PK/1P3P2/4R3 b - - 1 38",
+        "8/8/1p1kp1p1/p1pr1n1p/P6P/1R4P1/1P3PK1/1R6 b - - 15 45",
+        "8/8/1p1k2p1/p1prp2p/P2n3P/6P1/1P1R1PK1/4R3 b - - 5 49",
+        "8/8/1p4p1/p1p2k1p/P2npP1P/4K1P1/1P6/3R4 w - - 6 54",
+        "8/8/1p4p1/p1p2k1p/P2n1P1P/4K1P1/1P6/6R1 b - - 6 59",
+        "8/5k2/1p4p1/p1pK3p/P2n1P1P/6P1/1P6/4R3 b - - 14 63",
+        "8/1R6/1p1K1kp1/p6p/P1p2P1P/6P1/1Pn5/8 w - - 0 67",
+        "1rb1rn1k/p3q1bp/2p3p1/2p1p3/2P1P2N/PP1RQNP1/1B3P2/4R1K1 b - - 4 23",
+        "4rrk1/pp1n1pp1/q5p1/P1pP4/2n3P1/7P/1P3PB1/R1BQ1RK1 w - - 3 22",
+        "r2qr1k1/pb1nbppp/1pn1p3/2ppP3/3P4/2PB1NN1/PP3PPP/R1BQR1K1 w - - 4 12",
+        "2r2k2/8/4P1R1/1p6/8/P4K1N/7b/2B5 b - - 0 55",
+        "6k1/5pp1/8/2bKP2P/2P5/p4PNb/B7/8 b - - 1 44",
+        "2rqr1k1/1p3p1p/p2p2p1/P1nPb3/2B1P3/5P2/1PQ2NPP/R1R4K w - - 3 25",
+        "r1b2rk1/p1q1ppbp/6p1/2Q5/8/4BP2/PPP3PP/2KR1B1R b - - 2 14",
+        "6r1/5k2/p1b1r2p/1pB1p1p1/1Pp3PP/2P1R1K1/2P2P2/3R4 w - - 1 36",
+        "rnbqkb1r/pppppppp/5n2/8/2PP4/8/PP2PPPP/RNBQKBNR b KQkq c3 0 2",
+        "2rr2k1/1p4bp/p1q1p1p1/4Pp1n/2PB4/1PN3P1/P3Q2P/2RR2K1 w - f6 0 20",
+        "3br1k1/p1pn3p/1p3n2/5pNq/2P1p3/1PN3PP/P2Q1PB1/4R1K1 w - - 0 23",
+        "2r2b2/5p2/5k2/p1r1pP2/P2pB3/1P3P2/K1P3R1/7R w - - 23 93",
+        "5k2/4q1p1/3P1pQb/1p1B4/pP5p/P1PR4/5PP1/1K6 b - - 0 38",
+        "6k1/6p1/8/6KQ/1r6/q2b4/8/8 w - - 0 32",
+        "5rk1/1rP3pp/p4n2/3Pp3/1P2Pq2/2Q4P/P5P1/R3R1K1 b - - 0 32",
+        "4r1k1/4r1p1/8/p2R1P1K/5P1P/1QP3q1/1P6/3R4 b - - 0 1",
+        "R4r2/4q1k1/2p1bb1p/2n2B1Q/1N2pP2/1r2P3/1P5P/2B2KNR w - - 3 31",
+        "r6k/pbR5/1p2qn1p/P2pPr2/4n2Q/1P2RN1P/5PBK/8 w - - 2 31",
+        "rn2k3/4r1b1/pp1p1n2/1P1q1p1p/3P4/P3P1RP/1BQN1PR1/1K6 w - - 6 28",
+        "3q1k2/3P1rb1/p6r/1p2Rp2/1P5p/P1N2pP1/5B1P/3QRK2 w - - 1 42",
+        "4r2k/1p3rbp/2p1N1p1/p3n3/P2NB1nq/1P6/4R1P1/B1Q2RK1 b - - 4 32",
+        "4r1k1/1q1r3p/2bPNb2/1p1R3Q/pB3p2/n5P1/6B1/4R1K1 w - - 2 36",
+        "3qr2k/1p3rbp/2p3p1/p7/P2pBNn1/1P3n2/6P1/B1Q1RR1K b - - 1 30",
+        "3qk1b1/1p4r1/1n4r1/2P1b2B/p3N2p/P2Q3P/8/1R3R1K w - - 2 39"
+    };
+
+    /// <summary>
+    /// Positions taken from Akimbo
+    /// https://github.com/JacquesRW/akimbo/blob/main/resources/fens.txt
+    /// </summary>
+    public static (int TotalNodes, long Nps) Bench()
+    {
+        var engineChannel = Channel.CreateUnbounded<string>();
+        var engine = new Engine(engineChannel);
+        var stopwatch = new Stopwatch();
+
+        int totalNodes = 0;
+        long totalTime = 0;
+
+        foreach (var fen in _benchmarkFens)
+        {
+            Console.WriteLine($"Benchmarking {fen} at depth {Configuration.EngineSettings.BenchDepth}");
+
+            engine.AdjustPosition($"position fen {fen}");
+            stopwatch.Restart();
+
+            var result = engine.BestMove(new($"go depth {Configuration.EngineSettings.BenchDepth}"));
+            totalTime += stopwatch.ElapsedMilliseconds;
+            totalNodes += result.Nodes;
+        }
+
+        Console.WriteLine($"Total time: {totalTime}");
+
+        return (totalNodes, Utils.CalculateNps(totalNodes, totalTime));
+    }
+
+    public static void PrintBenchResults((int TotalNodes, long Nps) benchResult, Action<string> write) => write($"{benchResult.TotalNodes} nodes {benchResult.Nps} nps");
+    public static async ValueTask PrintBenchResults((int TotalNodes, long Nps) benchResult, Func<string, ValueTask> write) => await write($"{benchResult.TotalNodes} nodes {benchResult.Nps} nps");
+}