Validation: support GTP commands for each binary.

* Changed Validation and Game to support multiple GTP commands
  at start up but left the Validations options untouched.
* Separated engine options (as positional arguments) from match options.
  Replaced time settings option with ability to specify any GTP commands.
* Added --gtp-command options using the existing option parser.
  Also changed default binary options from -p 1600 to -v 3200.
* Each binary argument has to be preceded by "--".
* Changes to use Engine Objects.
* Exits on failed GTP command.

Added printing of GTP commands in gameStart() so users can see what
commands are actually sent to each engine.

Pull request #1652.

.gitignore

@@ -8,3 +8,5 @@ training/tf/venv
 leelaz-model*
 *.orig
 leelaz_opencl_tuning
+/build-autogtp-*
+/build-validation-*

autogtp/.gitignore

@@ -5,6 +5,7 @@
 /*.gz
 /moc_*.cpp
 /moc_*.h
+/autogtp.pro.user
 
 # Weight files
 /[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]

autogtp/Game.cpp

@@ -23,11 +23,12 @@
 #include <QFileInfo>
 #include "Game.h"
 
-Game::Game(const QString& weights, const QString& opt, const QString& binary) :
+Game::Game(const QString& weights, const QString& opt, const QString& binary,
+           const QStringList& commands) :
     QProcess(),
     m_cmdLine(""),
     m_binary(binary),
-    m_timeSettings("time_settings 0 1 0"),
+    m_commands(commands),
     m_resignation(false),
     m_blackToMove(true),
     m_blackResigned(false),
@@ -179,8 +180,15 @@ bool Game::gameStart(const VersionTuple &min_version) {
     // check any return values.
     checkVersion(min_version);
     QTextStream(stdout) << "Engine has started." << endl;
-    sendGtpCommand(m_timeSettings);
-    QTextStream(stdout) << "Infinite thinking time set." << endl;
+    for (auto command : m_commands) {
+        QTextStream(stdout) << command << endl;
+        if (!sendGtpCommand(command))
+        {
+            QTextStream(stdout) << "GTP failed on: " << command << endl;
+            exit(EXIT_FAILURE);
+        }
+    }
+    QTextStream(stdout) << "Thinking time set." << endl;
     return true;
 }
 

autogtp/Game.h

@@ -28,7 +28,8 @@ class Game : QProcess {
 public:
     Game(const QString& weights,
          const QString& opt,
-         const QString& binary = QString("./leelaz"));
+         const QString& binary = QString("./leelaz"),
+         const QStringList& commands = QStringList("time_settings 0 1 0"));
     ~Game() = default;
     bool gameStart(const VersionTuple& min_version);
     void move();
@@ -69,7 +70,7 @@ class Game : QProcess {
     };
     QString m_cmdLine;
     QString m_binary;
-    QString m_timeSettings;
+    QStringList m_commands;
     QString m_winner;
     QString m_fileName;
     QString m_moveDone;

validation/.gitignore

@@ -1,3 +1,5 @@
+validation.pro.user
+
 /Makefile
 /.qmake.stash
 /validation

validation/Validation.cpp

@@ -28,12 +28,14 @@ const VersionTuple min_leelaz_version{0, 10, 0};
 
 void ValidationWorker::run() {
     do {
-        Game first(m_firstNet,  m_firstOpts, m_firstBin);
+        Game first(m_engines[0].m_network, m_engines[0].m_options,
+                   m_engines[0].m_binary, m_engines[0].m_commands);
         if (!first.gameStart(min_leelaz_version)) {
             emit resultReady(Sprt::NoResult, Game::BLACK);
             return;
         }
-        Game second(m_secondNet, m_secondOpts, m_secondBin);
+        Game second(m_engines[1].m_network, m_engines[1].m_options,
+                    m_engines[1].m_binary, m_engines[1].m_commands);
         if (!second.gameStart(min_leelaz_version)) {
             emit resultReady(Sprt::NoResult, Game::BLACK);
             return;
@@ -93,9 +95,7 @@ void ValidationWorker::run() {
                 emit resultReady(Sprt::Loss, m_expected);
             }
             // Change color and play again
-            m_firstNet.swap(m_secondNet);
-            m_firstBin.swap(m_secondBin);
-            m_firstOpts.swap(m_secondOpts);
+            std::swap(m_engines[0], m_engines[1]);
             if (m_expected == Game::BLACK) {
                 m_expected = Game::WHITE;
             } else {
@@ -109,24 +109,14 @@ void ValidationWorker::run() {
 }
 
 void ValidationWorker::init(const QString& gpuIndex,
-                            const QString& firstNet,
-                            const QString& secondNet,
-                            const QString& firstBin,
-                            const QString& secondBin,
-                            const QString& firstOpts,
-                            const QString& secondOpts,
+                            const QVector<Engine>& engines,
                             const QString& keep,
                             int expected) {
-    m_firstOpts = firstOpts;
-    m_secondOpts = secondOpts;
+    m_engines = engines;
     if (!gpuIndex.isEmpty()) {
-        m_firstOpts.prepend(" --gpu=" + gpuIndex + " ");
-        m_secondOpts.prepend(" --gpu=" + gpuIndex + " ");
+        m_engines[0].m_options.prepend(" --gpu=" + gpuIndex + " ");
+        m_engines[1].m_options.prepend(" --gpu=" + gpuIndex + " ");
     }
-    m_firstNet = firstNet;
-    m_secondNet = secondNet;
-    m_firstBin = firstBin;
-    m_secondBin = secondBin;
     m_expected = expected;
     m_keepPath = keep;
     m_state.store(RUNNING);
@@ -135,14 +125,9 @@ void ValidationWorker::init(const QString& gpuIndex,
 Validation::Validation(const int gpus,
                        const int games,
                        const QStringList& gpuslist,
-                       const QString& firstNet,
-                       const QString& secondNet,
+                       QVector<Engine>& engines,
                        const QString& keep,
                        QMutex* mutex,
-                       const QString& firstBin,
-                       const QString& secondBin,
-                       const QString& firstOpts,
-                       const QString& secondOpts,
                        const float& h0,
                        const float& h1) :
 
@@ -152,21 +137,13 @@ Validation::Validation(const int gpus,
     m_games(games),
     m_gpus(gpus),
     m_gpusList(gpuslist),
-    m_firstNet(firstNet),
-    m_secondNet(secondNet),
-    m_firstBin(firstBin),
-    m_secondBin(secondBin),
-    m_firstOpts(firstOpts),
-    m_secondOpts(secondOpts),
+    m_engines(engines),
     m_keepPath(keep) {
     m_statistic.initialize(h0, h1, 0.05, 0.05);
     m_statistic.addGameResult(Sprt::Draw);
 }
 
 void Validation::startGames() {
-    QString n1, n2, b1 ,b2 ,o1, o2;
-    int expected;
-    QString myGpu;
     for (int gpu = 0; gpu < m_gpus; ++gpu) {
         for (int game = 0; game < m_games; ++game) {
             auto thread_index = gpu * m_games + game;
@@ -175,30 +152,21 @@ void Validation::startGames() {
                     this,
                     &Validation::getResult,
                     Qt::DirectConnection);
-            if (game % 2) {
-                n1 = m_firstNet;
-                n2 = m_secondNet;
-                b1 = m_firstBin;
-                b2 = m_secondBin;
-                o1 = m_firstOpts;
-                o2 = m_secondOpts;
-                expected = Game::BLACK;
-            } else {
-                n1 = m_secondNet;
-                n2 = m_firstNet;
-                b1 = m_secondBin;
-                b2 = m_firstBin;
-                o1 = m_secondOpts;
-                o2 = m_firstOpts;
+
+            auto engines = m_engines;
+            auto expected = Game::BLACK;
+            if (game & 1) {
+                std::swap(engines[0], engines[1]);
                 expected = Game::WHITE;
             }
-            if (m_gpusList.isEmpty()) {
-                myGpu = "";
-            } else {
+
+            auto myGpu = QString("");
+            if (!m_gpusList.isEmpty()) {
                 myGpu = m_gpusList.at(gpu);
             }
 
-            m_gamesThreads[thread_index].init(myGpu, n1, n2, b1, b2, o1, o2, m_keepPath, expected);
+            m_gamesThreads[thread_index].init(
+                myGpu, engines, m_keepPath, expected);
             m_gamesThreads[thread_index].start();
         }
     }
@@ -213,7 +181,7 @@ void Validation::saveSprt() {
     out << m_statistic;
     out << m_results;
     f.close();
-    m_results.printResults(m_firstNet, m_secondNet);
+    m_results.printResults(m_engines[0].m_network, m_engines[1].m_network);
     printSprtStatus(m_statistic.status());
 }
 
@@ -237,7 +205,7 @@ void Validation::loadSprt() {
     f.close();
     QFile::remove(fi.fileName());
     QTextStream(stdout) << "Initial Statistics" << endl;
-    m_results.printResults(m_firstNet, m_secondNet);
+    m_results.printResults(m_engines[0].m_network, m_engines[1].m_network);
     printSprtStatus(m_statistic.status());
 }
 
@@ -270,7 +238,7 @@ void Validation::getResult(Sprt::GameResult result, int net_one_color) {
             << "The first net is "
             <<  ((status.result ==  Sprt::AcceptH0) ? "worse " : "better ")
             << "than the second" << endl;
-        m_results.printResults(m_firstNet, m_secondNet);
+        m_results.printResults(m_engines[0].m_network, m_engines[1].m_network);
         //sendQuit();
     } else {
         printSprtStatus(status);

validation/Validation.h

@@ -28,9 +28,25 @@
 #include "../autogtp/Game.h"
 #include "Results.h"
 
+class Engine {
+public:
+    Engine(const QString& network,
+           const QString& options,
+           const QStringList& commands = QStringList("time_settings 0 1 0"),
+           const QString& binary = QString("./leelaz")) :
+        m_binary(binary), m_options(options),
+        m_network(network), m_commands(commands) {}
+    Engine() = default;
+    QString m_binary;
+    QString m_options;
+    QString m_network;
+    QStringList m_commands;
+};
+
 class ValidationWorker : public QThread {
     Q_OBJECT
 public:
+
     enum {
         RUNNING = 0,
         FINISHING
@@ -39,12 +55,7 @@ class ValidationWorker : public QThread {
     ValidationWorker(const ValidationWorker& w) : QThread(w.parent()) {}
     ~ValidationWorker() = default;
     void init(const QString& gpuIndex,
-              const QString& firstNet,
-              const QString& secondNet,
-              const QString& firstBin,
-              const QString& secondBin,
-              const QString& firstOpts,
-              const QString& secondOpts,
+              const QVector<Engine>& engines,
               const QString& keep,
               int expected);
     void run() override;
@@ -53,14 +64,9 @@ class ValidationWorker : public QThread {
 signals:
     void resultReady(Sprt::GameResult r, int net_one_color);
 private:
-    QString m_firstNet;
-    QString m_secondNet;
+    QVector<Engine> m_engines;
     int m_expected;
     QString m_keepPath;
-    QString m_firstBin;
-    QString m_secondBin;
-    QString m_firstOpts;
-    QString m_secondOpts;
     QAtomicInt m_state;
 };
 
@@ -70,14 +76,9 @@ class Validation : public QObject {
 public:
     Validation(const int gpus, const int games,
                const QStringList& gpusList,
-               const QString& firstNet,
-               const QString& secondNet,
+               QVector<Engine>& engines,
                const QString& keep,
                QMutex* mutex,
-               const QString& firstBin,
-               const QString& secondBin,
-               const QString& firstOpts,
-               const QString& secondOpts,
                const float& h0,
                const float& h1);
     ~Validation() = default;
@@ -98,12 +99,7 @@ public slots:
     int m_games;
     int m_gpus;
     QStringList m_gpusList;
-    QString m_firstNet;
-    QString m_secondNet;
-    QString m_firstBin;
-    QString m_secondBin;
-    QString m_firstOpts;
-    QString m_secondOpts;
+    QVector<Engine>& m_engines;
     QString m_keepPath;
     void quitThreads();
     void saveSprt();