Make conditional build possible, with Hanabi as the example.

The goal is to provide a simple recipe that people can imitate to do conditional dependencies.

PiperOrigin-RevId: 280158267
Change-Id: I8b2ce02ff734ee053ab029bd07d6a788b0ffe341

.travis.yml

@@ -21,9 +21,16 @@ git:
 # and the virtualenv.
 matrix:
   include:
+  # Build and run tests without all optional dependencies (default behavior)
   - os: linux
     dist: bionic # Ubuntu 18.04.2 LTS released on 26 April 2018
     env: OS_PYTHON_VERSION=3.6
+  # Build and run tests with all optional dependencies
+  - os: linux
+    dist: bionic # Ubuntu 18.04.2 LTS released on 26 April 2018
+    env:
+      - OS_PYTHON_VERSION=3.6
+      - BUILD_WITH_HANABI="ON"
   # - os: osx
   #   # macOS 10.14 (Mojave), release on September 24, 2018.
   #   osx_image: xcode11

docs/developer_guide.md

@@ -75,3 +75,33 @@ ideal to first be aware of the general API (see `spiel.h`).
         random games, to be used by integration tests to prevent any regression.
         `open_spiel/integration_tests/playthrough_test.py` will automatically
         load the playthroughs and compare them to newly generated playthroughs.
+
+## Conditional dependencies
+
+The goal is to make it possible to optionally include external dependencies and
+build against them. The setup was designed to met the following needs:
+
+-   **Single source of truth**: We want a single action to be sufficient to
+    manage the conditional install and build. Thus, we use bash environment
+    variables, that are read both by the install script (`install.sh`) to know
+    whether we should clone the dependency, and by CMake to know whether we
+    should include the files in the target. Tests can also access the bash
+    environment variable.
+-   **Light and safe defaults**: By default, we exclude the dependencies to
+    diminish install time and compilation time. If the bash variable is unset,
+    we download the dependency and we do not build against it.
+-   **Respect the user-defined values**: The `global_variables.sh` script, which
+    is included in all the scripts that needs to access the constant values, do
+    not override the constants but set them if and only if they are undefined.
+    This respects the user-defined values, e.g. on their `.bashrc` or on the
+    command line.
+
+When you add a new conditional dependency, you need to touch:
+
+-   the root CMakeLists.txt to add the option, with an OFF default
+-   add the option to `scripts/global_variables.sh`
+-   set the option to "ON" in `.travis.yml` for the "All dependencies"
+    integration test.
+-   change `install.sh` to make sure the dependency is installed
+-   use constructs like `if (${BUILD_WITH_HANABI})` in CMake to optionally add
+    the targets to build.

docs/install.md

@@ -5,8 +5,9 @@ The instructions here are for Linux and MacOS. For installation on Windows, see
 
 ## Summary
 
-1.  Install system packages and download some dependencies. Only needs to be
-    run once.
+1.  Install system packages and download some dependencies. Only needs to be run
+    once or if you enable some new conditional dependencies (see specific
+    section below).
 
     ```bash
     ./install.sh
@@ -47,6 +48,11 @@ Linux versions).
 
 ## Detailed steps
 
+### Configuration conditional dependencies
+
+See `open_spiel/scripts/global_variables.sh` to configure the conditional
+dependencies. See also the [Developer Guide](devloper_guide.md).
+
 ### Installing system-wide dependencies
 
 See `install.sh` for the required packages and cloned repositories.

install.sh

@@ -21,6 +21,9 @@
 set -e  # exit when any command fails
 set -x
 
+MYDIR="$(dirname "$(realpath "$0")")"
+source "${MYDIR}/open_spiel/scripts/global_variables.sh"
+
 # 1. Clone the external dependencies before installing systen packages, to make
 # sure they are present even if later commands fail.
 #
@@ -55,6 +58,12 @@ fi
   git clone -b 'master' --single-branch --depth 1 https://github.com/abseil/abseil-cpp.git \
   open_spiel/abseil-cpp
 
+# Optional dependencies.
+DIR="open_spiel/games/hanabi/hanabi-learning-environment"
+if [[ ${BUILD_WITH_HANABI:-"ON"} == "ON" ]] && [[ ! -d ${DIR} ]]; then
+  git clone -b 'master' --single-branch --depth 1 https://github.com/deepmind/hanabi-learning-environment.git ${DIR}
+fi
+
 
 # 2. Install other required system-wide dependencies
 if [[ "$OSTYPE" == "linux-gnu" ]]; then

open_spiel/CMakeLists.txt

@@ -21,6 +21,44 @@ endif()
 # Position-independent code is needed for Python extension modules.
 set (CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+## Optional dependencies
+# One can optionally build and link against specific external dependencies.
+# We expect these arguments to be always defined, when building using any script
+# in `open_spiel/scripts/`, thus, we emit a warning when it's not, with a
+# conservative default.
+# See the documentation in install.md.
+if(NOT WIN32)
+  string(ASCII 27 Esc)
+  set(ColourReset "${Esc}[m")
+  set(ColourBold  "${Esc}[1m")
+  set(Red         "${Esc}[31m")
+  set(Green       "${Esc}[32m")
+  set(Yellow      "${Esc}[33m")
+  set(Blue        "${Esc}[34m")
+  set(Magenta     "${Esc}[35m")
+  set(Cyan        "${Esc}[36m")
+  set(White       "${Esc}[37m")
+  set(BoldRed     "${Esc}[1;31m")
+  set(BoldGreen   "${Esc}[1;32m")
+  set(BoldYellow  "${Esc}[1;33m")
+  set(BoldBlue    "${Esc}[1;34m")
+  set(BoldMagenta "${Esc}[1;35m")
+  set(BoldCyan    "${Esc}[1;36m")
+  set(BoldWhite   "${Esc}[1;37m")
+endif()
+
+set (BUILD_WITH_HANABI OFF CACHE BOOL "Build against the Hanabi game.")
+if(NOT DEFINED ENV{BUILD_WITH_HANABI})
+    message("${BoldRed}BUILD_WITH_HANABI not set. Defaults to OFF${ColourReset}")
+    set (ENV{BUILD_WITH_HANABI} OFF)
+endif()
+set (BUILD_WITH_HANABI $ENV{BUILD_WITH_HANABI})
+message("${BoldYellow}BUILD_WITH_HANABI: ${BUILD_WITH_HANABI} ${ColourReset}")
+
+##
+
+
+
 # Needed to disable Abseil tests.
 set (BUILD_TESTING OFF)
 
@@ -78,9 +116,11 @@ set (OPEN_SPIEL_OBJECTS
   $<TARGET_OBJECTS:game_transforms>
   $<TARGET_OBJECTS:open_spiel_query>
   $<TARGET_OBJECTS:bridge_double_dummy_solver>
-  # $<TARGET_OBJECTS:hanabi_learning_environment>
   $<TARGET_OBJECTS:algorithms>
 )
+if (BUILD_WITH_HANABI)
+  set(OPEN_SPIEL_OBJECTS ${OPEN_SPIEL_OBJECTS} $<TARGET_OBJECTS:hanabi_learning_environment>)
+endif()
 
 # We have the parent of this directory in the include path, so that we can
 # include for example "open_spiel/spiel.h" (assuming this directory is named

open_spiel/games/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_library (games OBJECT
+set(GAME_SOURCES
   backgammon.cc
   backgammon.h
   blotto.cc
@@ -31,9 +31,6 @@ add_library (games OBJECT
   go/go_board.h
   goofspiel.cc
   goofspiel.h
-  # Uncomment to enable Hanabi.
-  # hanabi.cc
-  # hanabi.h
   havannah.cc
   havannah.h
   hex.cc
@@ -76,10 +73,18 @@ add_library (games OBJECT
   y.cc
   y.h
 )
+
+if (${BUILD_WITH_HANABI})
+  set(GAME_SOURCES ${GAME_SOURCES} hanabi.cc hanabi.h)
+endif()
+add_library (games OBJECT ${GAME_SOURCES})
+
+
 target_include_directories (games PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
-# Uncomment to enable Hanabi.
-# add_subdirectory (hanabi)
+if (${BUILD_WITH_HANABI})
+  add_subdirectory (hanabi)
+endif()
 
 # Uncomment to build the Ludii demo
 # add_subdirectory (ludii)

open_spiel/games/hanabi.h

@@ -26,14 +26,8 @@
 // can be found here: https://github.com/deepmind/hanabi-learning-environment
 //
 // Since Hanabi relies on an (optional) external dependency, it is not included
-// in the list of compiled games by default. To enable it, follow these steps:
-//   1. git clone the Hanabi Learning Environment above (e.g. in $HOME)
-//   2. Set the path of HANABI_HOME in games/hanabi/CMakeLists.txt
-//   3. Uncomment hanabi.cc, hanabi.h, and the add_subdirectory (hanabi) in
-//      games/CMakeLists.txt
-//   4. Uncomment the $<TARGET_OBJECTS:hanabi_learning_environment> in the
-//      top-level CMakeLists.txt
-//   5. Enjoy the fireworks!
+// in the list of compiled games by default. To enable it, read `install.md`
+// (TLDR: Set the environment variable BUILD_WITH_HANABI to ON).
 
 #include <memory>
 

open_spiel/games/hanabi/CMakeLists.txt

@@ -1,31 +1,29 @@
-set (HANABI_HOME $ENV{HOME}/hanabi-learning-environment)
-
 add_library(hanabi_learning_environment OBJECT
-  ${HANABI_HOME}/hanabi_lib/canonical_encoders.cc
-  ${HANABI_HOME}/hanabi_lib/canonical_encoders.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_card.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_card.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_game.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_game.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_hand.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_hand.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_history_item.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_history_item.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_move.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_move.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_observation.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_observation.h
-  ${HANABI_HOME}/hanabi_lib/hanabi_state.cc
-  ${HANABI_HOME}/hanabi_lib/hanabi_state.h
-  ${HANABI_HOME}/hanabi_lib/observation_encoder.h
-  ${HANABI_HOME}/hanabi_lib/util.cc
-  ${HANABI_HOME}/hanabi_lib/util.h
+  hanabi-learning-environment/hanabi_lib/canonical_encoders.cc
+  hanabi-learning-environment/hanabi_lib/canonical_encoders.h
+  hanabi-learning-environment/hanabi_lib/hanabi_card.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_card.h
+  hanabi-learning-environment/hanabi_lib/hanabi_game.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_game.h
+  hanabi-learning-environment/hanabi_lib/hanabi_hand.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_hand.h
+  hanabi-learning-environment/hanabi_lib/hanabi_history_item.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_history_item.h
+  hanabi-learning-environment/hanabi_lib/hanabi_move.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_move.h
+  hanabi-learning-environment/hanabi_lib/hanabi_observation.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_observation.h
+  hanabi-learning-environment/hanabi_lib/hanabi_state.cc
+  hanabi-learning-environment/hanabi_lib/hanabi_state.h
+  hanabi-learning-environment/hanabi_lib/observation_encoder.h
+  hanabi-learning-environment/hanabi_lib/util.cc
+  hanabi-learning-environment/hanabi_lib/util.h
 )
 
-target_include_directories (hanabi_learning_environment PUBLIC ${HANABI_HOME})
-target_include_directories (games PUBLIC ${HANABI_HOME})
+target_include_directories (hanabi_learning_environment PUBLIC hanabi-learning-environment)
+target_include_directories (games PUBLIC hanabi-learning-environment)
 
 add_executable(hanabi_test ../hanabi_test.cc ${OPEN_SPIEL_OBJECTS}
                $<TARGET_OBJECTS:tests>)
 add_test(hanabi_test hanabi_test)
-target_include_directories (hanabi_test PUBLIC ${HANABI_HOME})
+target_include_directories (hanabi_test PUBLIC hanabi-learning-environment)

open_spiel/games/hanabi/README.md

@@ -0,0 +1,5 @@
+# Hanabi
+
+OpenSpiel can support Hanabi, using the implementation in
+https://github.com/deepmind/hanabi-learning-environment. To enable this option,
+see `open_spiel/scripts/global_variables.sh`.

open_spiel/python/tests/pyspiel_test.py

@@ -18,6 +18,7 @@
 from __future__ import division
 from __future__ import print_function
 
+import os
 from absl.testing import absltest
 import six
 
@@ -31,7 +32,7 @@ def test_registered_names(self):
     game_names = pyspiel.registered_names()
 
     # Specify game names in alphabetical order, to make the test easier to read.
-    expected = [
+    expected = set([
         "backgammon",
         "blotto",
         "breakthrough",
@@ -74,7 +75,10 @@ def test_registered_names(self):
         "tiny_hanabi",
         "turn_based_simultaneous_game",
         "y",
-    ]
+    ])
+    if os.environ.get("BUILD_WITH_HANABI", "OFF") == "ON":
+      expected.add("hanabi")
+    expected = sorted(list(expected))
     self.assertCountEqual(game_names, expected)
 
   def test_no_mandatory_parameters(self):

open_spiel/scripts/global_variables.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generates a playthrough for a new game with optional parameters.
+# This script exists mainly as a reminder for the command to run.
+
+# This file contains the global variables that control conditional dependencies.
+# It is being used to know whether we should:
+# (a) download a dependency (done in install.sh)
+# (b) build it and link against it during the `cmake` build process
+#
+# Note that we do not change the value of the constants if they are already
+# defined by an enclosing scope (useful for command line overrides).
+
+export BUILD_WITH_HANABI=${BUILD_WITH_HANABI:-"OFF"}

open_spiel/scripts/travis_script.sh

@@ -23,5 +23,8 @@ source ./venv/bin/activate
 python --version
 pip3 install -r requirements.txt
 
+# Within the tests, we build against all possible optional dependencies.
+BUILD_WITH_HANABI="ON"
+
 ./open_spiel/scripts/build_and_run_tests.sh
 deactivate