Merge pull request #560 from Farama-Foundation/build-fixes

Building and CI/CD fixes

.github/workflows/build-and-test.yml

@@ -37,7 +37,9 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
 
     - name: Install dependencies on Ubuntu
       if: runner.os == 'Linux'
@@ -54,7 +56,7 @@ jobs:
       run: brew install cmake boost openal-soft sdl2
 
     - name: Set up Python ${{ matrix.python-version }} environment
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/build-docs.yml

@@ -14,6 +14,9 @@ jobs:
       SPHINX_GITHUB_CHANGELOG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
       - uses: actions/setup-python@v4
         with:
             python-version: '3.9'

.github/workflows/build-wheels.yml

@@ -25,6 +25,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Set up QEMU
         if: runner.os == 'Linux'
@@ -56,6 +58,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Build sdist
         run: pipx run build --sdist

CMakeLists.txt

@@ -20,73 +20,73 @@ option(BUILD_ENGINE "Build ViZDoom Engine (required to build Python package)" ON
 
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_LINK)
-if (COMMAND cmake_policy)
+if(COMMAND cmake_policy)
     cmake_policy(SET CMP0003 NEW)
-endif (COMMAND cmake_policy)
+endif()
 
 # Add cmake_modules to module
 set(CMAKE_MODULE_PATH
     ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}" "${PROJECT_SOURCE_DIR}"
     "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
 
 # Set the release mode if not specified
-if (NOT CMAKE_BUILD_TYPE)
+if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
-endif ()
+endif()
 
 
 # System dependent settings
 #-----------------------------------------------------------------------------------------------------------------------
 
-if (UNIX)
+if(UNIX)
     set(Boost_USE_STATIC_LIBS OFF)
     set(Boost_USE_MULTITHREADED ON)
     set(Boost_USE_STATIC_RUNTIME OFF)
-endif ()
+endif()
 
-if (WIN32)
+if(WIN32)
     set(Boost_USE_STATIC_LIBS ON)
     set(Boost_USE_MULTITHREADED ON)
     set(Boost_USE_STATIC_RUNTIME ON)
 
     macro(get_WIN32_WINNT version)
-        if (WIN32 AND CMAKE_SYSTEM_VERSION)
+        if(WIN32 AND CMAKE_SYSTEM_VERSION)
             set(ver ${CMAKE_SYSTEM_VERSION})
             string(REPLACE "10" "A" ver ${ver})
             string(REPLACE "." "" ver ${ver})
             string(REGEX REPLACE "([A-F0-9])" "0\\1" ver ${ver})
             string(SUBSTRING ${ver} 0 4 ver)
 
             set(${version} "0x${ver}")
-        endif ()
+        endif()
     endmacro()
 
     get_WIN32_WINNT(ver)
     add_definitions(-D_WIN32_WINNT=${ver})
-endif ()
+endif()
 
-if (APPLE)
+if(APPLE)
     set(CMAKE_MACOSX_RPATH ON)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -fvisibility-inlines-hidden")
-endif (APPLE)
+endif(APPLE)
 
 
 # Compiler dependent settings
 #-----------------------------------------------------------------------------------------------------------------------
 
-if (CMAKE_COMPILER_IS_GNUCXX)
+if(CMAKE_COMPILER_IS_GNUCXX)
     add_definitions("-fPIC")
     # Explicitly silence some warnings
     add_definitions("-Wno-implicit-fallthrough -Wno-shift-negative-value -Wno-stringop-truncation")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess -Wno-deprecated-copy") # C++ only
-endif ()
+endif()
 
-if (MSVC)
+if(MSVC)
     string(REPLACE "/MD " "/MT " CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
     string(REPLACE "/MD " "/MT " CMAKE_CXX_FLAGS_MINSIZEREL ${CMAKE_CXX_FLAGS_MINSIZEREL})
     string(REPLACE "/MD " "/MT " CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
     string(REPLACE "/MDd " "/MTd " CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
-endif ()
+endif()
 
 
 # Main library
@@ -110,9 +110,9 @@ set(VIZDOOM_LIBS
     ${Boost_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT})
 
-if (UNIX AND NOT APPLE)
+if(UNIX AND NOT APPLE)
     set(VIZDOOM_LIBS ${VIZDOOM_LIBS} rt)
-endif ()
+endif()
 
 configure_file(
     "${VIZDOOM_LIB_SRC_DIR}/ViZDoomVersion.h.in"
@@ -162,14 +162,14 @@ set_target_properties(libvizdoom_shared
 # ViZDoom Engine
 #-----------------------------------------------------------------------------------------------------------------------
 
-if (BUILD_ENGINE)
+if(BUILD_ENGINE)
     add_subdirectory(${VIZDOOM_SRC_DIR}/vizdoom)
-endif ()
+endif()
 
 
 # Python bindings
 #-----------------------------------------------------------------------------------------------------------------------
 
-if (BUILD_PYTHON)
+if(BUILD_PYTHON)
     add_subdirectory(${VIZDOOM_SRC_DIR}/lib_python)
-endif ()
+endif()

scripts/build_and_test.sh

@@ -14,7 +14,7 @@ cmake --version
 gcc --version
 
 # Report python version
-python3 -c "import sys; print('Python', sys.version)"
+python3 --version
 
 # Install
 python3 -m pip install .[test]

scripts/build_and_test_conda.sh

@@ -15,7 +15,6 @@ gcc --version
 
 # Report python version
 python --version
-python -c "import sys; print('Python', sys.version)"
 
 # Install
 export VIZDOOM_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=/root/miniconda/"

scripts/install_and_test_wheel.sh

@@ -9,10 +9,9 @@ ls -lha .
 
 # Report python version
 python3 --version
-python3 -c "import sys; print('Python', sys.version)"
 
 # Find matching wheel file in wheelhouse
-PYTHON_VERSION=$(python3 -c "import sys; print('{}{}'.format(sys.version_info.major, sys.version_info.minor))")
+PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
 PYTHON_WHEEL=$(ls wheelhouse/vizdoom-*-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}*.whl)
 
 # Updgrad pip and install test deps

setup.py

@@ -155,11 +155,14 @@ def run(self):
             shutil.copy(sndfile_dll, build_output_path)
             shutil.copy(openal_dll, build_output_path)
 
-        # python_standard_lib = sysconfig.get_python_lib(standard_lib=True)
         python_root_dir = os.path.dirname(sys.executable)
+        cmake_arg_list.append(f"-DPython_ROOT_DIR={python_root_dir}")
+        cmake_arg_list.append("-DPython_FIND_STRATEGY=LOCATION")
 
-        if python_root_dir and os.path.exists(python_root_dir):
-            cmake_arg_list.append(f"-DPython_ROOT_DIR={python_root_dir}")
+        # Manually specified artifacts
+        # python_executable = sys.executable
+        # python_include = sysconfig.get_python_inc()
+        # python_standard_lib = sysconfig.get_python_lib(standard_lib=True)
 
         if os.path.exists("CMakeCache.txt"):
             os.remove("CMakeCache.txt")

src/lib_python/CMakeLists.txt

@@ -3,11 +3,11 @@ set(BUILD_PYTHON_VERSION "" CACHE STRING "Version of Python to build bindings fo
 if("${BUILD_PYTHON_VERSION}" STREQUAL "")
     # Try to find Python 3 if BUILD_PYTHON_VERSION is not set
     message(STATUS "BUILD_PYTHON_VERSION is not set, trying to find Python 3, Python_ROOT_DIR is ${Python_ROOT_DIR}")
-    find_package(Python 3 REQUIRED)
+    find_package(Python 3 COMPONENTS Interpreter Development REQUIRED)
     set(BUILD_PYTHON_VERSION ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR})
 endif()
 
-set (PYBIND11_PYTHON_VERSION ${BUILD_PYTHON_VERSION})
+set(PYBIND11_PYTHON_VERSION ${BUILD_PYTHON_VERSION})
 
 set(VIZDOOM_PYTHON_OUTPUT_DIR ${VIZDOOM_OUTPUT_DIR}/python${BUILD_PYTHON_VERSION})
 set(VIZDOOM_PYTHON_SRC_DIR ${VIZDOOM_SRC_DIR}/lib_python)
@@ -67,15 +67,15 @@ set_target_properties(libvizdoom_python
     OUTPUT_NAME vizdoom
     PROJECT_LABEL "python${BUILD_PYTHON_VERSION} binding")
 
-if (UNIX)
+if(UNIX)
     add_custom_target(python_pip_package ALL
         COMMAND ${VIZDOOM_SCRIPTS_DIR}/assemble_pip_package.sh ${BUILD_PYTHON_VERSION} ${VIZDOOM_OUTPUT_DIR} ${CMAKE_SOURCE_DIR}
         COMMENT "Assembling pip package in ${VIZDOOM_PYTHON_OUTPUT_DIR}/pip_package")
-elseif (WIN32)
+elseif(WIN32)
     add_custom_target(python_pip_package ALL
         COMMAND ${VIZDOOM_SCRIPTS_DIR}/assemble_pip_package.bat ${BUILD_PYTHON_VERSION} ${VIZDOOM_OUTPUT_DIR} ${CMAKE_SOURCE_DIR}
         COMMENT "Assembling pip package in ${VIZDOOM_PYTHON_OUTPUT_DIR}/pip_package")
-endif ()
+endif()
 
 set_target_properties(python_pip_package
     PROPERTIES

src/lib_python/__init__.py.in

@@ -2,12 +2,12 @@ import sys as _sys
 
 _COMPILED_PYTHON_VERSION = "@ViZDoom_PYTHON_VERSION_STR@"
 
-_this_python_version = "{}.{}".format(_sys.version_info.major, _sys.version_info.minor)
+_this_python_version = f"{_sys.version_info.major}.{_sys.version_info.minor}"
 
 if _COMPILED_PYTHON_VERSION != _this_python_version:
     raise SystemError(
-        "This interpreter version: '{}' doesn't match with version of the interpreter ViZDoom was compiled with: {}".format(
-            _this_python_version, _COMPILED_PYTHON_VERSION))
+        f"This interpreter version: '{_this_python_version}' doesn't match with version of the interpreter ViZDoom was compiled with: '{_COMPILED_PYTHON_VERSION}'"
+    )
 
 from .vizdoom import __version__ as __version__
 from .vizdoom import *

tests/build_test_cibuildwheel_linux.sh

@@ -23,6 +23,7 @@ DOCKERFILES_TO_BUILD_AND_RUN=(
 )
 
 # Build wheels using cibuildwheel
+#export CIBW_BUILD_VERBOSITY=3  # Uncomment to see full build logs
 cibuildwheel --platform linux --arch $(uname -m)
 
 function create_dockerfile ( ) {
@@ -44,7 +45,7 @@ function create_dockerfile ( ) {
 for dockerfile_setting in "${DOCKERFILES_TO_BUILD_AND_RUN[@]}"; do
     create_dockerfile $dockerfile_setting
 
-    echo -n "Building and running $dockerfile, saving output to $dockerfile.log ... "
+    echo -n "Running $dockerfile, saving output to $dockerfile.log ... "
     filename=$( basename "$dockerfile" )
     dockerfile_dir=$( dirname "$dockerfile" )
     without_ext="${filename%.*}"

tests/build_test_local_linux_builds.sh

@@ -18,7 +18,6 @@ DOCKERFILES_TO_BUILD_AND_RUN=(
     "fedora:37 dnf-based.Dockerfile"
     "rockylinux:9 dnf-based.Dockerfile RUN dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb"
     "debian:11.6 apt-based.Dockerfile ENV LANG C.UTF-8"
-    "ubuntu:18.04 apt-based.Dockerfile"
     "ubuntu:20.04 apt-based.Dockerfile"
     "ubuntu:22.04 apt-based.Dockerfile"
     "ubuntu:20.04 apt+conda-based.Dockerfile"  # Ubuntu build with dependencies installed via conda