Enable coverage information generation for pytest tests with CMake (#226

)

* Enable coverage information generation for pytest tests with CMake

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Add comment about pytest-cov version requirement for --cov-branch

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Add --pytest-with-coverage to run_test.py and mention the env var

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Rename to AMENT_CMAKE_TEST_PYTEST_WITH_COVERAGE

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Fix missing quote

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Exclude gtests from pytest coverage explicitly

They were excluded before, but only because gtests didn't use --env or --append-end.

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Append pytest-cov flags in ament_add_pytest_test() directly

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Fix ament_has_pytest_cov()

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Change default logic to avoid overriding CLI params

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Remove --cov-append pytest_cov option

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Simplify indentation

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Remove QUIET arg from ament_has_pytest_cov()

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Change ament_has_pytest_cov() to ament_get_pytest_cov_version()

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Do not return() if pytest_cov is not found in ament_add_pytest_test()

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Fix missing empty <options> argument

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Simplify pytest_cov version regex match

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Write pytest_cov results to test-specific directory

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

* Make sure to create test-specific pytest_cov directory

Signed-off-by: Christophe Bedard <bedard.christophe@gmail.com>

ament_cmake_pytest/ament_cmake_pytest-extras.cmake

@@ -18,3 +18,4 @@ find_package(ament_cmake_test QUIET REQUIRED)
 
 include("${ament_cmake_pytest_DIR}/ament_add_pytest_test.cmake")
 include("${ament_cmake_pytest_DIR}/ament_has_pytest.cmake")
+include("${ament_cmake_pytest_DIR}/ament_get_pytest_cov_version.cmake")

ament_cmake_pytest/cmake/ament_add_pytest_test.cmake

@@ -98,6 +98,48 @@ function(ament_add_pytest_test testname path)
     list(APPEND cmd "-We")
   endif()
 
+  # enable pytest coverage by default if the package test_depends on pytest_cov
+  if("python3-pytest-cov" IN_LIST ${PROJECT_NAME}_TEST_DEPENDS)
+    set(coverage_default ON)
+  else()
+    set(coverage_default OFF)
+  endif()
+  option(AMENT_CMAKE_PYTEST_WITH_COVERAGE
+    "Generate coverage information for Python tests"
+    ${coverage_default})
+
+  if(AMENT_CMAKE_PYTEST_WITH_COVERAGE)
+    # get pytest-cov version, if available
+    ament_get_pytest_cov_version(pytest_cov_version
+      PYTHON_EXECUTABLE "${ARG_PYTHON_EXECUTABLE}"
+    )
+    if(NOT pytest_cov_version)
+      message(WARNING
+        "The Python module 'pytest-cov' was not found, test coverage will not be produced "
+        "(e.g. on Ubuntu/Debian install the package 'python3-pytest-cov')")
+    else()
+      set(coverage_directory "${CMAKE_CURRENT_BINARY_DIR}/pytest_cov/${testname}")
+      file(MAKE_DIRECTORY "${coverage_directory}")
+
+      list(APPEND cmd
+        "--cov=${CMAKE_CURRENT_SOURCE_DIR}"
+        "--cov-report=html:${coverage_directory}/coverage.html"
+        "--cov-report=xml:${coverage_directory}/coverage.xml"
+      )
+
+      if(pytest_cov_version VERSION_LESS "2.5.0")
+        message(WARNING
+          "Test coverage will be produced, but will not contain branch coverage information, "
+          "because the pytest extension 'cov' does not support it "
+          "(need 2.5.0, found '${pytest_cov_version}').")
+      else()
+        list(APPEND cmd "--cov-branch")
+      endif()
+
+      list(APPEND ARG_ENV "COVERAGE_FILE=${coverage_directory}/.coverage")
+    endif()
+  endif()
+
   if(ARG_ENV)
     set(ARG_ENV "ENV" ${ARG_ENV})
   endif()

ament_cmake_pytest/cmake/ament_get_pytest_cov_version.cmake

@@ -0,0 +1,56 @@
+# Copyright 2020 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Check if the Python module `pytest-cov` was found and get its version if it is.
+#
+# :param var: the output variable name
+# :type var: string
+# :param PYTHON_EXECUTABLE: absolute path to the Python interpreter to be used,
+#   default to the CMake variable with the same name returned by
+#   FindPythonInterp
+# :type PYTHON_EXECUTABLE: string
+#
+# @public
+#
+function(ament_get_pytest_cov_version var)
+  cmake_parse_arguments(ARG "" "PYTHON_EXECUTABLE" "" ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(FATAL_ERROR "ament_get_pytest_cov_version() called with unused arguments: "
+      "${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(NOT ARG_PYTHON_EXECUTABLE)
+    set(ARG_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}")
+  endif()
+
+  set(cmd "${ARG_PYTHON_EXECUTABLE}" "-m" "pytest" "--version")
+  execute_process(
+    COMMAND ${cmd}
+    RESULT_VARIABLE res
+    OUTPUT_VARIABLE output
+    ERROR_VARIABLE error)
+  if(res EQUAL 0)
+    # check if pytest-cov is in the list of plugins
+    # (actual output of the command is in ${error} and not ${output})
+    string(REGEX MATCH "pytest-cov-([0-9]\.[0-9]\.[0-9])" pytest_cov_full_version "${error}")
+    if(pytest_cov_full_version)
+      set(${var} ${CMAKE_MATCH_1} PARENT_SCOPE)
+    else()
+      set(${var} FALSE PARENT_SCOPE)
+    endif()
+  else()
+    set(${var} FALSE PARENT_SCOPE)
+  endif()
+endfunction()