Re-add cpp coverage (#705)

* add c++ coverage * re-add macOS tests allowing for failures * Re-enable test_netconf_tcp_client.cpp
CiscoDevNet · Mar 8, 2018 · 0e92490 · 0e92490
1 parent 82206af
commit 0e92490
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 32 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -8,30 +8,32 @@ matrix:
       dist: trusty
       go: 1.8
       group: edge
-#    - os: osx
-#      osx_image: xcode7.3
-#      language: generic
-#      go: 1.8
+    - os: osx
+      osx_image: xcode7.3
+      language: generic
+      go: 1.8
     - env: DOCKER=true OS_TYPE=centos OS_VERSION=centos7
     - env: DOCKER=true OS_TYPE=ubuntu OS_VERSION=xenial
     #- env: DOCKER=true OS_TYPE=centos OS_VERSION=centos6.9 #TODO needs more work
+  allow_failures:
+    - os: osx
 
 services:
   - docker
 
 before_install:
     - if [[ "$DOCKER" != "true" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update                            ; fi
-    - if [[ "$DOCKER" != "true" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install python3                   ; fi
     - if [[ "$DOCKER" != "true" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./test/dependencies_osx.sh             ; fi
     - if [[ "$DOCKER" != "true" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./test/dependencies_ubuntu.sh         ; fi
     - if [[ "$DOCKER" == "true" ]]; then docker pull ${OS_TYPE}:${OS_VERSION}                                       ; fi
 
 script:
     - if [[ "$DOCKER" != "true" ]]; then ./test/tests.sh ;fi
-    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "centos" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/dependencies_centos.sh && ./test/tests.sh && ./test/test_package_centos.sh" ; fi
-    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "centos" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/test_package_centos.sh" ; fi
-    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "ubuntu" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/dependencies_ubuntu.sh && ./test/tests.sh && ./test/test_package_ubuntu.sh" ; fi
-    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "ubuntu" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/test_package_ubuntu.sh" ; fi
+    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "centos" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/dependencies_centos.sh && ./test/tests.sh" ; fi
+    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "centos" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/dependencies_centos.sh && ./test/test_package_centos.sh" ; fi
+    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "ubuntu" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/dependencies_ubuntu.sh && ./test/tests.sh" ; fi
+    - if [[ "$DOCKER" == "true" ]] && [[ "$OS_TYPE" == "ubuntu" ]]; then docker run -v `pwd`:/ydk-gen ${OS_TYPE}:${OS_VERSION} bash -c "cd /ydk-gen && ./test/dependencies_ubuntu.sh && ./test/test_package_ubuntu.sh" ; fi
 
 after_success:
     - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" # Upload python report to CodeCov
+
diff --git a/sdk/cpp/core/CMakeLists.txt b/sdk/cpp/core/CMakeLists.txt
@@ -84,7 +84,7 @@ elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
         set (CPACK_RPM_PACKAGE_GROUP "Development/Libraries")
         set (CPACK_RPM_PACKAGE_VENDOR ${CPACK_PACKAGE_VENDOR})
         set (CPACK_RPM_PACKAGE_URL "https://github.com/CiscoDevNet/ydk-cpp")
-        set (CPACK_RPM_PACKAGE_REQUIRES "python-pip, python-devel, libxml2-devel, libxslt-devel, libssh-devel, libcurl-devel, libtool, clang, cmake3, pcre-devel")
+        set (CPACK_RPM_PACKAGE_REQUIRES "python-pip, python-devel, libxml2-devel, libxslt-devel, libssh-devel, libcurl-devel, libtool, cmake3, pcre-devel")
 
         # <name>-<version>-<release>.<architecture>.(src).rpm
         # libydk-0.6.0-0.1.alpha.x86_64.rpm  // RPM
@@ -115,8 +115,13 @@ set(HOME_DIRECTORY $ENV{HOME})
 
 set(CMAKE_CXX_FLAGS         "${CMAKE_CXX_FLAGS} -Wall -Wextra")
 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
-set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0")
 
+if(COVERAGE STREQUAL True)
+    set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0 -fprofile-arcs -ftest-coverage")
+    set(CMAKE_BUILD_TYPE debug)
+else()
+    set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0")
+endif()
 
 if(CMAKE_BUILD_TYPE STREQUAL debug)
     option(ENABLE_BUILD_TESTS "Build tests" ON)

diff --git a/sdk/cpp/tests/CMakeLists.txt b/sdk/cpp/tests/CMakeLists.txt
@@ -43,7 +43,12 @@ find_library(ydk_ydktest_new_location ydk_ydktest_new)
 
 set(CMAKE_CXX_FLAGS         "${CMAKE_CXX_FLAGS} -Wall -Wextra")
 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
-set(CMAKE_CXX_FLAGS_DEBUG   "-g -O0")
+if(COVERAGE STREQUAL True)
+    set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0 -fprofile-arcs -ftest-coverage")
+    set(CMAKE_BUILD_TYPE debug)
+else()
+    set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0")
+endif()
 
 
 # set default build type if not specified by user

diff --git a/sdk/cpp/tests/test_netconf_tcp_client.cpp b/sdk/cpp/tests/test_netconf_tcp_client.cpp
@@ -28,7 +28,7 @@ using namespace std;
 #define NC_VERB_VERBOSE 2
 
 // NOTE: The ./ ensures this test has to be explicitly run by name
-TEST_CASE("./tcp_xr")
+TEST_CASE("tcp_xr")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -62,7 +62,7 @@ TEST_CASE("./tcp_xr")
 }
 
 
-TEST_CASE("./tcp_create")
+TEST_CASE("tcp_create")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -74,7 +74,7 @@ TEST_CASE("./tcp_create")
     REQUIRE(result == OK);
 }
 
-TEST_CASE("./tcp_edit_get_config")
+TEST_CASE("tcp_edit_get_config")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -154,7 +154,7 @@ TEST_CASE("./tcp_edit_get_config")
     REQUIRE(NULL != strstr(reply.c_str(), "<ok/>"));
 }
 
-TEST_CASE("./tcp_validate")
+TEST_CASE("tcp_validate")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -175,7 +175,7 @@ TEST_CASE("./tcp_validate")
     REQUIRE(result == OK);
 }
 
-TEST_CASE("./tcp_lock_unlock")
+TEST_CASE("tcp_lock_unlock")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -269,7 +269,7 @@ TEST_CASE("./tcp_rpc_error")
     REQUIRE(result == OK);
 }
 
-TEST_CASE("./tcp_device_not_connected_execute")
+TEST_CASE("tcp_device_not_connected_execute")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     try
@@ -292,7 +292,7 @@ TEST_CASE("./tcp_device_not_connected_execute")
 
 }
 
-TEST_CASE("./tcp_rpc_invalid")
+TEST_CASE("tcp_rpc_invalid")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int ok = 0;
@@ -343,7 +343,7 @@ TEST_CASE("./tcp_rpc_invalid")
 //     REQUIRE(result == OK);
 // }
 
-TEST_CASE("./tcp_correct_xml_wrong_rpc")
+TEST_CASE("tcp_correct_xml_wrong_rpc")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -359,7 +359,7 @@ TEST_CASE("./tcp_correct_xml_wrong_rpc")
     REQUIRE(result == OK);
 }
 
-TEST_CASE("./tcp_empty_rpc")
+TEST_CASE("tcp_empty_rpc")
 {
     NetconfTCPClient client{"admin", "admin", "127.0.0.1", 12307};
     int OK = 0;
@@ -380,7 +380,7 @@ TEST_CASE("./tcp_empty_rpc")
 }
 
 // Timeout error
-TEST_CASE("./tcp_multiple_clients")
+TEST_CASE("tcp_multiple_clients")
 {
     NetconfTCPClient client1{"admin", "admin", "127.0.0.1", 12307};
     NetconfTCPClient client2{"admin", "admin", "127.0.0.1", 12307};

diff --git a/sdk/python/core/CMakeLists.txt b/sdk/python/core/CMakeLists.txt
@@ -10,7 +10,13 @@ set(python_src python.cpp)
 # set default build type if not specified by user
 set(CMAKE_CXX_FLAGS         "${CMAKE_CXX_FLAGS} -Wall -Wextra")
 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
-set(CMAKE_CXX_FLAGS_DEBUG   "-g -O0")
+
+if(COVERAGE STREQUAL True)
+    set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0 -fprofile-arcs -ftest-coverage")
+    set(CMAKE_BUILD_TYPE debug)
+else()
+    set(CMAKE_CXX_FLAGS_DEBUG   "-g3 -O0")
+endif()
 
 if(PYTHON_VERSION)
     message( "python version: ${PYTHON_VERSION}" )

diff --git a/sdk/python/core/setup.py b/sdk/python/core/setup.py
@@ -80,13 +80,17 @@ def build_extension(self, ext):
             import pybind11
 
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
+        coverage_compiler_flag = '-DCOVERAGE=False'
+        if 'YDK_COVERAGE' in os.environ:
+            coverage_compiler_flag = '-DCOVERAGE=True'
         cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={0}'.format(extdir),
                       '-DPYBIND11_INCLUDE={0};{1}'.format(
                                       pybind11.get_include(),
                                       pybind11.get_include(user=True)),
                       '-DPYTHON_VERSION={0}'.format(
                                       get_python_version()),
-                      '-DCMAKE_BUILD_TYPE=Release']
+                      '-DCMAKE_BUILD_TYPE=Release',
+                      coverage_compiler_flag]
 
         if not os.path.exists(self.build_temp):
             os.makedirs(self.build_temp)

diff --git a/test/dependencies_centos_basic.sh b/test/dependencies_centos_basic.sh
@@ -34,7 +34,7 @@ function install_dependencies {
     yum install https://centos7.iuscommunity.org/ius-release.rpm -y > /dev/null
     yum install git which libxml2-devel libxslt-devel libssh-devel libtool gcc-c++ pcre-devel \
                 cmake3 wget curl-devel unzip python-devel python-pip make go java sudo \
-                python36u-devel python36u-pip  rpm-build redhat-lsb -y  \
+                python36u-devel python36u-pip  rpm-build redhat-lsb lcov -y  \
                 > /dev/null
 
      print_msg "Installing gcc5"
@@ -43,6 +43,7 @@ function install_dependencies {
 
      ln -sf /opt/rh/devtoolset-4/root/usr/bin/gcc /usr/bin/cc
      ln -sf /opt/rh/devtoolset-4/root/usr/bin/g++ /usr/bin/c++
+     ln -sf /opt/rh/devtoolset-4/root/usr/bin/gcov /usr/bin/gcov
 
      which gcc
      gcc --version

diff --git a/test/dependencies_ubuntu_basic.sh b/test/dependencies_ubuntu_basic.sh
@@ -64,7 +64,8 @@ function install_dependencies {
                             openjdk-8-jre \
                             golang \
                             cmake \
-                            gdebi-core > /dev/null
+                            gdebi-core\
+                            lcov > /dev/null
 
     # gcc-5 and g++5 for modern c++
     sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y

diff --git a/test/test_package_centos.sh b/test/test_package_centos.sh
@@ -49,7 +49,6 @@ function init_confd {
 ######################################
 export YDKGEN_HOME="$(pwd)"
 
-./test/dependencies_centos.sh
 init_confd $YDKGEN_HOME/sdk/cpp/core/tests/confd/ydktest
 
 ######################################
@@ -60,6 +59,7 @@ yum install -y libydk*.rpm
 
 print_msg "Installing ydk-py"
 cd sdk/python/core
+export YDK_COVERAGE=
 python setup.py sdist
 pip install dist/ydk*.tar.gz
 

diff --git a/test/test_package_ubuntu.sh b/test/test_package_ubuntu.sh
@@ -49,7 +49,6 @@ function init_confd {
 ######################################
 export YDKGEN_HOME="$(pwd)"
 
-./test/dependencies_ubuntu.sh
 init_confd $YDKGEN_HOME/sdk/cpp/core/tests/confd/ydktest
 
 ######################################
@@ -60,6 +59,7 @@ gdebi -n libydk*.deb
 
 print_msg "Installing ydk-py"
 cd sdk/python/core
+export YDK_COVERAGE=
 python setup.py sdist
 pip install dist/ydk*.tar.gz
 

diff --git a/test/tests.sh b/test/tests.sh
@@ -165,9 +165,10 @@ function install_cpp_core {
     cd $YDKGEN_HOME
     mkdir -p $YDKGEN_HOME/sdk/cpp/core/build
     cd $YDKGEN_HOME/sdk/cpp/core/build
-
-    ${CMAKE_BIN} .. && sudo make install
-    make package
+
+    print_msg "Compiling with coverage"
+    ${CMAKE_BIN} -DCOVERAGE=True .. && sudo make install
+    sudo make package || true
     cp libydk*rpm libydk*deb /ydk-gen &> /dev/null
 }
 
@@ -196,6 +197,8 @@ function install_py_core {
     print_msg "Installing py core"
     cd $YDKGEN_HOME
     cd $YDKGEN_HOME/sdk/python/core
+    print_msg "Building python with coverage"
+    export YDK_COVERAGE=
     python setup.py sdist
     pip install dist/ydk*.tar.gz
 
@@ -214,6 +217,7 @@ function run_cpp_bundle_tests {
 
     cpp_sanity_ydktest_gen_install
     cpp_sanity_ydktest_test
+    collect_cpp_coverage
 }
 
 function generate_install_specified_cpp_bundle {
@@ -246,7 +250,8 @@ function cpp_sanity_ydktest_test {
 
     print_msg "Building and running cpp bundle tests"
     mkdir -p $YDKGEN_HOME/sdk/cpp/tests/build && cd sdk/cpp/tests/build
-    run_exec_test ${CMAKE_BIN} ..
+    print_msg "Compiling with coverage"
+    run_exec_test ${CMAKE_BIN} -DCOVERAGE=True  ..
     run_exec_test make
     make test
     local status=$?
@@ -286,6 +291,15 @@ function cpp_test_gen {
     # cpp_test_gen_test
 }
 
+function collect_cpp_coverage {
+    print_msg "Collecting coverage for C++"
+    cd ${YDKGEN_HOME}/sdk/cpp/core/build
+    lcov --directory . --capture --output-file coverage.info # capture coverage info
+    lcov --remove coverage.info '/usr/*' '/Applications/*' '/opt/*' '*/json.hpp' '*/catch.hpp' '*/network_topology.cpp' '*/spdlog/*' --output-file coverage.info # filter out system
+    lcov --list coverage.info #debug info
+    cp coverage.info ${YDKGEN_HOME}
+}
+
 ######################################################################
 # Go ydktest bundle install and test functions
 ######################################################################