From 5bcd47a53ced9ac28d08ef58ef6e6152689960bf Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sat, 15 Jun 2019 20:51:51 +0200 Subject: [PATCH 01/12] Expose docker daemon in CI containers --- .travis.yml | 9 +++++++-- .travis/debian10/Dockerfile | 14 ++++++++++++++ .travis/debian7/Dockerfile | 16 ++++++++++++++++ .travis/debian8/Dockerfile | 14 ++++++++++++++ .travis/debian9/Dockerfile | 14 ++++++++++++++ 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3d0a84679..656c08499 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,13 +75,17 @@ before_script: tangocs/tango-cs:latest - TANGO_HOST_IP=$(docker inspect -f '{{ .NetworkSettings.IPAddress }}' tango_cs) - TANGO_HOST=${TANGO_HOST_IP}:10000 - - docker build --build-arg APP_UID=$(id -u) --build-arg APP_GID=$(id -g) -t cpp_tango .travis/${OS_TYPE} + - > + docker build + --build-arg APP_UID=$(id -u) + --build-arg APP_GID=$(id -g) + -t cpp_tango + .travis/${OS_TYPE} - > docker run --rm --name cpp_tango -e TANGO_HOST=${TANGO_HOST} - -e TANGO_HOST2=${TANGO_HOST_IP2}:10000 -e BINTRAY_USER_NAME=tango-ci -e BINTRAY_API_KEY=${CI_BINTRAY_API_KEY} -e COVERALLS_REPO_TOKEN=${COVERALLS_REPO_TOKEN} @@ -92,6 +96,7 @@ before_script: -v `pwd`/tango_admin:/home/tango/tango_admin -v `pwd`/coveralls-cmake:/home/tango/coveralls-cmake -v `pwd`/build-wrapper-linux-x86:/home/tango/build-wrapper-linux-x86 + -v /var/run/docker.sock:/var/run/docker.sock -dit cpp_tango - .travis/install_tango_idl.sh diff --git a/.travis/debian10/Dockerfile b/.travis/debian10/Dockerfile index dbe76fa8f..c036e8c4d 100644 --- a/.travis/debian10/Dockerfile +++ b/.travis/debian10/Dockerfile @@ -20,10 +20,24 @@ RUN apt-get update && apt-get install -y \ omniidl \ python2 +RUN apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian buster stable" \ + && apt-get update \ + && apt-get install -y docker-ce + RUN groupadd -g "$APP_GID" tango RUN useradd -u "$APP_UID" -g "$APP_GID" -ms /bin/bash tango +RUN usermod -a -G docker tango + ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig USER tango diff --git a/.travis/debian7/Dockerfile b/.travis/debian7/Dockerfile index f16f6a0c7..1bab58876 100644 --- a/.travis/debian7/Dockerfile +++ b/.travis/debian7/Dockerfile @@ -28,10 +28,26 @@ RUN wget --no-check-certificate https://cmake.org/files/v3.10/cmake-3.10.0-Linux && chmod +x /tmp/cmake-install.sh \ && /tmp/cmake-install.sh --skip-license --exclude-subdir +RUN apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common \ + python-software-properties \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian wheezy stable" \ + && sed -i -e '/^deb-src.*docker.*/d' /etc/apt/sources.list \ + && apt-get update \ + && apt-get install -y docker-ce + RUN groupadd -g "$APP_GID" tango RUN useradd -u "$APP_UID" -g "$APP_GID" -ms /bin/bash tango +RUN usermod -a -G docker tango + ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig USER tango diff --git a/.travis/debian8/Dockerfile b/.travis/debian8/Dockerfile index 7f364e2aa..30004da17 100644 --- a/.travis/debian8/Dockerfile +++ b/.travis/debian8/Dockerfile @@ -21,10 +21,24 @@ RUN apt-get update && apt-get install -y \ libomnithread3-dev \ libzmq3-dev +RUN apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian jessie stable" \ + && apt-get update \ + && apt-get install -y docker-ce + RUN groupadd -g "$APP_GID" tango RUN useradd -u "$APP_UID" -g "$APP_GID" -ms /bin/bash tango +RUN usermod -a -G docker tango + ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig USER tango diff --git a/.travis/debian9/Dockerfile b/.travis/debian9/Dockerfile index 7e8dd7dfb..388778fa7 100644 --- a/.travis/debian9/Dockerfile +++ b/.travis/debian9/Dockerfile @@ -19,10 +19,24 @@ RUN apt-get update && apt-get install -y \ libomnithread3-dev \ libzmq3-dev +RUN apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \ + && apt-get update \ + && apt-get install -y docker-ce + RUN groupadd -g "$APP_GID" tango RUN useradd -u "$APP_UID" -g "$APP_GID" -ms /bin/bash tango +RUN usermod -a -G docker tango + ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig USER tango From 1c1e205fb7d44884f3ee35c81723065d9fa6bc38 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sat, 15 Jun 2019 23:13:01 +0200 Subject: [PATCH 02/12] Add new test runner and server start/stop scripts The test runner will start a fresh environment (database and device servers) for each test case. --- cpp_test_suite/environment/CMakeLists.txt | 9 ++ .../environment/kill_server.sh.cmake | 16 +++ .../environment/run_with_fixture.sh.cmake | 112 ++++++++++++++++++ .../environment/start_server.sh.cmake | 33 ++++++ 4 files changed, 170 insertions(+) create mode 100755 cpp_test_suite/environment/kill_server.sh.cmake create mode 100755 cpp_test_suite/environment/run_with_fixture.sh.cmake create mode 100755 cpp_test_suite/environment/start_server.sh.cmake diff --git a/cpp_test_suite/environment/CMakeLists.txt b/cpp_test_suite/environment/CMakeLists.txt index f22e4b28b..ceaa1a3f8 100644 --- a/cpp_test_suite/environment/CMakeLists.txt +++ b/cpp_test_suite/environment/CMakeLists.txt @@ -21,3 +21,12 @@ execute_process(COMMAND chmod +x ${CMAKE_CURRENT_SOURCE_DIR}/shutdown.sh) add_custom_target(stop-tango ${CMAKE_CURRENT_SOURCE_DIR}/shutdown.sh WORKING_DIRECTORY ${PROJECT_BINARY_DIR} SOURCES shutdown.sh) + +configure_file(run_with_fixture.sh.cmake run_with_fixture.sh @ONLY) +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/run_with_fixture.sh) + +configure_file(start_server.sh.cmake start_server.sh @ONLY) +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/start_server.sh) + +configure_file(kill_server.sh.cmake kill_server.sh @ONLY) +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/kill_server.sh) diff --git a/cpp_test_suite/environment/kill_server.sh.cmake b/cpp_test_suite/environment/kill_server.sh.cmake new file mode 100755 index 000000000..04766286e --- /dev/null +++ b/cpp_test_suite/environment/kill_server.sh.cmake @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# vim: syntax=sh + +pids="$(find \ + "$TANGO_TEST_CASE_DIRECTORY/server_pids/" \ + -maxdepth 1 \ + -mindepth 1 \ + -type d \ + -printf "%f\n" \ + 2> /dev/null)" + +echo "Killing PIDS: " $pids + +kill $pids &>/dev/null + +sleep 2 diff --git a/cpp_test_suite/environment/run_with_fixture.sh.cmake b/cpp_test_suite/environment/run_with_fixture.sh.cmake new file mode 100755 index 000000000..21fe3f61c --- /dev/null +++ b/cpp_test_suite/environment/run_with_fixture.sh.cmake @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# vim: syntax=sh + +set -e + +if [ $# -lt 1 ]; then + echo "usage: $0 [args...]" + exit 1 +fi + +tc_program="$1" +tc_run_name="$(basename "$tc_program")_$(date '+%Y%m%d.%H%M%S.%N')" +shift 1 + +tc_tango_container="tango_cs_${tc_run_name}" +tc_mysql_container="mysql_db_${tc_run_name}" + +export TANGO_TEST_CASE_DIRECTORY="@PROJECT_BINARY_DIR@/test_results/${tc_run_name}" +mkdir -p "$TANGO_TEST_CASE_DIRECTORY" + +function teardown { + ( + # close stdout and stderr to allow ctest finish the test early + exec 1<&- + exec 2<&- + "@CMAKE_CURRENT_BINARY_DIR@/kill_server.sh" &>/dev/null || true + docker stop "$tc_tango_container" "$tc_mysql_container" &>/dev/null || true + ) & +} + + +if [[ -z "$TANGO_TEST_CASE_SKIP_FIXTURE" ]]; then + + trap teardown EXIT + +# --- setup database ---------------------------------------------------------- + + docker run \ + --name "$tc_mysql_container" \ + --rm \ + -e MYSQL_ROOT_PASSWORD=root \ + -e MYSQL_INITDB_SKIP_TZINFO=1 \ + -d \ + tangocs/mysql:9.2.2 \ + --sql-mode="" \ + --innodb=OFF \ + --default-storage-engine=MyISAM \ + > /dev/null + + docker run \ + --name "$tc_tango_container" \ + --rm \ + -e TANGO_HOST=127.0.0.1:10000 \ + -e MYSQL_HOST=mysql_db:3306 \ + -e MYSQL_USER=tango \ + -e MYSQL_PASSWORD=tango \ + -e MYSQL_DATABASE=tango \ + --link "$tc_mysql_container":mysql_db \ + -d \ + tangocs/tango-cs:latest \ + > /dev/null + + tc_tango_ipaddr="$(docker inspect \ + -f '{{ .NetworkSettings.IPAddress }}' \ + "$tc_tango_container")" + + export TANGO_HOST="$tc_tango_ipaddr:10000" + + if hash tango_admin 2>/dev/null; then + if ! tango_admin --ping-database 30; then + echo "Failed to ping database" + exit 1 + fi + else + sleep 30 + fi + +# -- setup devices ------------------------------------------------------------ + + "@PROJECT_BINARY_DIR@/new_tests/conf_devtest" \ + @SERV_NAME@/@INST_NAME@ \ + @DEV1@ \ + @DEV2@ \ + @DEV3@ \ + @DEV1_ALIAS@ \ + @ATTR_ALIAS@ \ + @FWD_DEV@ \ + @DEV20@ \ + &> "${TANGO_TEST_CASE_DIRECTORY}/conf_devtest.out" + + "@CMAKE_CURRENT_BINARY_DIR@/start_server.sh" @INST_NAME@ DevTest cpp_test_ds + + "@CMAKE_CURRENT_BINARY_DIR@/start_server.sh" @INST_NAME@ FwdTest cpp_test_ds/fwd_ds + +fi # TANGO_TEST_CASE_SKIP_FIXTURE + + +# --- run test ---------------------------------------------------------------- + +set +e +set -o pipefail + +"$tc_program" "$@" 2>&1 | tee "${TANGO_TEST_CASE_DIRECTORY}/testcase.out" + +tc_exit_code="$?" + +set +o pipefail +set -e + +echo "$tc_exit_code" > "${TANGO_TEST_CASE_DIRECTORY}/testcase_exit_code.out" + +exit "$tc_exit_code" diff --git a/cpp_test_suite/environment/start_server.sh.cmake b/cpp_test_suite/environment/start_server.sh.cmake new file mode 100755 index 000000000..0bbc117df --- /dev/null +++ b/cpp_test_suite/environment/start_server.sh.cmake @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# vim: syntax=sh + +set -e + +instance="$1" +server="${2:-DevTest}" +server_path="${3:-cpp_test_ds}" + +index="$(find \ + "$TANGO_TEST_CASE_DIRECTORY" \ + -maxdepth 1 \ + -mindepth 1 \ + -type f \ + -name "${server}_${instance}.*.out" | wc -l)" + +echo "Starting $server/$instance" + +"@PROJECT_BINARY_DIR@/${server_path}/${server}" "$instance" -v5 \ + &> "${TANGO_TEST_CASE_DIRECTORY}/${server}_${instance}.${index}.out" & + +server_pid="$!" + +mkdir -p "${TANGO_TEST_CASE_DIRECTORY}/server_pids/${server_pid}" + +if hash tango_admin 2>/dev/null; then + instance_lower="$(echo "$server/$instance" | awk '{print tolower($0)}')" + tango_admin --ping-device "dserver/${instance_lower}" 7 +else + sleep 7 +fi + +echo "$server/$instance PID: ${server_pid}" From c243cb28fa478a9bc068c99a07c3f4542f8fa4bf Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sat, 15 Jun 2019 23:39:09 +0200 Subject: [PATCH 03/12] Enable new test runner script (9d6a727) --- .travis.yml | 28 ---------- .travis/test.sh | 13 ----- cpp_test_suite/CMakeLists.txt | 9 +++- cpp_test_suite/CTestCustom.cmake | 2 - cpp_test_suite/asyn/CMakeLists.txt | 22 ++++---- cpp_test_suite/cxxtest/CMakeLists.txt | 2 +- cpp_test_suite/cxxtest/include/config.h.cmake | 6 +-- cpp_test_suite/event/CMakeLists.txt | 34 ++++++------ cpp_test_suite/old_tests/CMakeLists.txt | 52 +++++++++---------- 9 files changed, 65 insertions(+), 103 deletions(-) delete mode 100644 cpp_test_suite/CTestCustom.cmake diff --git a/.travis.yml b/.travis.yml index 656c08499..28e44abd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,31 +50,6 @@ before_install: - wget https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip && unzip build-wrapper-linux-x86.zip before_script: - - > - docker run - --rm - --name mysql_db - -e MYSQL_ROOT_PASSWORD=root - -e MYSQL_INITDB_SKIP_TZINFO=1 - -d - tangocs/mysql:9.2.2 - --sql-mode="" - --innodb=OFF - --default-storage-engine=MyISAM - - > - docker run - --rm - --name tango_cs - -e TANGO_HOST=127.0.0.1:10000 - -e MYSQL_HOST=mysql_db:3306 - -e MYSQL_USER=tango - -e MYSQL_PASSWORD=tango - -e MYSQL_DATABASE=tango - --link mysql_db:mysq_db - -d - tangocs/tango-cs:latest - - TANGO_HOST_IP=$(docker inspect -f '{{ .NetworkSettings.IPAddress }}' tango_cs) - - TANGO_HOST=${TANGO_HOST_IP}:10000 - > docker build --build-arg APP_UID=$(id -u) @@ -89,7 +64,6 @@ before_script: -e BINTRAY_USER_NAME=tango-ci -e BINTRAY_API_KEY=${CI_BINTRAY_API_KEY} -e COVERALLS_REPO_TOKEN=${COVERALLS_REPO_TOKEN} - --link tango_cs -v `pwd`:/home/tango/src -v `pwd`/idl:/home/tango/idl -v `pwd`/cppzmq:/home/tango/cppzmq @@ -124,5 +98,3 @@ deploy: after-script: - docker stop cpp_tango - - docker stop tango_cs - - docker stop mysql_db diff --git a/.travis/test.sh b/.travis/test.sh index 2ec8fb145..41db77546 100755 --- a/.travis/test.sh +++ b/.travis/test.sh @@ -12,12 +12,6 @@ then TEST_COMMAND="exec make coveralls" fi -echo "PreTest" -docker exec cpp_tango /bin/sh -c 'cd /home/tango/src/build/cpp_test_suite/environment; exec ./pre_test.sh' -if [ $? -ne "0" ] -then - exit -1 -fi echo "Test" echo "TEST_COMMAND=$TEST_COMMAND" docker exec cpp_tango /bin/sh -c "cd /home/tango/src/build; $TEST_COMMAND" @@ -25,10 +19,3 @@ if [ $? -ne "0" ] then exit -1 fi - -echo "PostTest" -docker exec cpp_tango /bin/sh -c 'cd /home/tango/src/build/cpp_test_suite/environment; exec ./post_test.sh' -if [ $? -ne "0" ] -then - exit -1 -fi diff --git a/cpp_test_suite/CMakeLists.txt b/cpp_test_suite/CMakeLists.txt index 468d2c356..d28647afa 100644 --- a/cpp_test_suite/CMakeLists.txt +++ b/cpp_test_suite/CMakeLists.txt @@ -2,8 +2,6 @@ project(test_suite) set(CMAKE_CXX_FLAGS_RELEASE "-O0 -g") -configure_file(CTestCustom.cmake CTestCustom.cmake) - #TODO different OS set(SERV_NAME "DevTest") set(INST_NAME "test") @@ -33,6 +31,13 @@ macro(TEST_SUITE_ADD_TEST test) # add_test(NAME "CPP::${test}" COMMAND $ ${DEV1} ${DEV2} ${DEV3} ${DEV1_ALIAS}) endmacro() +function(tango_add_test name_tag name command_tag command) + add_test( + NAME "${name}" + COMMAND "${CMAKE_BINARY_DIR}/cpp_test_suite/environment/run_with_fixture.sh" "${command}" + ${ARGN}) +endfunction() + add_subdirectory(cxxtest) add_subdirectory(old_tests) diff --git a/cpp_test_suite/CTestCustom.cmake b/cpp_test_suite/CTestCustom.cmake deleted file mode 100644 index 44e09541c..000000000 --- a/cpp_test_suite/CTestCustom.cmake +++ /dev/null @@ -1,2 +0,0 @@ -set(CTEST_CUSTOM_PRE_TEST ${PROJECT_BINARY_DIR}/environment/pre_test.sh) -set(CTEST_CUSTOM_POST_TEST ${PROJECT_BINARY_DIR}/environment/post_test.sh) \ No newline at end of file diff --git a/cpp_test_suite/asyn/CMakeLists.txt b/cpp_test_suite/asyn/CMakeLists.txt index 2f8ae5c70..3947ee77e 100644 --- a/cpp_test_suite/asyn/CMakeLists.txt +++ b/cpp_test_suite/asyn/CMakeLists.txt @@ -16,14 +16,14 @@ foreach(TEST ${TESTS}) TEST_SUITE_ADD_TEST(${TEST}) endforeach(TEST) -add_test(NAME "asyn::asyn_cmd" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_attr" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_attr_multi" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_write_attr" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_write_attr_multi" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_cb" COMMAND $ ${DEV1} ${DEV2}) -add_test(NAME "asyn::asyn_cb2" COMMAND $ ${DEV1} ${DEV2}) -add_test(NAME "asyn::asyn_cb_cmd" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_attr_cb" COMMAND $ ${DEV1}) -add_test(NAME "asyn::asyn_write_cb" COMMAND $ ${DEV1}) -add_test(NAME "asyn::auto_asyn_cmd" COMMAND $ ${DEV1}) \ No newline at end of file +tango_add_test(NAME "asyn::asyn_cmd" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_attr" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_attr_multi" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_write_attr" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_write_attr_multi" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_cb" COMMAND $ ${DEV1} ${DEV2}) +tango_add_test(NAME "asyn::asyn_cb2" COMMAND $ ${DEV1} ${DEV2}) +tango_add_test(NAME "asyn::asyn_cb_cmd" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_attr_cb" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::asyn_write_cb" COMMAND $ ${DEV1}) +tango_add_test(NAME "asyn::auto_asyn_cmd" COMMAND $ ${DEV1}) diff --git a/cpp_test_suite/cxxtest/CMakeLists.txt b/cpp_test_suite/cxxtest/CMakeLists.txt index 82ca5da20..677fdfff5 100644 --- a/cpp_test_suite/cxxtest/CMakeLists.txt +++ b/cpp_test_suite/cxxtest/CMakeLists.txt @@ -27,7 +27,7 @@ macro(CXX_GENERATE_TEST name) tango_target_test_pch(${name}) endif() - add_test(NAME "CXX::${name}" COMMAND $ + tango_add_test(NAME "CXX::${name}" COMMAND $ --device1=${DEV1} --device2=${DEV2} --device3=${DEV3} diff --git a/cpp_test_suite/cxxtest/include/config.h.cmake b/cpp_test_suite/cxxtest/include/config.h.cmake index e01262334..bee04809a 100644 --- a/cpp_test_suite/cxxtest/include/config.h.cmake +++ b/cpp_test_suite/cxxtest/include/config.h.cmake @@ -5,8 +5,8 @@ namespace Tango { const std::string kProjectBinaryDir{"@PROJECT_BINARY_DIR@"}; - const std::string kStartServerCmd{"@CMAKE_CURRENT_BINARY_DIR@/bin/start_server.sh "}; - const std::string kKillServerCmd{"@CMAKE_CURRENT_BINARY_DIR@/bin/kill_server.sh"}; + const std::string kStartServerCmd{"@CMAKE_BINARY_DIR@/cpp_test_suite/environment/start_server.sh "}; + const std::string kKillServerCmd{"@CMAKE_BINARY_DIR@/cpp_test_suite/environment/kill_server.sh"}; } -#endif \ No newline at end of file +#endif diff --git a/cpp_test_suite/event/CMakeLists.txt b/cpp_test_suite/event/CMakeLists.txt index e042432e9..6ada0b6ee 100644 --- a/cpp_test_suite/event/CMakeLists.txt +++ b/cpp_test_suite/event/CMakeLists.txt @@ -24,20 +24,20 @@ foreach(TEST ${TESTS}) TEST_SUITE_ADD_TEST(${TEST}) endforeach(TEST) -add_test(NAME "event::per_event" COMMAND $ ${DEV1} ${DEV2}) -add_test(NAME "event::att_type_event" COMMAND $ ${DEV1}) -add_test(NAME "event::change_event" COMMAND $ ${DEV1}) -add_test(NAME "event::change_event64" COMMAND $ ${DEV1}) -add_test(NAME "event::change_event_buffer" COMMAND $ ${DEV1}) -add_test(NAME "event::archive_event" COMMAND $ ${DEV1}) -add_test(NAME "event::user_event" COMMAND $ ${DEV1}) -add_test(NAME "event::multi_event" COMMAND $ ${DEV1}) -add_test(NAME "event::state_event" COMMAND $ ${DEV1}) -add_test(NAME "event::att_conf_event" COMMAND $ ${DEV1}) -add_test(NAME "event::att_conf_event_buffer" COMMAND $ ${DEV1}) -add_test(NAME "event::data_ready_event" COMMAND $ ${DEV1}) -add_test(NAME "event::data_ready_event_buffer" COMMAND $ ${DEV1}) -add_test(NAME "event::multi_dev_event" COMMAND $ ${DEV1} ${DEV2} ${DEV3}) -add_test(NAME "event::dev_intr_event" COMMAND $ ${DEV1}) -add_test(NAME "event::pipe_event" COMMAND $ ${DEV1}) -add_test(NAME "event::event_lock" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::per_event" COMMAND $ ${DEV1} ${DEV2}) +tango_add_test(NAME "event::att_type_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::change_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::change_event64" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::change_event_buffer" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::archive_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::user_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::multi_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::state_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::att_conf_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::att_conf_event_buffer" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::data_ready_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::data_ready_event_buffer" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::multi_dev_event" COMMAND $ ${DEV1} ${DEV2} ${DEV3}) +tango_add_test(NAME "event::dev_intr_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::pipe_event" COMMAND $ ${DEV1}) +tango_add_test(NAME "event::event_lock" COMMAND $ ${DEV1}) diff --git a/cpp_test_suite/old_tests/CMakeLists.txt b/cpp_test_suite/old_tests/CMakeLists.txt index 668a8a6b8..33aa48228 100644 --- a/cpp_test_suite/old_tests/CMakeLists.txt +++ b/cpp_test_suite/old_tests/CMakeLists.txt @@ -51,33 +51,33 @@ endforeach(TEST) configure_file(locked_device_cmd.h.cmake locked_device_cmd.h @ONLY) target_include_directories(lock PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -add_test(NAME "old_tests::misc_devproxy" COMMAND $ ${DEV1} ${SERV_NAME}/${INST_NAME}) +tango_add_test(NAME "old_tests::misc_devproxy" COMMAND $ ${DEV1} ${SERV_NAME}/${INST_NAME}) -add_test(NAME "old_tests::cmd_types" COMMAND $ ${DEV1} 10) +tango_add_test(NAME "old_tests::cmd_types" COMMAND $ ${DEV1} 10) -add_test(NAME "old_tests::attr_types" COMMAND $ ${DEV1} 10) -add_test(NAME "old_tests::att_conf" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::misc_devdata" COMMAND $) -add_test(NAME "old_tests::misc_devattr" COMMAND $) -add_test(NAME "old_tests::write_attr" COMMAND $ ${DEV1} 10) -add_test(NAME "old_tests::copy_devproxy" COMMAND $ ${DEV1} ${DEV2} ${DEV3}) -add_test(NAME "old_tests::attr_misc" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::obj_prop" COMMAND $) -add_test(NAME "old_tests::attr_proxy" COMMAND $ ${DEV1}/Short_attr_rw) -add_test(NAME "old_tests::write_attr_3" COMMAND $ ${DEV1} 10) -add_test(NAME "old_tests::read_hist_ext" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::ring_depth" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::mem_att" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::state_attr" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::rds" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::ds_cache" COMMAND $) -add_test(NAME "old_tests::w_r_attr" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::lock" COMMAND $ ${DEV1} ${DEV2}) -add_test(NAME "old_tests::sub_dev" COMMAND $ ${DEV1} ${DEV2} ${DEV3}) -add_test(NAME "old_tests::print_data" COMMAND $ ${DEV1}) -add_test(NAME "old_tests::attr_manip" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::attr_types" COMMAND $ ${DEV1} 10) +tango_add_test(NAME "old_tests::att_conf" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::misc_devdata" COMMAND $) +tango_add_test(NAME "old_tests::misc_devattr" COMMAND $) +tango_add_test(NAME "old_tests::write_attr" COMMAND $ ${DEV1} 10) +tango_add_test(NAME "old_tests::copy_devproxy" COMMAND $ ${DEV1} ${DEV2} ${DEV3}) +tango_add_test(NAME "old_tests::attr_misc" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::obj_prop" COMMAND $) +tango_add_test(NAME "old_tests::attr_proxy" COMMAND $ ${DEV1}/Short_attr_rw) +tango_add_test(NAME "old_tests::write_attr_3" COMMAND $ ${DEV1} 10) +tango_add_test(NAME "old_tests::read_hist_ext" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::ring_depth" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::mem_att" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::state_attr" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::rds" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::ds_cache" COMMAND $) +tango_add_test(NAME "old_tests::w_r_attr" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::lock" COMMAND $ ${DEV1} ${DEV2}) +tango_add_test(NAME "old_tests::sub_dev" COMMAND $ ${DEV1} ${DEV2} ${DEV3}) +tango_add_test(NAME "old_tests::print_data" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::attr_manip" COMMAND $ ${DEV1}) if (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 4.9.2) - add_test(NAME "old_tests::size" COMMAND $) + tango_add_test(NAME "old_tests::size" COMMAND $) endif() -add_test(NAME "old_tests::ConfEventBugClient" COMMAND $ ${DEV3}) -add_test(NAME "old_tests::attr_conf_test" COMMAND $ ${DEV1}) +tango_add_test(NAME "old_tests::ConfEventBugClient" COMMAND $ ${DEV3}) +tango_add_test(NAME "old_tests::attr_conf_test" COMMAND $ ${DEV1}) From 445764a8c7ea0979ac608df38a87ec3a5ce5c129 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sun, 15 Dec 2019 09:41:43 +0100 Subject: [PATCH 04/12] Remove old scripts for running the tests The scripts are not needed anymore as ctest command sets the test environment up. --- cpp_test_suite/cxxtest/CMakeLists.txt | 14 ----- .../cxxtest/bin/kill_server.sh.cmake | 15 ------ .../cxxtest/bin/start_server.sh.cmake | 17 ------ cpp_test_suite/environment/CMakeLists.txt | 22 -------- cpp_test_suite/environment/post_test.sh.cmake | 11 ---- cpp_test_suite/environment/pre_test.sh.cmake | 32 ----------- cpp_test_suite/environment/run.sh.cmake | 4 -- cpp_test_suite/environment/setup.sh | 53 ------------------- cpp_test_suite/environment/shutdown.sh | 10 ---- cpp_test_suite/environment/tango_host.cmake | 2 - 10 files changed, 180 deletions(-) delete mode 100644 cpp_test_suite/cxxtest/bin/kill_server.sh.cmake delete mode 100644 cpp_test_suite/cxxtest/bin/start_server.sh.cmake delete mode 100644 cpp_test_suite/environment/post_test.sh.cmake delete mode 100644 cpp_test_suite/environment/pre_test.sh.cmake delete mode 100644 cpp_test_suite/environment/run.sh.cmake delete mode 100755 cpp_test_suite/environment/setup.sh delete mode 100755 cpp_test_suite/environment/shutdown.sh delete mode 100644 cpp_test_suite/environment/tango_host.cmake diff --git a/cpp_test_suite/cxxtest/CMakeLists.txt b/cpp_test_suite/cxxtest/CMakeLists.txt index 677fdfff5..9c981d0ad 100644 --- a/cpp_test_suite/cxxtest/CMakeLists.txt +++ b/cpp_test_suite/cxxtest/CMakeLists.txt @@ -91,18 +91,4 @@ CXX_GENERATE_TEST(cxx_stateless_subscription) CXX_GENERATE_TEST(cxx_nan_inf_in_prop) CXX_GENERATE_TEST(cxx_asyn_reconnection) -#utilities -configure_file(bin/start_server.sh.cmake ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/start_server.sh @ONLY) -configure_file(bin/kill_server.sh.cmake ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/kill_server.sh @ONLY) - -# now copy the temporary into the final destination, setting the permissions -file(COPY ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/start_server.sh - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/bin - FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ - GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) -file(COPY ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/kill_server.sh - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/bin - FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ - GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) - configure_file(include/config.h.cmake include/config.h @ONLY) diff --git a/cpp_test_suite/cxxtest/bin/kill_server.sh.cmake b/cpp_test_suite/cxxtest/bin/kill_server.sh.cmake deleted file mode 100644 index 84ab8a367..000000000 --- a/cpp_test_suite/cxxtest/bin/kill_server.sh.cmake +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -kill_servers(){ - PIDS=`ps -e | grep DevTest | grep -v grep | awk '{print $1}'` - echo "PIDS = $PIDS" - for pid in $PIDS - do - echo "Killing process with PID $pid" - kill $pid - done - - sleep 2 -} - -kill_servers \ No newline at end of file diff --git a/cpp_test_suite/cxxtest/bin/start_server.sh.cmake b/cpp_test_suite/cxxtest/bin/start_server.sh.cmake deleted file mode 100644 index d26502e94..000000000 --- a/cpp_test_suite/cxxtest/bin/start_server.sh.cmake +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -start_server(){ - echo "Starting DevTest $1" - @PROJECT_BINARY_DIR@/cpp_test_ds/DevTest $1 -v5 1>@PROJECT_BINARY_DIR@/cpp_test_ds/DevTest_$1.out 2>&1 & - echo $! > @PROJECT_BINARY_DIR@/cpp_test_ds/DevTest_$1.pid - - if hash tango_admin 2>/dev/null; then - tango_admin --ping-device "dserver/devtest/$1" 7 - else - sleep 7 - fi - - echo "Done. PID="`cat @PROJECT_BINARY_DIR@/cpp_test_ds/DevTest_$1.pid` -} - -start_server $1 diff --git a/cpp_test_suite/environment/CMakeLists.txt b/cpp_test_suite/environment/CMakeLists.txt index ceaa1a3f8..e4947b71f 100644 --- a/cpp_test_suite/environment/CMakeLists.txt +++ b/cpp_test_suite/environment/CMakeLists.txt @@ -1,26 +1,4 @@ #TODO windows -configure_file(tango_host.cmake tango_host @ONLY) - -configure_file(pre_test.sh.cmake pre_test.sh @ONLY) -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/pre_test.sh) -configure_file(post_test.sh.cmake post_test.sh @ONLY) -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/post_test.sh) - -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_SOURCE_DIR}/setup.sh) -add_custom_target(start-tango ${CMAKE_CURRENT_SOURCE_DIR}/setup.sh - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - SOURCES setup.sh) - -configure_file(run.sh.cmake run.sh @ONLY) -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/run.sh) -add_custom_target(run-tests ${CMAKE_CURRENT_BINARY_DIR}/run.sh - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - SOURCES run.sh.cmake) - -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_SOURCE_DIR}/shutdown.sh) -add_custom_target(stop-tango ${CMAKE_CURRENT_SOURCE_DIR}/shutdown.sh - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - SOURCES shutdown.sh) configure_file(run_with_fixture.sh.cmake run_with_fixture.sh @ONLY) execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/run_with_fixture.sh) diff --git a/cpp_test_suite/environment/post_test.sh.cmake b/cpp_test_suite/environment/post_test.sh.cmake deleted file mode 100644 index 34d4c1ce4..000000000 --- a/cpp_test_suite/environment/post_test.sh.cmake +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -#TODO Starter? - -PID=$(<@PROJECT_BINARY_DIR@/cpp_test_ds/DevTest_@INST_NAME@.pid) -kill $PID - -PID=$(<@PROJECT_BINARY_DIR@/cpp_test_ds/fwd_ds/FwdTest_@INST_NAME@.pid) -kill $PID - -exit 0 diff --git a/cpp_test_suite/environment/pre_test.sh.cmake b/cpp_test_suite/environment/pre_test.sh.cmake deleted file mode 100644 index 1bf9dffe9..000000000 --- a/cpp_test_suite/environment/pre_test.sh.cmake +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -source tango_host - -echo "TANGO_HOST=$TANGO_HOST" - -echo "Run conf_devtest" -@PROJECT_BINARY_DIR@/new_tests/conf_devtest @SERV_NAME@/@INST_NAME@ @DEV1@ @DEV2@ @DEV3@ @DEV1_ALIAS@ @ATTR_ALIAS@ @FWD_DEV@ @DEV20@ - -echo "Start DevTest" -echo "DevTest libraries:" -echo "`ldd @PROJECT_BINARY_DIR@/cpp_test_ds/DevTest`" -#TODO Starter? -@PROJECT_BINARY_DIR@/cpp_test_ds/DevTest @INST_NAME@ -v5 1>@PROJECT_BINARY_DIR@/cpp_test_ds/DevTest_@INST_NAME@.out 2>&1 & -echo $! > @PROJECT_BINARY_DIR@/cpp_test_ds/DevTest_@INST_NAME@.pid - -if hash tango_admin 2>/dev/null; then - tango_admin --ping-device "dserver/devtest/@INST_NAME@" 3 -else - sleep 3 -fi - -echo "Start FwdTest" -echo "FwdTest libraries:" -echo "`ldd @PROJECT_BINARY_DIR@/cpp_test_ds/fwd_ds/FwdTest`" -@PROJECT_BINARY_DIR@/cpp_test_ds/fwd_ds/FwdTest @INST_NAME@ -v5 1>@PROJECT_BINARY_DIR@/cpp_test_ds/fwd_ds/FwdTest_@INST_NAME@.out 2>&1 & -echo $! > @PROJECT_BINARY_DIR@/cpp_test_ds/fwd_ds/FwdTest_@INST_NAME@.pid - -if hash tango_admin 2>/dev/null; then - tango_admin --ping-device "dserver/fwdtest/@INST_NAME@" 3 -else - sleep 3 -fi diff --git a/cpp_test_suite/environment/run.sh.cmake b/cpp_test_suite/environment/run.sh.cmake deleted file mode 100644 index 3aaeabb43..000000000 --- a/cpp_test_suite/environment/run.sh.cmake +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -source tango_host -echo "TANGO_HOST=$TANGO_HOST" -@CMAKE_CTEST_COMMAND@ -V \ No newline at end of file diff --git a/cpp_test_suite/environment/setup.sh b/cpp_test_suite/environment/setup.sh deleted file mode 100755 index 3c6e2f4d2..000000000 --- a/cpp_test_suite/environment/setup.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -echo "Setup test environment" - -function run_mysql_container { - docker run \ - --rm \ - --name $1 \ - -e MYSQL_ROOT_PASSWORD=root \ - -e MYSQL_INITDB_SKIP_TZINFO=1 \ - -d \ - tangocs/mysql:9.2.2 \ - --sql-mode="" \ - --innodb=OFF \ - --default-storage-engine=MyISAM -} - -function run_tango_container { - docker run \ - --rm \ - --name $1 \ - -e TANGO_HOST=127.0.0.1:10000 \ - -e MYSQL_HOST=$2:3306 \ - -e MYSQL_USER=tango \ - -e MYSQL_PASSWORD=tango \ - -e MYSQL_DATABASE=tango \ - --link $2:$2 \ - -d \ - tangocs/tango-cs:latest -} - -run_mysql_container mysql_db -run_tango_container tango_cs mysql_db - -IPADDR=$(docker inspect -f '{{ .NetworkSettings.IPAddress }}' tango_cs) - -export TANGO_HOST=$IPADDR:10000 - -echo "TANGO_HOST=$TANGO_HOST" - -echo "Create tango_host file" - -cat << EOF > tango_host -#!/bin/bash -export TANGO_HOST=$TANGO_HOST -EOF - -echo "Wait till tango-cs is online" -if hash tango_admin 2>/dev/null; then - tango_admin --ping-database 30 -else - sleep 30 -fi diff --git a/cpp_test_suite/environment/shutdown.sh b/cpp_test_suite/environment/shutdown.sh deleted file mode 100755 index 2487a56cb..000000000 --- a/cpp_test_suite/environment/shutdown.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -echo "Shutdown test environment" -docker stop tango_cs -docker stop mysql_db - -echo "Revert tango_host file" -echo "TANGO_HOST=$TANGO_HOST" -echo "#!/bin/bash" > tango_host -echo "export TANGO_HOST=$TANGO_HOST" >> tango_host diff --git a/cpp_test_suite/environment/tango_host.cmake b/cpp_test_suite/environment/tango_host.cmake deleted file mode 100644 index 3a38ccf06..000000000 --- a/cpp_test_suite/environment/tango_host.cmake +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -export TANGO_HOST=@TANGO_HOST@ \ No newline at end of file From 7cd2351a06503243d20a733cde5d93f20e5e6005 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sun, 15 Dec 2019 11:05:04 +0100 Subject: [PATCH 05/12] Run tests in parallel in CI If some tests failed during parallel run, try to re-run them sequentially to see if the failure is permanent. --- .travis/run.sh | 3 ++- .travis/run_ctest.sh | 14 ++++++++++++++ .travis/test.sh | 18 +++++++----------- 3 files changed, 23 insertions(+), 12 deletions(-) create mode 100755 .travis/run_ctest.sh diff --git a/.travis/run.sh b/.travis/run.sh index a44c596a2..fc04459ea 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -31,7 +31,8 @@ docker exec cpp_tango cmake \ -DTANGO_USE_USING_NAMESPACE=${TANGO_USE_USING_NAMESPACE} \ -DWARNINGS_AS_ERRORS=${WARNINGS_AS_ERRORS} \ -DCOVERALLS=${COVERALLS} \ - -DCOVERALLS_MODULE_PATH=${COVERALLS_MODULE_PATH} + -DCOVERALLS_MODULE_PATH=${COVERALLS_MODULE_PATH} \ + -DCMAKE_CTEST_COMMAND="$([[ "$COVERALLS" == "ON" ]] && echo '/home/tango/src/.travis/run_ctest.sh' || echo 'ctest')" if [[ "$COVERALLS" == "ON" ]] then diff --git a/.travis/run_ctest.sh b/.travis/run_ctest.sh new file mode 100755 index 000000000..87f36c681 --- /dev/null +++ b/.travis/run_ctest.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +if ! ctest \ + --output-on-failure \ + --parallel 8 +then + if ! ctest \ + --output-on-failure \ + --rerun-failed \ + --repeat-until-fail 2 + then + exit 1 + fi +fi diff --git a/.travis/test.sh b/.travis/test.sh index 41db77546..998b16f2c 100755 --- a/.travis/test.sh +++ b/.travis/test.sh @@ -6,16 +6,12 @@ then exit 0 fi -TEST_COMMAND="exec ctest --output-on-failure" -if [ $COVERALLS = "ON" ] -then - TEST_COMMAND="exec make coveralls" -fi +build_dir="/home/tango/src/build" -echo "Test" -echo "TEST_COMMAND=$TEST_COMMAND" -docker exec cpp_tango /bin/sh -c "cd /home/tango/src/build; $TEST_COMMAND" -if [ $? -ne "0" ] -then - exit -1 +set -e + +if [[ "$COVERALLS" == "ON" ]]; then + docker exec -w "${build_dir}" cpp_tango make coveralls +else + docker exec -w "${build_dir}" cpp_tango ../.travis/run_ctest.sh fi From 7ab4262120cf3acfd658deb62e425510c6945cb3 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sun, 16 Jun 2019 16:44:47 +0200 Subject: [PATCH 06/12] Update instructions how to run the tests --- INSTALL.md | 151 +++++++++++++++++++---------------------------------- 1 file changed, 54 insertions(+), 97 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index ac664421c..60b516c82 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -112,128 +112,85 @@ $ cmake .. $ make ``` -### Start TANGO environment - -``` -$ make start-tango -Setup test environment -557e3c8a3daa2b75aac4fe04562bac32570db0ace08edd06a23cebaa7fd86f5e -CONTAINER=27bad3659305155c33d99505c4836b616d9c2a6de3431229e79b71a020f18455 -TANGO_HOST=172.17.0.3:10000 -Create tango_host file -Wait till tango-cs is online -``` - -This process takes ~30 s - ### Run tests -``` -$ make run-tests -TANGO_HOST=172.17.0.3:10000 -Run conf_devtest -Added test server : DevTest/test -> test/debian8/10, class : DevTest -Added test server : DevTest/test -> test/debian8/11, class : DevTest -Added test server : DevTest/test -> test/debian8/12, class : DevTest - -Added test server : FwdTest/test -> test/fwd_debian8/10, class : FwdTest +> Note: The test runner automatically starts database and all required +> device servers for each test. If you want to set up the environment +> manually, run ctest with following environment variable: +> +> `TANGO_TEST_CASE_SKIP_FIXTURE=1 ctest -V -R ds_cache` -Added pseudo server : DsCache/test -> test/cache1/1, class : CacheTest1 -Added pseudo server : DsCache/test -> test/cache1/2, class : CacheTest1 -Added pseudo server : DsCache/test -> test/cache2/1, class : CacheTest2 +From `build/` directory run: - - -Constructing a list of tests -Done constructing a list of tests -Checking test dependency graph... -Checking test dependency graph end -test 1 - Start 1: log4tango_test - - - -100% tests passed, 0 tests failed out of 59 - -Total Test time (real) = 843.30 sec - -Run command: /home/tango/src/build/cpp_test_suite/environment/post_test.sh ``` +$ ctest --output-on-failure --parallel 4 -The whole test suite takes ~ 15 min +Test project /home/tango-cs/Documents/cppTango/build + Start 54: old_tests::ring_depth + Start 66: asyn::asyn_cmd + Start 77: event::per_event + Start 83: event::user_event + 1/93 Test #83: event::user_event ................. Passed 22.65 sec -### Setup and run individual tests +... -To run individual tests use TANGO_HOST provided by the `start-tango` target. +90/93 Test #1: log4tango_test .................... Passed 0.20 sec +91/93 Test #39: CXX::cxx_nan_inf_in_prop .......... Passed 13.28 sec +92/93 Test #43: old_tests::attr_types ............. Passed 12.89 sec +93/93 Test #14: CXX::cxx_attr_conf ................ Passed 12.81 sec -Make sure TANGO_HOST is set correctly: +100% tests passed, 0 tests failed out of 93 +Total Test time (real) = 546.93 sec ``` -$ cd build -$ cat tango_host -#!/bin/bash -export TANGO_HOST=172.17.0.3:10000 -``` - -TANGO_HOST must be the same as what `start-tango` has returned. -``` -$ . tango_host -``` +Test output and device server logs are collected in `build/cpp_test_suite/test_results`. -Now run some test: +### Run individual tests ``` -$ ctest -R attr_misc -V -UpdateCTestConfiguration from :/storage/Projects/org.tango/git/cppTango/build/DartConfiguration.tcl -Parse Config file:/storage/Projects/org.tango/git/cppTango/build/DartConfiguration.tcl - Add coverage exclude regular expressions. -UpdateCTestConfiguration from :/storage/Projects/org.tango/git/cppTango/build/DartConfiguration.tcl -Parse Config file:/storage/Projects/org.tango/git/cppTango/build/DartConfiguration.tcl -Test project /storage/Projects/org.tango/git/cppTango/build -Run command: /storage/Projects/org.tango/git/cppTango/build/cpp_test_suite/environment/pre_test.sh -TANGO_HOST=172.17.0.3:10000 -Run conf_devtest - +$ ctest -R old_tests::attr_misc -V + +UpdateCTestConfiguration from :/home/tango-cs/Documents/cppTango/build/DartConfiguration.tcl +Parse Config file:/home/tango-cs/Documents/cppTango/build/DartConfiguration.tcl +UpdateCTestConfiguration from :/home/tango-cs/Documents/cppTango/build/DartConfiguration.tcl +Parse Config file:/home/tango-cs/Documents/cppTango/build/DartConfiguration.tcl +Test project /home/tango-cs/Documents/cppTango/build Constructing a list of tests Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements Checking test dependency graph... Checking test dependency graph end -test 12 - Start 12: old_tests::attr_misc - -12: Test command: /storage/Projects/org.tango/git/cppTango/build/cpp_test_suite/old_tests/attr_misc "test/debian8/10" -12: Test timeout computed to be: 1500 -12: -12: new DeviceProxy(test/debian8/10) returned -12: -12: Setting/Getting attribute info --> OK -12: Writing outside attribute limits --> OK -12: Min alarm detection (on a float spectrum) --> OK -12: Reset min alarm detection --> OK -12: Max alarm detection (on a float spectrum) --> OK -12: Reset max alarm detection --> OK -12: Min alarm detection (on a unsigned short spectrum) --> OK -12: Reset min alarm detection --> OK -12: Max alarm detection (on a unsigned short spectrum) --> OK -12: Reset max alarm detection --> OK -12: Setting/Getting V5 attribute info --> OK -12: Alarm, Warning level detection --> OK -12: Exception when trying to change "hard coded" properties --> OK -1/1 Test #12: old_tests::attr_misc ............. Passed 0.29 sec +test 49 + Start 49: old_tests::attr_misc + +49: Test command: /home/tango-cs/Documents/cppTango/build/cpp_test_suite/environment/run_with_fixture.sh "/home/tango-cs/Documents/cppTango/build/cpp_test_suite/old_tests/attr_misc" "test/debian8/10" +49: Test timeout computed to be: 1500 +49: +49: new DeviceProxy(test/debian8/10) returned +49: +49: Setting/Getting attribute info --> OK +49: Writing outside attribute limits --> OK +49: Min alarm detection (on a float spectrum) --> OK +49: Reset min alarm detection --> OK +49: Max alarm detection (on a float spectrum) --> OK +49: Reset max alarm detection --> OK +49: Min alarm detection (on a unsigned short spectrum) --> OK +49: Reset min alarm detection --> OK +49: Max alarm detection (on a unsigned short spectrum) --> OK +49: Reset max alarm detection --> OK +49: Setting/Getting V5 attribute info --> OK +49: Alarm, Warning level detection --> OK +49: Exception when trying to change "hard coded" properties --> OK +1/1 Test #49: old_tests::attr_misc ............. Passed 8.55 sec The following tests passed: old_tests::attr_misc 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.33 sec +Total Test time (real) = 8.56 sec ``` See [CTest guide](https://cmake.org/Wiki/CMake/Testing_With_CTest) - -### Stop TANGO environment - -``` -$ make stop-tango -``` From 7671bb1cea94e489fbe0e0003b090bccf11bbfce Mon Sep 17 00:00:00 2001 From: mliszcz Date: Sun, 15 Dec 2019 22:27:23 +0100 Subject: [PATCH 07/12] Extract test fixture setup to a separate script Extract setup scripts so that it is possible to run the setup manually and then attach the debugger or perform some additional configuration. --- INSTALL.md | 23 ++++-- cpp_test_suite/environment/CMakeLists.txt | 18 +++-- .../environment/run_with_fixture.sh.cmake | 79 +++---------------- .../environment/setup_database.sh.cmake | 45 +++++++++++ .../environment/setup_devices.sh.cmake | 21 +++++ 5 files changed, 103 insertions(+), 83 deletions(-) create mode 100755 cpp_test_suite/environment/setup_database.sh.cmake create mode 100755 cpp_test_suite/environment/setup_devices.sh.cmake diff --git a/INSTALL.md b/INSTALL.md index 60b516c82..38b530b21 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -114,12 +114,6 @@ $ make ### Run tests -> Note: The test runner automatically starts database and all required -> device servers for each test. If you want to set up the environment -> manually, run ctest with following environment variable: -> -> `TANGO_TEST_CASE_SKIP_FIXTURE=1 ctest -V -R ds_cache` - From `build/` directory run: ``` @@ -146,6 +140,8 @@ Total Test time (real) = 546.93 sec Test output and device server logs are collected in `build/cpp_test_suite/test_results`. +For more details on testing with CTest, [see the guide](https://cmake.org/Wiki/CMake/Testing_With_CTest). + ### Run individual tests ``` @@ -193,4 +189,17 @@ The following tests passed: Total Test time (real) = 8.56 sec ``` -See [CTest guide](https://cmake.org/Wiki/CMake/Testing_With_CTest) +### Setting environment up manually + +The test runner automatically starts database and all required +device servers for each test. If you want to set up the environment +manually, from `build/` directory run: + +```bash +source ./cpp_test_suite/environment/setup_database.sh # source to get TANGO_HOST +./cpp_test_suite/environment/setup_devices.sh +# attach the debugger or perform some additional configuration +TANGO_TEST_CASE_SKIP_FIXTURE=1 ctest -V -R ds_cache +killall DevTest FwdTest +docker stop tango_cs mysql_db +``` diff --git a/cpp_test_suite/environment/CMakeLists.txt b/cpp_test_suite/environment/CMakeLists.txt index e4947b71f..fa64b382e 100644 --- a/cpp_test_suite/environment/CMakeLists.txt +++ b/cpp_test_suite/environment/CMakeLists.txt @@ -1,10 +1,14 @@ #TODO windows -configure_file(run_with_fixture.sh.cmake run_with_fixture.sh @ONLY) -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/run_with_fixture.sh) +SET(SCRIPTS + run_with_fixture.sh + setup_database.sh + setup_devices.sh + start_server.sh + kill_server.sh +) -configure_file(start_server.sh.cmake start_server.sh @ONLY) -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/start_server.sh) - -configure_file(kill_server.sh.cmake kill_server.sh @ONLY) -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/kill_server.sh) +foreach(SCRIPT ${SCRIPTS}) + configure_file(${SCRIPT}.cmake ${SCRIPT} @ONLY) + execute_process(COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${SCRIPT}) +endforeach(SCRIPT) diff --git a/cpp_test_suite/environment/run_with_fixture.sh.cmake b/cpp_test_suite/environment/run_with_fixture.sh.cmake index 21fe3f61c..1d52699c0 100755 --- a/cpp_test_suite/environment/run_with_fixture.sh.cmake +++ b/cpp_test_suite/environment/run_with_fixture.sh.cmake @@ -12,8 +12,8 @@ tc_program="$1" tc_run_name="$(basename "$tc_program")_$(date '+%Y%m%d.%H%M%S.%N')" shift 1 -tc_tango_container="tango_cs_${tc_run_name}" tc_mysql_container="mysql_db_${tc_run_name}" +tc_tango_container="tango_cs_${tc_run_name}" export TANGO_TEST_CASE_DIRECTORY="@PROJECT_BINARY_DIR@/test_results/${tc_run_name}" mkdir -p "$TANGO_TEST_CASE_DIRECTORY" @@ -28,85 +28,26 @@ function teardown { ) & } - if [[ -z "$TANGO_TEST_CASE_SKIP_FIXTURE" ]]; then - trap teardown EXIT - -# --- setup database ---------------------------------------------------------- - - docker run \ - --name "$tc_mysql_container" \ - --rm \ - -e MYSQL_ROOT_PASSWORD=root \ - -e MYSQL_INITDB_SKIP_TZINFO=1 \ - -d \ - tangocs/mysql:9.2.2 \ - --sql-mode="" \ - --innodb=OFF \ - --default-storage-engine=MyISAM \ - > /dev/null - - docker run \ - --name "$tc_tango_container" \ - --rm \ - -e TANGO_HOST=127.0.0.1:10000 \ - -e MYSQL_HOST=mysql_db:3306 \ - -e MYSQL_USER=tango \ - -e MYSQL_PASSWORD=tango \ - -e MYSQL_DATABASE=tango \ - --link "$tc_mysql_container":mysql_db \ - -d \ - tangocs/tango-cs:latest \ - > /dev/null - - tc_tango_ipaddr="$(docker inspect \ - -f '{{ .NetworkSettings.IPAddress }}' \ - "$tc_tango_container")" - - export TANGO_HOST="$tc_tango_ipaddr:10000" - - if hash tango_admin 2>/dev/null; then - if ! tango_admin --ping-database 30; then - echo "Failed to ping database" - exit 1 - fi - else - sleep 30 - fi - -# -- setup devices ------------------------------------------------------------ - - "@PROJECT_BINARY_DIR@/new_tests/conf_devtest" \ - @SERV_NAME@/@INST_NAME@ \ - @DEV1@ \ - @DEV2@ \ - @DEV3@ \ - @DEV1_ALIAS@ \ - @ATTR_ALIAS@ \ - @FWD_DEV@ \ - @DEV20@ \ - &> "${TANGO_TEST_CASE_DIRECTORY}/conf_devtest.out" - - "@CMAKE_CURRENT_BINARY_DIR@/start_server.sh" @INST_NAME@ DevTest cpp_test_ds - - "@CMAKE_CURRENT_BINARY_DIR@/start_server.sh" @INST_NAME@ FwdTest cpp_test_ds/fwd_ds - -fi # TANGO_TEST_CASE_SKIP_FIXTURE - - -# --- run test ---------------------------------------------------------------- + eval $( + set -e + source "@CMAKE_CURRENT_BINARY_DIR@/setup_database.sh" \ + "$tc_mysql_container" \ + "$tc_tango_container" + echo "export TANGO_HOST=$TANGO_HOST" + ) + "@CMAKE_CURRENT_BINARY_DIR@/setup_devices.sh" +fi set +e set -o pipefail "$tc_program" "$@" 2>&1 | tee "${TANGO_TEST_CASE_DIRECTORY}/testcase.out" - tc_exit_code="$?" set +o pipefail set -e echo "$tc_exit_code" > "${TANGO_TEST_CASE_DIRECTORY}/testcase_exit_code.out" - exit "$tc_exit_code" diff --git a/cpp_test_suite/environment/setup_database.sh.cmake b/cpp_test_suite/environment/setup_database.sh.cmake new file mode 100755 index 000000000..fdbe14468 --- /dev/null +++ b/cpp_test_suite/environment/setup_database.sh.cmake @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# vim: syntax=sh + +mysql_container="${1:-mysql_db}" +tango_container="${2:-tango_cs}" + +docker run \ + --name "$mysql_container" \ + --rm \ + -e MYSQL_ROOT_PASSWORD=root \ + -e MYSQL_INITDB_SKIP_TZINFO=1 \ + -d \ + tangocs/mysql:9.2.2 \ + --sql-mode="" \ + --innodb=OFF \ + --default-storage-engine=MyISAM \ + > /dev/null + +docker run \ + --name "$tango_container" \ + --rm \ + -e TANGO_HOST=127.0.0.1:10000 \ + -e MYSQL_HOST=mysql_db:3306 \ + -e MYSQL_USER=tango \ + -e MYSQL_PASSWORD=tango \ + -e MYSQL_DATABASE=tango \ + --link "$mysql_container":mysql_db \ + -d \ + tangocs/tango-cs:latest \ + > /dev/null + +tango_ipaddr="$(docker inspect \ + -f '{{ .NetworkSettings.IPAddress }}' \ + "$tango_container")" + +export TANGO_HOST="${tango_ipaddr}:10000" + +if hash tango_admin 2>/dev/null; then + if ! tango_admin --ping-database 30; then + echo "Failed to ping database" + exit 1 + fi +else + sleep 30 +fi diff --git a/cpp_test_suite/environment/setup_devices.sh.cmake b/cpp_test_suite/environment/setup_devices.sh.cmake new file mode 100755 index 000000000..24ba39813 --- /dev/null +++ b/cpp_test_suite/environment/setup_devices.sh.cmake @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# vim: syntax=sh + +set -e + +export TANGO_TEST_CASE_DIRECTORY="${TANGO_TEST_CASE_DIRECTORY:-.}" + +"@PROJECT_BINARY_DIR@/new_tests/conf_devtest" \ + @SERV_NAME@/@INST_NAME@ \ + @DEV1@ \ + @DEV2@ \ + @DEV3@ \ + @DEV1_ALIAS@ \ + @ATTR_ALIAS@ \ + @FWD_DEV@ \ + @DEV20@ \ + &> "${TANGO_TEST_CASE_DIRECTORY}/conf_devtest.out" + +"@CMAKE_CURRENT_BINARY_DIR@/start_server.sh" @INST_NAME@ DevTest cpp_test_ds + +"@CMAKE_CURRENT_BINARY_DIR@/start_server.sh" @INST_NAME@ FwdTest cpp_test_ds/fwd_ds From ee64cece86b8142ea6a9392e39cf8edbe87b121c Mon Sep 17 00:00:00 2001 From: mliszcz Date: Thu, 19 Dec 2019 09:31:30 +0100 Subject: [PATCH 08/12] Set number of parallel tests basing on CPU count Run up to (num of CPUs) tests in parallel to avoid overloading the machine and to reduce the number of failures of unstable tests. --- .travis/run_ctest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/run_ctest.sh b/.travis/run_ctest.sh index 87f36c681..bed25000f 100755 --- a/.travis/run_ctest.sh +++ b/.travis/run_ctest.sh @@ -2,7 +2,7 @@ if ! ctest \ --output-on-failure \ - --parallel 8 + --parallel $(nproc) then if ! ctest \ --output-on-failure \ From 6dc48da7c57e3ea63089e533610612d53ec28265 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Mon, 20 Jan 2020 09:19:49 +0100 Subject: [PATCH 09/12] Do not re-run the tests on a failure in CI Since we are running up to (num of CPUs) tests in parallel, no failures are expected. --- .travis/run.sh | 3 +-- .travis/run_ctest.sh | 14 -------------- .travis/test.sh | 12 ++++++++++-- 3 files changed, 11 insertions(+), 18 deletions(-) delete mode 100755 .travis/run_ctest.sh diff --git a/.travis/run.sh b/.travis/run.sh index fc04459ea..a44c596a2 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -31,8 +31,7 @@ docker exec cpp_tango cmake \ -DTANGO_USE_USING_NAMESPACE=${TANGO_USE_USING_NAMESPACE} \ -DWARNINGS_AS_ERRORS=${WARNINGS_AS_ERRORS} \ -DCOVERALLS=${COVERALLS} \ - -DCOVERALLS_MODULE_PATH=${COVERALLS_MODULE_PATH} \ - -DCMAKE_CTEST_COMMAND="$([[ "$COVERALLS" == "ON" ]] && echo '/home/tango/src/.travis/run_ctest.sh' || echo 'ctest')" + -DCOVERALLS_MODULE_PATH=${COVERALLS_MODULE_PATH} if [[ "$COVERALLS" == "ON" ]] then diff --git a/.travis/run_ctest.sh b/.travis/run_ctest.sh deleted file mode 100755 index bed25000f..000000000 --- a/.travis/run_ctest.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -if ! ctest \ - --output-on-failure \ - --parallel $(nproc) -then - if ! ctest \ - --output-on-failure \ - --rerun-failed \ - --repeat-until-fail 2 - then - exit 1 - fi -fi diff --git a/.travis/test.sh b/.travis/test.sh index 998b16f2c..91276b2bc 100755 --- a/.travis/test.sh +++ b/.travis/test.sh @@ -8,10 +8,18 @@ fi build_dir="/home/tango/src/build" +function run_in_container { + docker exec \ + -w "${build_dir}" \ + -e CTEST_PARALLEL_LEVEL=$(nproc) \ + -e CTEST_OUTPUT_ON_FAILURE=ON \ + cpp_tango "$@" +} + set -e if [[ "$COVERALLS" == "ON" ]]; then - docker exec -w "${build_dir}" cpp_tango make coveralls + run_in_container make coveralls else - docker exec -w "${build_dir}" cpp_tango ../.travis/run_ctest.sh + run_in_container ctest fi From 0c13be4d06d11483ba736c058e9753fe8d35ee69 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Mon, 20 Jan 2020 09:43:28 +0100 Subject: [PATCH 10/12] Install Docker in Ubuntu 20.04 container in CI --- .travis/ubuntu-20.04/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis/ubuntu-20.04/Dockerfile b/.travis/ubuntu-20.04/Dockerfile index 157a9c614..791659a4f 100644 --- a/.travis/ubuntu-20.04/Dockerfile +++ b/.travis/ubuntu-20.04/Dockerfile @@ -18,12 +18,15 @@ RUN apt-get update && apt-get install -y \ libzmq3-dev \ lsb-release \ omniidl \ - python2 + python2 \ + docker.io RUN groupadd -g "$APP_GID" tango RUN useradd -u "$APP_UID" -g "$APP_GID" -ms /bin/bash tango +RUN usermod -a -G docker tango + ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig USER tango From 6329017a68f91e995b98ca0e038ba5c1bb6fe587 Mon Sep 17 00:00:00 2001 From: mliszcz Date: Wed, 22 Jan 2020 09:45:05 +0100 Subject: [PATCH 11/12] Add cpp_tango user to group owning docker socket The owning group of docker socket varies between different docker packages and different operating systems. Official docker-ce package uses gid 957 while docker.io package on Ubuntu 20.04 uses gid 103. New group with matching gid matching docker host is added to the cpp_tango container to allow tango user access the docker socket. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 28e44abd3..1ee5d3fbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,10 @@ before_script: -v /var/run/docker.sock:/var/run/docker.sock -dit cpp_tango + - > + docker exec -u root cpp_tango + groupadd -g $(grep docker /etc/group | awk -F ':' '{print $3}') dockerhost || true + - docker exec -u root cpp_tango usermod -a -G dockerhost tango || true - .travis/install_tango_idl.sh - (test ${STOCK_CPPZMQ} = "OFF" && .travis/install_cppzmq.sh) || true #work around gcov ignored by sonar From 890fd5947dc288e1746baca6cbaf4ddfa8d06b0f Mon Sep 17 00:00:00 2001 From: mliszcz Date: Wed, 22 Jan 2020 13:42:29 +0100 Subject: [PATCH 12/12] Enable Sonar and Coveralls integration in CI build Sonar and Coveralls were disabled in 226d70c and this was unnoticed until now. We again enable those integrations. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1ee5d3fbd..1205e8593 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ env: - OS_TYPE=debian10 USE_PCH=OFF RUN_TESTS=OFF - OS_TYPE=debian10 CMAKE_BUILD_TYPE=Release - OS_TYPE=debian9 - - OS_TYPE=debian8 STOCK_CPPZMQ=OFF + - OS_TYPE=debian8 SONAR_SCANNER=ON COVERALLS=ON STOCK_CPPZMQ=OFF - OS_TYPE=debian7 notifications: