From 1ae4579be94742e5ed309b31c73589e0ba462464 Mon Sep 17 00:00:00 2001 From: Guilherme Amadio Date: Thu, 15 Jun 2023 11:27:46 +0200 Subject: [PATCH 1/5] [Tests] CppUnit tests fail if run in parallel This is because some of them may find a port unavailable as port numbers are fixed in the tests to 9999. --- tests/XrdClTests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/XrdClTests/CMakeLists.txt b/tests/XrdClTests/CMakeLists.txt index d8094c7cbcf..238e2509899 100644 --- a/tests/XrdClTests/CMakeLists.txt +++ b/tests/XrdClTests/CMakeLists.txt @@ -60,6 +60,7 @@ foreach(TEST_SUITE ) add_test(NAME XrdCl::${TEST_SUITE} COMMAND $ $ "All Tests/${TEST_SUITE}Test") + set_tests_properties(XrdCl::${TEST_SUITE} PROPERTIES RUN_SERIAL TRUE) endforeach() #------------------------------------------------------------------------------- From 552bdb0df22138b3e90f5d0ca39737b0c388bd1b Mon Sep 17 00:00:00 2001 From: Guilherme Amadio Date: Thu, 15 Jun 2023 14:58:53 +0200 Subject: [PATCH 2/5] [XrdEcTests] Use different directory name for each test This is necessary to avoid test failures when running tests in parallel, as they'd be trying to overwrite each other's data in that case, and also failing tests do not clean up after themselves, so the following tests all fail if a given test fails. --- tests/XrdEcTests/MicroTest.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/XrdEcTests/MicroTest.cc b/tests/XrdEcTests/MicroTest.cc index 7668e4c6a66..d02743351b9 100644 --- a/tests/XrdEcTests/MicroTest.cc +++ b/tests/XrdEcTests/MicroTest.cc @@ -280,13 +280,10 @@ void MicroTest::Init( bool usecrc32c ) objcfg.reset( new ObjCfg( "test.txt", nbdata, nbparity, chsize, usecrc32c, true ) ); rawdata.clear(); - char cwdbuff[1024]; - char *cwdptr = getcwd( cwdbuff, sizeof( cwdbuff ) ); - CPPUNIT_ASSERT( cwdptr ); - std::string cwd = cwdptr; + char tmpdir[32] = "/tmp/xrootd-xrdec-XXXXXX"; // create the data directory - datadir = cwd + "/data"; - CPPUNIT_ASSERT( mkdir( datadir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ) == 0 ); + CPPUNIT_ASSERT( mkdtemp(tmpdir) ); + datadir = tmpdir; // create a directory for each stripe size_t nbstrps = objcfg->nbdata + 2 * objcfg->nbparity; for( size_t i = 0; i < nbstrps; ++i ) From 78fc7eddf1bcf353d84f5a7738d7e23d74d997cc Mon Sep 17 00:00:00 2001 From: Guilherme Amadio Date: Thu, 15 Jun 2023 14:20:25 +0200 Subject: [PATCH 3/5] [Tests] Add XrdEc tests to main test target --- tests/XrdEcTests/CMakeLists.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/XrdEcTests/CMakeLists.txt b/tests/XrdEcTests/CMakeLists.txt index eaa8317ddda..753c30bc730 100644 --- a/tests/XrdEcTests/CMakeLists.txt +++ b/tests/XrdEcTests/CMakeLists.txt @@ -12,6 +12,25 @@ target_link_libraries( target_link_libraries(XrdEcTests PRIVATE ${ISAL_LIBRARIES}) target_include_directories(XrdEcTests PRIVATE ../common ${CPPUNIT_UNCLUDE_DIRS} ${ISAL_INCLUDE_DIRS}) +foreach(TEST + AlignedWriteTest + SmallWriteTest + BigWriteTest + VectorReadTest + IllegalVectorReadTest + AlignedWrite1MissingTest + AlignedWrite2MissingTest + AlignedWriteTestIsalCrcNoMt + SmallWriteTestIsalCrcNoMt + BigWriteTestIsalCrcNoMt + AlignedWrite1MissingTestIsalCrcNoMt + AlignedWrite2MissingTestIsalCrcNoMt) + add_test(NAME XrdEc::${TEST} + COMMAND $ $ + "All Tests/MicroTest/MicroTest::${TEST}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +endforeach() + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- From d0f93f8ef2657ed65c9101606935fd9bcdfd0ab8 Mon Sep 17 00:00:00 2001 From: Guilherme Amadio Date: Fri, 16 Jun 2023 16:10:23 +0200 Subject: [PATCH 4/5] [CMake] Add -fprofile-update=atomic to coverage flags This is to avoid corruption when running multi-threaded tests. See also https://gcc.gnu.org/bugzilla/show_bug.cgi?id=68080. --- test.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.cmake b/test.cmake index ca12a1f90a2..b35d4ed8694 100644 --- a/test.cmake +++ b/test.cmake @@ -106,7 +106,7 @@ set(CMAKE_ARGS $ENV{CMAKE_ARGS} ${CMAKE_ARGS}) if(COVERAGE) find_program(CTEST_COVERAGE_COMMAND NAMES gcov) - list(PREPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS=--coverage") + list(PREPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS=--coverage -fprofile-update=atomic") endif() if(MEMCHECK) From 9abc5c20224096a8733b126e38c876ee7807bc35 Mon Sep 17 00:00:00 2001 From: Guilherme Amadio Date: Fri, 16 Jun 2023 17:39:08 +0200 Subject: [PATCH 5/5] [Tests] Add a simple set of smoke tests for server+client --- tests/CMakeLists.txt | 2 + tests/XRootD/CMakeLists.txt | 33 ++++++++++++ tests/XRootD/smoke.sh | 103 ++++++++++++++++++++++++++++++++++++ tests/XRootD/xrootd.cfg | 8 +++ 4 files changed, 146 insertions(+) create mode 100644 tests/XRootD/CMakeLists.txt create mode 100755 tests/XRootD/smoke.sh create mode 100644 tests/XRootD/xrootd.cfg diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d26a902851f..880c0f1a964 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,3 +13,5 @@ endif() if( BUILD_CEPH ) add_subdirectory( XrdCephTests ) endif() + +add_subdirectory(XRootD) diff --git a/tests/XRootD/CMakeLists.txt b/tests/XRootD/CMakeLists.txt new file mode 100644 index 00000000000..f25d24211d1 --- /dev/null +++ b/tests/XRootD/CMakeLists.txt @@ -0,0 +1,33 @@ +if(XRDCL_ONLY) + return() +endif() + +execute_process(COMMAND id -u OUTPUT_VARIABLE UID OUTPUT_STRIP_TRAILING_WHITESPACE) + +if (UID EQUAL 0) + return() +endif() + +set(XRD_TEST_PORT "10940" CACHE STRING "Port for XRootD Test Server") + +list(APPEND XRDENV "XRDCP=$") +list(APPEND XRDENV "XRDFS=$") +list(APPEND XRDENV "CRC32C=$") +list(APPEND XRDENV "ADLER32=$") +list(APPEND XRDENV "HOST=root://localhost:${XRD_TEST_PORT}") + +configure_file(xrootd.cfg xrootd.cfg @ONLY) + +add_test(NAME XRootD::start + COMMAND sh -c "mkdir -p data && \ + $ -b -k fifo -l xrootd.log -s xrootd.pid -c xrootd.cfg") +set_tests_properties(XRootD::start PROPERTIES FIXTURES_SETUP XRootD) + +add_test(NAME XRootD::stop COMMAND sh -c "sleep 1 && rm -rf data && kill $(< xrootd.pid)") +set_tests_properties(XRootD::stop PROPERTIES FIXTURES_CLEANUP XRootD) + +add_test(NAME XRootD::smoke-test + COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh") + +set_tests_properties(XRootD::smoke-test PROPERTIES + ENVIRONMENT "${XRDENV}" FIXTURES_REQUIRED XRootD) diff --git a/tests/XRootD/smoke.sh b/tests/XRootD/smoke.sh new file mode 100755 index 00000000000..ed2f3bfba33 --- /dev/null +++ b/tests/XRootD/smoke.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +: ${ADLER32:=$(command -v xrdadler32)} +: ${CRC32C:=$(command -v xrdcrc32c)} +: ${XRDCP:=$(command -v xrdcp)} +: ${XRDFS:=$(command -v xrdfs)} +: ${OPENSSL:=$(command -v openssl)} +: ${HOST:=root://localhost:${PORT:-1094}} + +for PROG in ${ADLER32} ${CRC32C} ${XRDCP} ${XRDFS} ${OPENSSL}; do + if [[ ! -x "${PROG}" ]]; then + echo 1>&2 "$(basename $0): error: '${PROG}': command not found" + exit 1 + fi +done + +# This script assumes that ${HOST} exports an empty / as read/write. +# It also assumes that any authentication required is already setup. + +set -e + +${XRDCP} --version +${XRDFS} ${HOST} query config version + +# query some common server configurations + +CONFIG_PARAMS=( version role sitename ) + +for PARAM in ${CONFIG_PARAMS[@]}; do + ${XRDFS} ${HOST} query config ${PARAM} +done + +# some extra query commands that don't make any changes + +${XRDFS} ${HOST} stat / +${XRDFS} ${HOST} statvfs / +${XRDFS} ${HOST} spaceinfo / + +# create local temporary directory +TMPDIR=$(mktemp -d /tmp/xrdfs-test-XXXXXX) + +# cleanup after ourselves if something fails +trap "rm -rf ${TMPDIR}" EXIT + +# create remote temporary directory +# this will get cleaned up by CMake upon fixture tear down +${XRDFS} ${HOST} mkdir -p ${TMPDIR} + +# create local files with random contents using OpenSSL + +FILES=$(seq -w 1 ${NFILES:-10}) + +for i in $FILES; do + ${OPENSSL} rand -out "${TMPDIR}/${i}.ref" $((1024 * $RANDOM)) +done + +# upload local files to the server in parallel + +for i in $FILES; do + ${XRDCP} ${TMPDIR}/${i}.ref ${HOST}/${TMPDIR}/${i}.ref +done + +# list uploaded files, then download them to check for corruption + +${XRDFS} ${HOST} ls -l ${TMPDIR} + +for i in $FILES; do + ${XRDCP} ${HOST}/${TMPDIR}/${i}.ref ${TMPDIR}/${i}.dat +done + +# check that all checksums for downloaded files match + +for i in $FILES; do + REF32C=$(${CRC32C} < ${TMPDIR}/${i}.ref | cut -d' ' -f1) + NEW32C=$(${CRC32C} < ${TMPDIR}/${i}.dat | cut -d' ' -f1) + SRV32C=$(${XRDFS} ${HOST} query checksum ${TMPDIR}/${i}.ref?cks.type=crc32c | cut -d' ' -f2) + + REFA32=$(${ADLER32} < ${TMPDIR}/${i}.ref | cut -d' ' -f1) + NEWA32=$(${ADLER32} < ${TMPDIR}/${i}.dat | cut -d' ' -f1) + SRVA32=$(${XRDFS} ${HOST} query checksum ${TMPDIR}/${i}.ref?cks.type=adler32 | cut -d' ' -f2) + echo "${i}: crc32c: reference: ${REF32C}, server: ${SRV32C}, downloaded: ${REFA32}" + echo "${i}: adler32: reference: ${NEW32C}, server: ${SRVA32}, downloaded: ${NEWA32}" + + if [[ "${NEWA32}" != "${REFA32}" || "${SRVA32}" != "${REFA32}" ]]; then + echo 1>&2 "$(basename $0): error: adler32 checksum check failed for file: ${i}.dat" + exit 1 + fi + if [[ "${NEW32C}" != "${REF32C}" || "${SRV32C}" != "${REF32C}" ]]; then + echo 1>&2 "$(basename $0): error: crc32 checksum check failed for file: ${i}.dat" + exit 1 + fi +done + +for i in $FILES; do + ${XRDFS} ${HOST} rm ${TMPDIR}/${i}.ref & +done + +wait + +${XRDFS} ${HOST} rmdir ${TMPDIR} + +echo "ALL TESTS PASSED" +exit 0 diff --git a/tests/XRootD/xrootd.cfg b/tests/XRootD/xrootd.cfg new file mode 100644 index 00000000000..922179d5219 --- /dev/null +++ b/tests/XRootD/xrootd.cfg @@ -0,0 +1,8 @@ +# This minimal configuration file starts a standalone server +# that exports the data directory as / without authentication. + +all.export / +all.sitename XRootD +oss.localroot @CMAKE_CURRENT_BINARY_DIR@/data +xrd.port @XRD_TEST_PORT@ +xrootd.chksum chkcgi adler32 crc32c