diff --git a/doc/c-test-api.txt b/doc/c-test-api.txt index 9f104ecd741..711b445d9fe 100644 --- a/doc/c-test-api.txt +++ b/doc/c-test-api.txt @@ -93,14 +93,35 @@ in range of [0, '.tcnt' - 1]. IMPORTANT: Only one of '.test' and '.test_all' can be set at a time. -Each test has a default timeout set to 300s. The default timeout can be -overridden by setting '.timeout' in the test structure or by calling -'tst_set_timeout()' in the test 'setup()'. There are a few testcases whose run -time may vary arbitrarily, for these timeout can be disabled by setting it to --1. +Each test has a limit on how long it can run and the limit composes of two +parts max_runtime and timeout. The max_runtime is a limit for how long can the +'.test_all' or a set of '.test' functions take and the timeout is static part +that should cover the duration of test setup and cleanup plus some safety. -Test can find out how much time (in seconds) is remaining to timeout, -by calling 'tst_timeout_remaining()'. +Any test that runs for more than a second or two has to make sure to: + +- set the runtime either by setting the '.max_runtime' in tst_test or by + calling 'tst_set_runtime()' in the test setup + +- monitor remaning runtime by regular calls to 'tst_remaining_runtime()' and + exit when runtime has been used up + +Test is free to exit before max_runtime has been used up for example when +minimal number of iteration was finished. + +The limit is applied to a single call of the '.test_all' function that means +that for example when '.test_variants' or '.all_filesystems' is set the whole +test will be limited by 'variants * (max_runtime + timeout)' seconds and the +test runtime will be likely close to 'variants * max_runtime' seconds. + +[source,c] +------------------------------------------------------------------------------- +/* + * Returns number of seconds or zero in case that runtime has been used up. + */ + +int tst_remaining_runtime(void); +------------------------------------------------------------------------------- LAPI headers ++++++++++++ @@ -377,12 +398,13 @@ WARNING: This function is not thread safe. [source,c] ------------------------------------------------------------------------------- -void tst_set_timeout(unsigned int timeout); +void tst_set_max_runtime(int max_runtime); ------------------------------------------------------------------------------- -Allows for setting timeout per test iteration dynamically in the test setup(), +Allows for setting max_runtime per test iteration dynamically in the test setup(), the timeout is specified in seconds. There are a few testcases whose runtime -can vary arbitrarily, these can disable timeouts by setting it to -1. +can vary arbitrarily, these can disable timeouts by setting it to +TST_UNLIMITED_RUNTIME. [source,c] ------------------------------------------------------------------------------- diff --git a/doc/user-guide.txt b/doc/user-guide.txt index f41cbc73379..96c9800ed4e 100644 --- a/doc/user-guide.txt +++ b/doc/user-guide.txt @@ -22,9 +22,13 @@ For running LTP network tests see `testcases/network/README.md`. | 'LTP_SINGLE_FS_TYPE' | Testing only - specifies filesystem instead all supported (for tests with '.all_filesystems'). | 'LTP_DEV_FS_TYPE' | Filesystem used for testing (default: 'ext2'). -| 'LTP_TIMEOUT_MUL' | Multiply timeout, must be number >= 1 (> 1 is useful for +| 'LTP_TIMEOUT_MUL' | Multiplies timeout, must be number >= 0.1 (> 1 is useful for slow machines to avoid unexpected timeout). Variable is also used in shell tests, but ceiled to int. +| 'LTP_RUNTIME_MUL' | Multiplies maximal test iteration runtime. Tests + that run for more than a second or two are capped on + runtime. You can scale the default runtime both up + and down with this multiplier. | 'LTP_VIRT_OVERRIDE' | Overrides virtual machine detection in the test library. Setting it to empty string tell the library that system is not a virtual machine. Other possible @@ -36,3 +40,27 @@ For running LTP network tests see `testcases/network/README.md`. and others, which imply it, shell: 'TST_NEEDS_TMPDIR=1'). | 'TST_NO_CLEANUP' | Disable running test cleanup (defined in 'TST_CLEANUP'). |============================================================================== + + +2. Test execution time and timeout +---------------------------------- + +The limit on how long a test can run does compose of two parts max_runtime and +timeout. The limit does apply to a single test variant, that means for example +that tests that run for all available filesystems will apply this limit for a +single filesytem only. + +The max_runtime is a cap on how long the run() function can take, for most +testcases this part is set to zero. For tests that do run for more than a +second or two the max_runtime has to be defined and the run() function has to +check actively how much runtime is left. + +Test runtime can be scaled up and down with 'LTP_RUNTIME_MUL' environment +variable or set on a commandline by the '-I' parameter. Hoewever be vary that +setting the runtime too low will cause long running tests to exit prematurely +possibly before the have a chance actually test anyting. + +The timeout part is a limit for the test setup and cleanup and also safety +margin for the runtime accounting. It's currently set to 30 seconds but may +change later. If your target machine is too slow it can be scaled up with the +'LTP_TIMEOUT_MUL' environment variable. diff --git a/include/tst_test.h b/include/tst_test.h index dbe303bc879..01be56cd89f 100644 --- a/include/tst_test.h +++ b/include/tst_test.h @@ -134,6 +134,8 @@ extern unsigned int tst_variant; #define TST_NO_HUGEPAGES ((unsigned long)-1) +#define TST_UNLIMITED_RUNTIME (-1) + struct tst_test { /* number of tests available in test() function */ unsigned int tcnt; @@ -236,6 +238,18 @@ struct tst_test { /* override default timeout per test run, disabled == -1 */ int timeout; + /* + * Maximal test runtime in seconds. + * + * Any test that runs for more than a second or two should set this and + * also use tst_remaining_runtime() to exit when runtime was used up. + * Tests may finish sooner, for example if requested number of + * iterations was reached before the runtime runs out. + * + * If test runtime cannot be know in advance it should be set to + * TST_UNLIMITED_RUNTIME. + */ + int max_runtime; void (*setup)(void); void (*cleanup)(void); @@ -323,6 +337,19 @@ unsigned int tst_timeout_remaining(void); unsigned int tst_multiply_timeout(unsigned int timeout); void tst_set_timeout(int timeout); +/* + * Returns remaining test runtime. Test that runs for more than a few seconds + * should check if they should exit by calling this function regularly. + * + * The function returns remaining runtime in seconds. If runtime was used up + * zero is returned. + */ +unsigned int tst_remaining_runtime(void); + +/* + * Sets maximal test runtime in seconds. + */ +void tst_set_max_runtime(int max_runtime); /* * Returns path to the test temporary directory in a newly allocated buffer. diff --git a/lib/newlib_tests/.gitignore b/lib/newlib_tests/.gitignore index f4414f6a1c3..59b57d06382 100644 --- a/lib/newlib_tests/.gitignore +++ b/lib/newlib_tests/.gitignore @@ -7,9 +7,7 @@ test06 test07 test08 test09 -test10 test11 -test12 test13 test14 test15 @@ -22,7 +20,6 @@ tst_safe_fileops tst_res_hexd tst_strstatus tst_print_result -test18 test19 test20 test22 @@ -56,3 +53,5 @@ tst_needs_cmds05 tst_needs_cmds06 tst_needs_cmds07 tst_needs_cmds08 +test_runtime01 +test_runtime02 diff --git a/lib/newlib_tests/runtest.sh b/lib/newlib_tests/runtest.sh index 327460e7bf5..f136bcb88b5 100755 --- a/lib/newlib_tests/runtest.sh +++ b/lib/newlib_tests/runtest.sh @@ -1,7 +1,7 @@ #!/bin/sh # Copyright (c) 2021 Petr Vorel -LTP_C_API_TESTS="${LTP_C_API_TESTS:-test05 test07 test09 test12 test15 test18 +LTP_C_API_TESTS="${LTP_C_API_TESTS:-test05 test07 test09 test15 test_runtime01 tst_needs_cmds01 tst_needs_cmds02 tst_needs_cmds03 tst_needs_cmds06 tst_needs_cmds07 tst_bool_expr test_exec test_timer tst_res_hexd tst_strstatus tst_fuzzy_sync03 test_zero_hugepage.sh test_kconfig.sh diff --git a/lib/newlib_tests/test10.c b/lib/newlib_tests/test10.c deleted file mode 100644 index df61908ac40..00000000000 --- a/lib/newlib_tests/test10.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2016 Linux Test Project - */ - -/* - * Test for watchdog timeout. - */ - -#include "tst_test.h" - - -static void do_test(void) -{ - sleep(2); - tst_res(TPASS, "Not reached"); -} - -static struct tst_test test = { - .test_all = do_test, - .timeout = 1, -}; diff --git a/lib/newlib_tests/test12.c b/lib/newlib_tests/test12.c deleted file mode 100644 index b4f0d63038d..00000000000 --- a/lib/newlib_tests/test12.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2016 Linux Test Project - */ - -/* - * Test for timeout override. - */ - -#include "tst_test.h" - -static void do_test(void) -{ - sleep(1); - tst_res(TPASS, "Passed!"); -} - -static struct tst_test test = { - .timeout = 2, - .test_all = do_test, -}; diff --git a/lib/newlib_tests/test13.c b/lib/newlib_tests/test13.c index c447dc3dcfa..83c48f734db 100644 --- a/lib/newlib_tests/test13.c +++ b/lib/newlib_tests/test13.c @@ -20,7 +20,6 @@ static void do_test(void) } static struct tst_test test = { - .timeout = 1, .forks_child = 1, .test_all = do_test, }; diff --git a/lib/newlib_tests/test18.c b/lib/newlib_tests/test18.c deleted file mode 100644 index 026435d7d5a..00000000000 --- a/lib/newlib_tests/test18.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2018, Linux Test Project - */ - -#include -#include -#include "tst_test.h" - -static void run(void) -{ - do { - sleep(1); - } while (tst_timeout_remaining() >= 4); - - tst_res(TPASS, "Timeout remaining: %d", tst_timeout_remaining()); -} - -static struct tst_test test = { - .test_all = run, - .timeout = 5 -}; diff --git a/lib/newlib_tests/test_children_cleanup.c b/lib/newlib_tests/test_children_cleanup.c index 2b1ca5f9c8e..4a1313f6d6f 100644 --- a/lib/newlib_tests/test_children_cleanup.c +++ b/lib/newlib_tests/test_children_cleanup.c @@ -39,5 +39,4 @@ static void run(void) static struct tst_test test = { .test_all = run, .forks_child = 1, - .timeout = 10, }; diff --git a/lib/newlib_tests/test_runtime01.c b/lib/newlib_tests/test_runtime01.c new file mode 100644 index 00000000000..5e027546db3 --- /dev/null +++ b/lib/newlib_tests/test_runtime01.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018, Linux Test Project + * + * Runs for 4 seconds for each test iteration. + */ +#include +#include +#include "tst_test.h" + +static void run(void) +{ + int runtime; + + tst_res(TINFO, "Running variant %i", tst_variant); + + do { + runtime = tst_remaining_runtime(); + tst_res(TINFO, "Remaining runtime %d", runtime); + sleep(1); + } while (runtime); + + tst_res(TPASS, "Finished loop!"); +} + +static struct tst_test test = { + .test_all = run, + .max_runtime = 4, + .test_variants = 2, +}; diff --git a/lib/newlib_tests/test_runtime02.c b/lib/newlib_tests/test_runtime02.c new file mode 100644 index 00000000000..6d89cb5310e --- /dev/null +++ b/lib/newlib_tests/test_runtime02.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021, Linux Test Project + */ +/* + * This test is set up so that the timeout is not long enough to guarantee + * enough runtime for two iterations, i.e. the timeout without offset and after + * scaling is too small and the tests ends up with TBROK. + * + * The default timeout in the test library is set to 30 seconds. The test + * runtime is set to 5 so the test should timeout after 35 seconds. + */ + +#include +#include +#include "tst_test.h" + +static void run(void) +{ + tst_res(TINFO, "Sleeping for 40 seconds"); + sleep(40); + tst_res(TFAIL, "Still alive"); +} + +static struct tst_test test = { + .test_all = run, + .max_runtime = 5, +}; diff --git a/lib/tst_test.c b/lib/tst_test.c index 8e258594a4d..ef2e614bb27 100644 --- a/lib/tst_test.c +++ b/lib/tst_test.c @@ -45,6 +45,8 @@ const char *TCID __attribute__((weak)); #define GLIBC_GIT_URL "https://sourceware.org/git/?p=glibc.git;a=commit;h=" #define CVE_DB_URL "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-" +#define DEFAULT_TIMEOUT 30 + struct tst_test *tst_test; static const char *tid; @@ -63,6 +65,7 @@ struct results { int warnings; int broken; unsigned int timeout; + int max_runtime; }; static struct results *results; @@ -464,6 +467,40 @@ pid_t safe_clone(const char *file, const int lineno, return pid; } +static void parse_mul(float *mul, const char *env_name, float min, float max) +{ + char *str_mul; + int ret; + + if (*mul > 0) + return; + + str_mul = getenv(env_name); + + if (!str_mul) { + *mul = 1; + return; + } + + ret = tst_parse_float(str_mul, mul, min, max); + if (ret) { + tst_brk(TBROK, "Failed to parse %s: %s", + env_name, tst_strerrno(ret)); + } +} + +static int multiply_runtime(int max_runtime) +{ + static float runtime_mul = -1; + + if (max_runtime <= 0) + return max_runtime; + + parse_mul(&runtime_mul, "LTP_RUNTIME_MUL", 0.0099, 100); + + return max_runtime * runtime_mul; +} + static struct option { char *optstr; char *help; @@ -477,6 +514,7 @@ static struct option { static void print_help(void) { unsigned int i; + int timeout, runtime; /* see doc/user-guide.txt, which lists also shell API variables */ fprintf(stderr, "Environment Variables\n"); @@ -489,10 +527,32 @@ static void print_help(void) fprintf(stderr, "LTP_DEV_FS_TYPE Filesystem used for testing (default: %s)\n", DEFAULT_FS_TYPE); fprintf(stderr, "LTP_SINGLE_FS_TYPE Testing only - specifies filesystem instead all supported (for .all_filesystems)\n"); fprintf(stderr, "LTP_TIMEOUT_MUL Timeout multiplier (must be a number >=1)\n"); + fprintf(stderr, "LTP_RUNTIME_MUL Runtime multiplier (must be a number >=1)\n"); fprintf(stderr, "LTP_VIRT_OVERRIDE Overrides virtual machine detection (values: \"\"|kvm|microsoft|xen|zvm)\n"); fprintf(stderr, "TMPDIR Base directory for template directory (for .needs_tmpdir, default: %s)\n", TEMPDIR); fprintf(stderr, "\n"); + fprintf(stderr, "Timeout and runtime\n"); + fprintf(stderr, "-------------------\n"); + + if (tst_test->max_runtime) { + runtime = multiply_runtime(tst_test->max_runtime); + + if (runtime == TST_UNLIMITED_RUNTIME) { + fprintf(stderr, "Test iteration runtime is not limited\n"); + } else { + fprintf(stderr, "Test iteration runtime cap %ih %im %is\n", + runtime/3600, (runtime%3600)/60, runtime % 60); + } + } + + timeout = tst_multiply_timeout(DEFAULT_TIMEOUT); + + fprintf(stderr, "Test timeout (not including runtime) %ih %im %is\n", + timeout/3600, (timeout%3600)/60, timeout % 60); + + fprintf(stderr, "\n"); + fprintf(stderr, "Options\n"); fprintf(stderr, "-------\n"); @@ -620,7 +680,10 @@ static void parse_opts(int argc, char *argv[]) iterations = atoi(optarg); break; case 'I': - duration = atof(optarg); + if (tst_test->max_runtime > 0) + tst_test->max_runtime = atoi(optarg); + else + duration = atof(optarg); break; case 'C': #ifdef UCLINUX @@ -1034,6 +1097,11 @@ static void do_setup(int argc, char *argv[]) if (!tst_test) tst_brk(TBROK, "No tests to run"); + if (tst_test->max_runtime < -1) { + tst_brk(TBROK, "Invalid runtime value %i", + results->max_runtime); + } + if (tst_test->tconf_msg) tst_brk(TCONF, "%s", tst_test->tconf_msg); @@ -1404,39 +1472,36 @@ static void sigint_handler(int sig LTP_ATTRIBUTE_UNUSED) } unsigned int tst_timeout_remaining(void) +{ + tst_brk(TBROK, "Stub called!"); + return 0; +} + +unsigned int tst_remaining_runtime(void) { static struct timespec now; - unsigned int elapsed; + int elapsed; + + if (results->max_runtime == TST_UNLIMITED_RUNTIME) + return UINT_MAX; + + if (results->max_runtime == 0) + tst_brk(TBROK, "Runtime not set!"); if (tst_clock_gettime(CLOCK_MONOTONIC, &now)) tst_res(TWARN | TERRNO, "tst_clock_gettime() failed"); - elapsed = (tst_timespec_diff_ms(now, tst_start_time) + 500) / 1000; - if (results->timeout > elapsed) - return results->timeout - elapsed; + elapsed = tst_timespec_diff_ms(now, tst_start_time) / 1000; + if (results->max_runtime > elapsed) + return results->max_runtime - elapsed; return 0; } + unsigned int tst_multiply_timeout(unsigned int timeout) { - char *mul; - int ret; - - if (timeout_mul == -1) { - mul = getenv("LTP_TIMEOUT_MUL"); - if (mul) { - if ((ret = tst_parse_float(mul, &timeout_mul, 1, 10000))) { - tst_brk(TBROK, "Failed to parse LTP_TIMEOUT_MUL: %s", - tst_strerrno(ret)); - } - } else { - timeout_mul = 1; - } - } - if (timeout_mul < 1) - tst_brk(TBROK, "LTP_TIMEOUT_MUL must to be int >= 1! (%.2f)", - timeout_mul); + parse_mul(&timeout_mul, "LTP_TIMEOUT_MUL", 0.099, 10000); if (timeout < 1) tst_brk(TBROK, "timeout must to be >= 1! (%d)", timeout); @@ -1446,37 +1511,47 @@ unsigned int tst_multiply_timeout(unsigned int timeout) void tst_set_timeout(int timeout) { - if (timeout == -1) { + tst_brk(TBROK, "Stub called!"); +} + +static void set_timeout(void) +{ + unsigned int timeout = DEFAULT_TIMEOUT; + + if (results->max_runtime == TST_UNLIMITED_RUNTIME) { tst_res(TINFO, "Timeout per run is disabled"); return; } - if (timeout < 1) - tst_brk(TBROK, "timeout must to be >= 1! (%d)", timeout); + if (results->max_runtime < 0) { + tst_brk(TBROK, "max_runtime must to be >= -1! (%d)", + results->max_runtime); + } - results->timeout = tst_multiply_timeout(timeout); + results->timeout = tst_multiply_timeout(timeout) + results->max_runtime; tst_res(TINFO, "Timeout per run is %uh %02um %02us", results->timeout/3600, (results->timeout%3600)/60, results->timeout % 60); +} - if (getpid() == lib_pid) - alarm(results->timeout); - else - heartbeat(); +void tst_set_max_runtime(int max_runtime) +{ + results->max_runtime = multiply_runtime(max_runtime); + tst_res(TINFO, "Updating max runtime to %uh %02um %02us", + max_runtime/3600, (max_runtime%3600)/60, max_runtime % 60); + set_timeout(); + heartbeat(); } static int fork_testrun(void) { int status; - if (tst_test->timeout) - tst_set_timeout(tst_test->timeout); - else - tst_set_timeout(300); - SAFE_SIGNAL(SIGINT, sigint_handler); + alarm(results->timeout); + test_pid = fork(); if (test_pid < 0) tst_brk(TBROK | TERRNO, "fork()"); @@ -1568,6 +1643,11 @@ void tst_run_tcases(int argc, char *argv[], struct tst_test *self) SAFE_SIGNAL(SIGALRM, alarm_handler); SAFE_SIGNAL(SIGUSR1, heartbeat_handler); + if (tst_test->max_runtime) + results->max_runtime = multiply_runtime(tst_test->max_runtime); + + set_timeout(); + if (tst_test->test_variants) test_variants = tst_test->test_variants;