From 6bf0459a4272daa02fd9f0ef3b783c8ebacb6b4c Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 25 Jan 2023 09:36:07 +0000 Subject: [PATCH 1/3] tests: update clar test runner Update to the latest main version of clar, which includes improved xml summary output. --- tests/clar/clar.c | 42 ++++++++++++++++++++++++++++++--------- tests/clar/clar.h | 6 +++--- tests/clar/clar/print.h | 41 ++++++++++++++++++++++++-------------- tests/clar/clar/summary.h | 39 ++++++++++++++++++++++-------------- 4 files changed, 86 insertions(+), 42 deletions(-) diff --git a/tests/clar/clar.c b/tests/clar/clar.c index ca508d073ff..b3fd176d40e 100644 --- a/tests/clar/clar.c +++ b/tests/clar/clar.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include /* required for sandboxing */ #include @@ -86,6 +88,8 @@ typedef struct stat STAT_T; #endif +#define MAX(x, y) (((x) > (y)) ? (x) : (y)) + #include "clar.h" static void fs_rm(const char *_source); @@ -117,6 +121,8 @@ struct clar_report { const char *suite; enum cl_test_status status; + time_t start; + double elapsed; struct clar_error *errors; struct clar_error *last_error; @@ -145,7 +151,7 @@ static struct { int report_errors_only; int exit_on_error; - int report_suite_names; + int verbosity; int write_summary; char *summary_filename; @@ -186,7 +192,7 @@ struct clar_suite { static void clar_print_init(int test_count, int suite_count, const char *suite_names); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); -static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status failed); +static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed); static void clar_print_onsuite(const char *suite_name, int suite_index); static void clar_print_onabort(const char *msg, ...); @@ -247,10 +253,14 @@ clar_report_all(void) static void clar_run_test( + const struct clar_suite *suite, const struct clar_func *test, const struct clar_func *initialize, const struct clar_func *cleanup) { + struct timeval start, end; + struct timezone tz; + _clar.trampoline_enabled = 1; CL_TRACE(CL_TRACE__TEST__BEGIN); @@ -259,16 +269,24 @@ clar_run_test( if (initialize->ptr != NULL) initialize->ptr(); + _clar.last_report->start = time(NULL); + gettimeofday(&start, &tz); + CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); test->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_END); } + gettimeofday(&end, &tz); + _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; + _clar.last_report->elapsed = ((double)end.tv_sec + (double)end.tv_usec / 1.0E6) - + ((double)start.tv_sec + (double)start.tv_usec / 1.0E6); + if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -286,7 +304,7 @@ clar_run_test( if (_clar.report_errors_only) { clar_report_errors(_clar.last_report); } else { - clar_print_ontest(test->name, _clar.tests_ran, _clar.last_report->status); + clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status); } } @@ -352,7 +370,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.last_report = report; - clar_run_test(&test[i], &suite->initialize, &suite->cleanup); + clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) return; @@ -427,7 +445,7 @@ clar_parse_args(int argc, char **argv) ++found; if (!exact) - _clar.report_suite_names = 1; + _clar.verbosity = MAX(_clar.verbosity, 1); switch (action) { case 's': { @@ -486,13 +504,13 @@ clar_parse_args(int argc, char **argv) } case 'v': - _clar.report_suite_names = 1; + _clar.verbosity++; break; case 'r': _clar.write_summary = 1; free(_clar.summary_filename); - _clar.summary_filename = strdup(*(argument + 2) ? (argument + 2) : "summary.xml"); + _clar.summary_filename = *(argument + 2) ? strdup(argument + 2) : NULL; break; default: @@ -504,6 +522,8 @@ clar_parse_args(int argc, char **argv) void clar_test_init(int argc, char **argv) { + const char *summary_env; + if (argc > 1) clar_parse_args(argc, argv); @@ -513,11 +533,15 @@ clar_test_init(int argc, char **argv) "" ); - if ((_clar.summary_filename = getenv("CLAR_SUMMARY")) != NULL) { + if (!_clar.summary_filename && + (summary_env = getenv("CLAR_SUMMARY")) != NULL) { _clar.write_summary = 1; - _clar.summary_filename = strdup(_clar.summary_filename); + _clar.summary_filename = strdup(summary_env); } + if (_clar.write_summary && !_clar.summary_filename) + _clar.summary_filename = strdup("summary.xml"); + if (_clar.write_summary && !(_clar.summary = clar_summary_init(_clar.summary_filename))) { clar_print_onabort("Failed to open the summary file\n"); diff --git a/tests/clar/clar.h b/tests/clar/clar.h index 3f659c2f6c4..8c22382bd56 100644 --- a/tests/clar/clar.h +++ b/tests/clar/clar.h @@ -13,12 +13,12 @@ enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, CL_TEST_SKIP, - CL_TEST_NOTRUN + CL_TEST_NOTRUN, }; enum cl_output_format { CL_OUTPUT_CLAP, - CL_OUTPUT_TAP + CL_OUTPUT_TAP, }; /** Setup clar environment */ @@ -60,7 +60,7 @@ typedef enum cl_trace_event { CL_TRACE__TEST__END, CL_TRACE__TEST__RUN_BEGIN, CL_TRACE__TEST__RUN_END, - CL_TRACE__TEST__LONGJMP + CL_TRACE__TEST__LONGJMP, } cl_trace_event; typedef void (cl_trace_cb)( diff --git a/tests/clar/clar/print.h b/tests/clar/clar/print.h index dbfd276550a..c17e2f693bd 100644 --- a/tests/clar/clar/print.h +++ b/tests/clar/clar/print.h @@ -36,24 +36,35 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con fflush(stdout); } -static void clar_print_clap_ontest(const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) { (void)test_name; (void)test_number; - switch(status) { - case CL_TEST_OK: printf("."); break; - case CL_TEST_FAILURE: printf("F"); break; - case CL_TEST_SKIP: printf("S"); break; - case CL_TEST_NOTRUN: printf("N"); break; + if (_clar.verbosity > 1) { + printf("%s::%s: ", suite_name, test_name); + + switch (status) { + case CL_TEST_OK: printf("ok\n"); break; + case CL_TEST_FAILURE: printf("fail\n"); break; + case CL_TEST_SKIP: printf("skipped"); break; + case CL_TEST_NOTRUN: printf("notrun"); break; + } + } else { + switch (status) { + case CL_TEST_OK: printf("."); break; + case CL_TEST_FAILURE: printf("F"); break; + case CL_TEST_SKIP: printf("S"); break; + case CL_TEST_NOTRUN: printf("N"); break; + } + + fflush(stdout); } - - fflush(stdout); } static void clar_print_clap_onsuite(const char *suite_name, int suite_index) { - if (_clar.report_suite_names) + if (_clar.verbosity == 1) printf("\n%s", suite_name); (void)suite_index; @@ -102,7 +113,7 @@ static void print_escaped(const char *str) printf("%s", str); } -static void clar_print_tap_ontest(const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) { const struct clar_error *error = _clar.last_report->errors; @@ -111,10 +122,10 @@ static void clar_print_tap_ontest(const char *test_name, int test_number, enum c switch(status) { case CL_TEST_OK: - printf("ok %d - %s::%s\n", test_number, _clar.active_suite, test_name); + printf("ok %d - %s::%s\n", test_number, suite_name, test_name); break; case CL_TEST_FAILURE: - printf("not ok %d - %s::%s\n", test_number, _clar.active_suite, test_name); + printf("not ok %d - %s::%s\n", test_number, suite_name, test_name); printf(" ---\n"); printf(" reason: |\n"); @@ -132,7 +143,7 @@ static void clar_print_tap_ontest(const char *test_name, int test_number, enum c break; case CL_TEST_SKIP: case CL_TEST_NOTRUN: - printf("ok %d - # SKIP %s::%s\n", test_number, _clar.active_suite, test_name); + printf("ok %d - # SKIP %s::%s\n", test_number, suite_name, test_name); break; } @@ -181,9 +192,9 @@ static void clar_print_error(int num, const struct clar_report *report, const st PRINT(error, num, report, error); } -static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) { - PRINT(ontest, test_name, test_number, status); + PRINT(ontest, suite_name, test_name, test_number, status); } static void clar_print_onsuite(const char *suite_name, int suite_index) diff --git a/tests/clar/clar/summary.h b/tests/clar/clar/summary.h index 6279f5057d7..a259ff56956 100644 --- a/tests/clar/clar/summary.h +++ b/tests/clar/clar/summary.h @@ -2,7 +2,7 @@ #include #include -static int clar_summary_close_tag( +int clar_summary_close_tag( struct clar_summary *summary, const char *tag, int indent) { const char *indt; @@ -14,14 +14,14 @@ static int clar_summary_close_tag( return fprintf(summary->fp, "%s\n", indt, tag); } -static int clar_summary_testsuites(struct clar_summary *summary) +int clar_summary_testsuites(struct clar_summary *summary) { return fprintf(summary->fp, "\n"); } -static int clar_summary_testsuite(struct clar_summary *summary, - int idn, const char *name, const char *pkg, time_t timestamp, - double elapsed, int test_count, int fail_count, int error_count) +int clar_summary_testsuite(struct clar_summary *summary, + int idn, const char *name, time_t timestamp, + int test_count, int fail_count, int error_count) { struct tm *tm = localtime(×tamp); char iso_dt[20]; @@ -29,20 +29,18 @@ static int clar_summary_testsuite(struct clar_summary *summary, if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", tm) == 0) return -1; - return fprintf(summary->fp, "\tfp, "\t\n", - idn, name, pkg, iso_dt, elapsed, test_count, fail_count, error_count); + idn, name, iso_dt, test_count, fail_count, error_count); } -static int clar_summary_testcase(struct clar_summary *summary, +int clar_summary_testcase(struct clar_summary *summary, const char *name, const char *classname, double elapsed) { return fprintf(summary->fp, @@ -50,7 +48,7 @@ static int clar_summary_testcase(struct clar_summary *summary, name, classname, elapsed); } -static int clar_summary_failure(struct clar_summary *summary, +int clar_summary_failure(struct clar_summary *summary, const char *type, const char *message, const char *desc) { return fprintf(summary->fp, @@ -58,15 +56,23 @@ static int clar_summary_failure(struct clar_summary *summary, type, message, desc); } +int clar_summary_skipped(struct clar_summary *summary) +{ + return fprintf(summary->fp, "\t\t\t\n"); +} + struct clar_summary *clar_summary_init(const char *filename) { struct clar_summary *summary; FILE *fp; - if ((fp = fopen(filename, "w")) == NULL) + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); return NULL; + } if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); fclose(fp); return NULL; } @@ -90,14 +96,14 @@ int clar_summary_shutdown(struct clar_summary *summary) struct clar_error *error = report->errors; if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_testsuite(summary, 0, report->suite, "", - time(NULL), 0, _clar.tests_ran, _clar.total_errors, 0) < 0) + if (clar_summary_testsuite(summary, 0, report->suite, + report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) goto on_error; } last_suite = report->suite; - clar_summary_testcase(summary, report->test, "what", 0); + clar_summary_testcase(summary, report->test, report->suite, report->elapsed); while (error != NULL) { if (clar_summary_failure(summary, "assert", @@ -107,6 +113,9 @@ int clar_summary_shutdown(struct clar_summary *summary) error = error->next; } + if (report->status == CL_TEST_SKIP) + clar_summary_skipped(summary); + if (clar_summary_close_tag(summary, "testcase", 2) < 0) goto on_error; From 4e15c9c87fe238364fd54b01f108c9671093b5d0 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 25 Jan 2023 09:36:48 +0000 Subject: [PATCH 2/3] tests: fix clar declarations --- tests/clar/clar/summary.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/clar/clar/summary.h b/tests/clar/clar/summary.h index a259ff56956..4dd352e28b8 100644 --- a/tests/clar/clar/summary.h +++ b/tests/clar/clar/summary.h @@ -2,7 +2,7 @@ #include #include -int clar_summary_close_tag( +static int clar_summary_close_tag( struct clar_summary *summary, const char *tag, int indent) { const char *indt; @@ -14,12 +14,12 @@ int clar_summary_close_tag( return fprintf(summary->fp, "%s\n", indt, tag); } -int clar_summary_testsuites(struct clar_summary *summary) +static int clar_summary_testsuites(struct clar_summary *summary) { return fprintf(summary->fp, "\n"); } -int clar_summary_testsuite(struct clar_summary *summary, +static int clar_summary_testsuite(struct clar_summary *summary, int idn, const char *name, time_t timestamp, int test_count, int fail_count, int error_count) { @@ -40,7 +40,7 @@ int clar_summary_testsuite(struct clar_summary *summary, idn, name, iso_dt, test_count, fail_count, error_count); } -int clar_summary_testcase(struct clar_summary *summary, +static int clar_summary_testcase(struct clar_summary *summary, const char *name, const char *classname, double elapsed) { return fprintf(summary->fp, @@ -48,7 +48,7 @@ int clar_summary_testcase(struct clar_summary *summary, name, classname, elapsed); } -int clar_summary_failure(struct clar_summary *summary, +static int clar_summary_failure(struct clar_summary *summary, const char *type, const char *message, const char *desc) { return fprintf(summary->fp, @@ -56,7 +56,7 @@ int clar_summary_failure(struct clar_summary *summary, type, message, desc); } -int clar_summary_skipped(struct clar_summary *summary) +static int clar_summary_skipped(struct clar_summary *summary) { return fprintf(summary->fp, "\t\t\t\n"); } From ed26a908febf143d4639245b679c2c7797160cc9 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 25 Jan 2023 09:37:22 +0000 Subject: [PATCH 3/3] clar: cross-platform elapsed time counter Abstract time counter for tests; use gettimeofday on Unix and GetTickCount on Windows. --- tests/clar/clar.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/tests/clar/clar.c b/tests/clar/clar.c index b3fd176d40e..c9c3fde3827 100644 --- a/tests/clar/clar.c +++ b/tests/clar/clar.c @@ -13,7 +13,6 @@ #include #include #include -#include /* required for sandboxing */ #include @@ -251,6 +250,37 @@ clar_report_all(void) } } +#ifdef WIN32 +# define clar_time DWORD + +static void clar_time_now(clar_time *out) +{ + *out = GetTickCount(); +} + +static double clar_time_diff(clar_time *start, clar_time *end) +{ + return ((double)*end - (double)*start) / 1000; +} +#else +# include + +# define clar_time struct timeval + +static void clar_time_now(clar_time *out) +{ + struct timezone tz; + + gettimeofday(out, &tz); +} + +static double clar_time_diff(clar_time *start, clar_time *end) +{ + return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) - + ((double)start->tv_sec + (double)start->tv_usec / 1.0E6); +} +#endif + static void clar_run_test( const struct clar_suite *suite, @@ -258,34 +288,32 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - struct timeval start, end; - struct timezone tz; + clar_time start, end; _clar.trampoline_enabled = 1; CL_TRACE(CL_TRACE__TEST__BEGIN); + _clar.last_report->start = time(NULL); + clar_time_now(&start); + if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) initialize->ptr(); - _clar.last_report->start = time(NULL); - gettimeofday(&start, &tz); - CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); test->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_END); } - gettimeofday(&end, &tz); + clar_time_now(&end); _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = ((double)end.tv_sec + (double)end.tv_usec / 1.0E6) - - ((double)start.tv_sec + (double)start.tv_usec / 1.0E6); + _clar.last_report->elapsed = clar_time_diff(&start, &end); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload);