Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests: update clar test runner #6459

Merged
merged 3 commits into from Jan 25, 2023
Merged

tests: update clar test runner #6459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6bf0459
tests: update clar test runner
ethomson Jan 25, 2023
4e15c9c
tests: fix clar declarations
ethomson Jan 25, 2023
ed26a90
clar: cross-platform elapsed time counter
ethomson Jan 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
70 changes: 61 additions & 9 deletions tests/clar/clar.c
View file
Open in desktop
Expand Up @@ -12,6 +12,7 @@
#include <math.h>
#include <stdarg.h>
#include <wchar.h>
#include <time.h>

/* required for sandboxing */
#include <sys/types.h>
Expand Down Expand Up @@ -86,6 +87,8 @@
typedef struct stat STAT_T;
#endif

#define MAX(x, y) (((x) > (y)) ? (x) : (y))

#include "clar.h"

static void fs_rm(const char *_source);
Expand Down Expand Up @@ -117,6 +120,8 @@ struct clar_report {
const char *suite;

enum cl_test_status status;
time_t start;
double elapsed;

struct clar_error *errors;
struct clar_error *last_error;
Expand Down Expand Up @@ -145,7 +150,7 @@ static struct {

int report_errors_only;
int exit_on_error;
int report_suite_names;
int verbosity;

int write_summary;
char *summary_filename;
Expand Down Expand Up @@ -186,7 +191,7 @@ struct clar_suite {
static void clar_print_init(int test_count, int suite_count, const char *suite_names);
static void clar_print_shutdown(int test_count, int suite_count, int error_count);
static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error);
static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status failed);
static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed);
static void clar_print_onsuite(const char *suite_name, int suite_index);
static void clar_print_onabort(const char *msg, ...);

Expand Down Expand Up @@ -245,16 +250,53 @@ clar_report_all(void)
}
}

#ifdef WIN32
# define clar_time DWORD

static void clar_time_now(clar_time *out)
{
*out = GetTickCount();
}

static double clar_time_diff(clar_time *start, clar_time *end)
{
return ((double)*end - (double)*start) / 1000;
}
#else
# include <sys/time.h>

# define clar_time struct timeval

static void clar_time_now(clar_time *out)
{
struct timezone tz;

gettimeofday(out, &tz);
}

static double clar_time_diff(clar_time *start, clar_time *end)
{
return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) -
((double)start->tv_sec + (double)start->tv_usec / 1.0E6);
}
#endif

static void
clar_run_test(
const struct clar_suite *suite,
const struct clar_func *test,
const struct clar_func *initialize,
const struct clar_func *cleanup)
{
clar_time start, end;

_clar.trampoline_enabled = 1;

CL_TRACE(CL_TRACE__TEST__BEGIN);

_clar.last_report->start = time(NULL);
clar_time_now(&start);

if (setjmp(_clar.trampoline) == 0) {
if (initialize->ptr != NULL)
initialize->ptr();
Expand All @@ -264,11 +306,15 @@ clar_run_test(
CL_TRACE(CL_TRACE__TEST__RUN_END);
}

clar_time_now(&end);

_clar.trampoline_enabled = 0;

if (_clar.last_report->status == CL_TEST_NOTRUN)
_clar.last_report->status = CL_TEST_OK;

_clar.last_report->elapsed = clar_time_diff(&start, &end);

if (_clar.local_cleanup != NULL)
_clar.local_cleanup(_clar.local_cleanup_payload);

Expand All @@ -286,7 +332,7 @@ clar_run_test(
if (_clar.report_errors_only) {
clar_report_errors(_clar.last_report);
} else {
clar_print_ontest(test->name, _clar.tests_ran, _clar.last_report->status);
clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status);
}
}

Expand Down Expand Up @@ -352,7 +398,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)

_clar.last_report = report;

clar_run_test(&test[i], &suite->initialize, &suite->cleanup);
clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup);

if (_clar.exit_on_error && _clar.total_errors)
return;
Expand Down Expand Up @@ -427,7 +473,7 @@ clar_parse_args(int argc, char **argv)
++found;

if (!exact)
_clar.report_suite_names = 1;
_clar.verbosity = MAX(_clar.verbosity, 1);

switch (action) {
case 's': {
Expand Down Expand Up @@ -486,13 +532,13 @@ clar_parse_args(int argc, char **argv)
}

case 'v':
_clar.report_suite_names = 1;
_clar.verbosity++;
break;

case 'r':
_clar.write_summary = 1;
free(_clar.summary_filename);
_clar.summary_filename = strdup(*(argument + 2) ? (argument + 2) : "summary.xml");
_clar.summary_filename = *(argument + 2) ? strdup(argument + 2) : NULL;
break;

default:
Expand All @@ -504,6 +550,8 @@ clar_parse_args(int argc, char **argv)
void
clar_test_init(int argc, char **argv)
{
const char *summary_env;

if (argc > 1)
clar_parse_args(argc, argv);

Expand All @@ -513,11 +561,15 @@ clar_test_init(int argc, char **argv)
""
);

if ((_clar.summary_filename = getenv("CLAR_SUMMARY")) != NULL) {
if (!_clar.summary_filename &&
(summary_env = getenv("CLAR_SUMMARY")) != NULL) {
_clar.write_summary = 1;
_clar.summary_filename = strdup(_clar.summary_filename);
_clar.summary_filename = strdup(summary_env);
}

if (_clar.write_summary && !_clar.summary_filename)
_clar.summary_filename = strdup("summary.xml");

if (_clar.write_summary &&
!(_clar.summary = clar_summary_init(_clar.summary_filename))) {
clar_print_onabort("Failed to open the summary file\n");
Expand Down
6 changes: 3 additions & 3 deletions tests/clar/clar.h
View file
Open in desktop
Expand Up @@ -13,12 +13,12 @@ enum cl_test_status {
CL_TEST_OK,
CL_TEST_FAILURE,
CL_TEST_SKIP,
CL_TEST_NOTRUN
CL_TEST_NOTRUN,
};

enum cl_output_format {
CL_OUTPUT_CLAP,
CL_OUTPUT_TAP
CL_OUTPUT_TAP,
};

/** Setup clar environment */
Expand Down Expand Up @@ -60,7 +60,7 @@ typedef enum cl_trace_event {
CL_TRACE__TEST__END,
CL_TRACE__TEST__RUN_BEGIN,
CL_TRACE__TEST__RUN_END,
CL_TRACE__TEST__LONGJMP
CL_TRACE__TEST__LONGJMP,
} cl_trace_event;

typedef void (cl_trace_cb)(
Expand Down
41 changes: 26 additions & 15 deletions tests/clar/clar/print.h
View file
Open in desktop
Expand Up @@ -36,24 +36,35 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con
fflush(stdout);
}

static void clar_print_clap_ontest(const char *test_name, int test_number, enum cl_test_status status)
static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
{
(void)test_name;
(void)test_number;

switch(status) {
case CL_TEST_OK: printf("."); break;
case CL_TEST_FAILURE: printf("F"); break;
case CL_TEST_SKIP: printf("S"); break;
case CL_TEST_NOTRUN: printf("N"); break;
if (_clar.verbosity > 1) {
printf("%s::%s: ", suite_name, test_name);

switch (status) {
case CL_TEST_OK: printf("ok\n"); break;
case CL_TEST_FAILURE: printf("fail\n"); break;
case CL_TEST_SKIP: printf("skipped"); break;
case CL_TEST_NOTRUN: printf("notrun"); break;
}
} else {
switch (status) {
case CL_TEST_OK: printf("."); break;
case CL_TEST_FAILURE: printf("F"); break;
case CL_TEST_SKIP: printf("S"); break;
case CL_TEST_NOTRUN: printf("N"); break;
}

fflush(stdout);
}

fflush(stdout);
}

static void clar_print_clap_onsuite(const char *suite_name, int suite_index)
{
if (_clar.report_suite_names)
if (_clar.verbosity == 1)
printf("\n%s", suite_name);

(void)suite_index;
Expand Down Expand Up @@ -102,7 +113,7 @@ static void print_escaped(const char *str)
printf("%s", str);
}

static void clar_print_tap_ontest(const char *test_name, int test_number, enum cl_test_status status)
static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
{
const struct clar_error *error = _clar.last_report->errors;

Expand All @@ -111,10 +122,10 @@ static void clar_print_tap_ontest(const char *test_name, int test_number, enum c

switch(status) {
case CL_TEST_OK:
printf("ok %d - %s::%s\n", test_number, _clar.active_suite, test_name);
printf("ok %d - %s::%s\n", test_number, suite_name, test_name);
break;
case CL_TEST_FAILURE:
printf("not ok %d - %s::%s\n", test_number, _clar.active_suite, test_name);
printf("not ok %d - %s::%s\n", test_number, suite_name, test_name);

printf(" ---\n");
printf(" reason: |\n");
Expand All @@ -132,7 +143,7 @@ static void clar_print_tap_ontest(const char *test_name, int test_number, enum c
break;
case CL_TEST_SKIP:
case CL_TEST_NOTRUN:
printf("ok %d - # SKIP %s::%s\n", test_number, _clar.active_suite, test_name);
printf("ok %d - # SKIP %s::%s\n", test_number, suite_name, test_name);
break;
}

Expand Down Expand Up @@ -181,9 +192,9 @@ static void clar_print_error(int num, const struct clar_report *report, const st
PRINT(error, num, report, error);
}

static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status status)
static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
{
PRINT(ontest, test_name, test_number, status);
PRINT(ontest, suite_name, test_name, test_number, status);
}

static void clar_print_onsuite(const char *suite_name, int suite_index)
Expand Down
29 changes: 19 additions & 10 deletions tests/clar/clar/summary.h
View file
Open in desktop
Expand Up @@ -20,26 +20,24 @@ static int clar_summary_testsuites(struct clar_summary *summary)
}

static int clar_summary_testsuite(struct clar_summary *summary,
int idn, const char *name, const char *pkg, time_t timestamp,
double elapsed, int test_count, int fail_count, int error_count)
int idn, const char *name, time_t timestamp,
int test_count, int fail_count, int error_count)
{
struct tm *tm = localtime(&timestamp);
char iso_dt[20];

if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", tm) == 0)
return -1;

return fprintf(summary->fp, "\t<testsuite "
return fprintf(summary->fp, "\t<testsuite"
" id=\"%d\""
" name=\"%s\""
" package=\"%s\""
" hostname=\"localhost\""
" timestamp=\"%s\""
" time=\"%.2f\""
" tests=\"%d\""
" failures=\"%d\""
" errors=\"%d\">\n",
idn, name, pkg, iso_dt, elapsed, test_count, fail_count, error_count);
idn, name, iso_dt, test_count, fail_count, error_count);
}

static int clar_summary_testcase(struct clar_summary *summary,
Expand All @@ -58,15 +56,23 @@ static int clar_summary_failure(struct clar_summary *summary,
type, message, desc);
}

static int clar_summary_skipped(struct clar_summary *summary)
{
return fprintf(summary->fp, "\t\t\t<skipped />\n");
}

struct clar_summary *clar_summary_init(const char *filename)
{
struct clar_summary *summary;
FILE *fp;

if ((fp = fopen(filename, "w")) == NULL)
if ((fp = fopen(filename, "w")) == NULL) {
perror("fopen");
return NULL;
}

if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
perror("malloc");
fclose(fp);
return NULL;
}
Expand All @@ -90,14 +96,14 @@ int clar_summary_shutdown(struct clar_summary *summary)
struct clar_error *error = report->errors;

if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
if (clar_summary_testsuite(summary, 0, report->suite, "",
time(NULL), 0, _clar.tests_ran, _clar.total_errors, 0) < 0)
if (clar_summary_testsuite(summary, 0, report->suite,
report->start, _clar.tests_ran, _clar.total_errors, 0) < 0)
goto on_error;
}

last_suite = report->suite;

clar_summary_testcase(summary, report->test, "what", 0);
clar_summary_testcase(summary, report->test, report->suite, report->elapsed);

while (error != NULL) {
if (clar_summary_failure(summary, "assert",
Expand All @@ -107,6 +113,9 @@ int clar_summary_shutdown(struct clar_summary *summary)
error = error->next;
}

if (report->status == CL_TEST_SKIP)
clar_summary_skipped(summary);

if (clar_summary_close_tag(summary, "testcase", 2) < 0)
goto on_error;

Expand Down