From 3ac525db195f11c748fecfe8701e042f50300e40 Mon Sep 17 00:00:00 2001 From: Peter Hutterer Date: Wed, 27 Nov 2019 07:28:27 +1000 Subject: [PATCH] test: write our test case results out as junit xml files libcheck has the ability to write out XML files for test results, but converting those into junit isn't ideal, for a number of reasons: - junit xml is different to libcheck's xml, so not all data is available or useful. Especially with our litest wrappers around it. - litest forking off tests means we have to wrap around everything anyway to avoid multiple forks writing to the same test file. This is the minimal implementation since it's only user is likely the CI which we control fairly tightly. So there are a few corners we can skip: - no filename validation is performed by litest - we write out a lot of junit xml files (one per litest fork). Rather than collating those we just rely on the CI to find the files. Signed-off-by: Peter Hutterer --- .gitlab-ci.yml | 3 ++ meson.build | 6 ++-- test/litest.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 866ef4d20..98f393749 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -625,6 +625,9 @@ fedora:31@default-build: variables: FEDORA_VERSION: 31 needs: ['fedora:31@container-prep'] + artifacts: + reports: + junit: '$MESON_BUILDDIR/junit-*.xml' fedora:30@default-build: stage: distro diff --git a/meson.build b/meson.build index 0e5523fef..f79ef4f12 100644 --- a/meson.build +++ b/meson.build @@ -964,7 +964,8 @@ if get_option('tests') test('libinput-test-suite-@0@'.format(group), libinput_test_runner, suite : ['all', 'valgrind', 'root', 'hardware'], - args : ['--filter-group=@0@:*'.format(group)], + args : ['--filter-group=@0@:*'.format(group), + '--xml-output=junit-@0@-XXXXXX.xml'.format(group)], is_parallel : false, timeout : 1200) endforeach @@ -972,7 +973,8 @@ if get_option('tests') test('libinput-test-deviceless', libinput_test_runner, suite : ['all', 'valgrind'], - args: ['--filter-deviceless']) + args: ['--filter-deviceless', + '--xml-output=junit-deviceless-XXXXXX.xml']) valgrind = find_program('valgrind', required : false) if valgrind.found() diff --git a/test/litest.c b/test/litest.c index d517c4610..8c4e52ee9 100644 --- a/test/litest.c +++ b/test/litest.c @@ -82,6 +82,7 @@ static bool use_system_rules_quirks = false; const char *filter_test = NULL; const char *filter_device = NULL; const char *filter_group = NULL; +const char *xml_prefix = NULL; static struct quirks_context *quirks_context; struct created_file { @@ -831,6 +832,68 @@ quirk_log_handler(struct libinput *unused, vfprintf(stderr, format, args); } +static void +litest_export_xml(SRunner *sr, const char *xml_prefix) +{ + TestResult **results; + int nresults, nfailed; + char *filename; + int fd; + + /* This is the minimum-effort implementation here because its only + * real purpose is to make test logs look pretty in the gitlab CI. + * + * Which means: + * - there's no filename validation, if you supply a filename that + * mkstemps doesn't like, things go boom. + * - every fork writes out a separate junit.xml file. gitlab is better + * at collecting lots of files than I am at writing code to collect + * this across forks to write out only one file. + * - most of the content is pretty useless because libcheck only gives + * us minimal information. the libcheck XML file has more info like + * the duration of each test but it's more complicated to extract + * and we don't need it for now. + */ + filename = safe_strdup(xml_prefix); + fd = mkstemps(filename, 4); + + results = srunner_results(sr); + nresults = srunner_ntests_run(sr); + nfailed = srunner_ntests_failed(sr); + + dprintf(fd, "\n"); + dprintf(fd, "\n", + filename, + nresults, + nfailed); + dprintf(fd, " \n"); + for (int i = 0; i < nresults; i++) { + TestResult *r = results[i]; + + dprintf(fd, " \n", + tr_tcname(r), + tr_tcname(r), + tr_rtype(r) == CK_PASS ? "/" : ""); + if (tr_rtype(r) != CK_PASS) { + dprintf(fd, " \n", + tr_lfile(r), + tr_lno(r)); + dprintf(fd, " %s:%d\n", tr_lfile(r), tr_lno(r)); + dprintf(fd, " %s\n", tr_tcname(r)); + dprintf(fd, "\n"); + dprintf(fd, " %s\n", tr_msg(r)); + dprintf(fd, " \n"); + dprintf(fd, " \n"); + } + } + dprintf(fd, " \n"); + dprintf(fd, "\n"); + + free(results); + close(fd); + free(filename); +} + static int litest_run_suite(struct list *tests, int which, int max, int error_fd) { @@ -928,6 +991,10 @@ litest_run_suite(struct list *tests, int which, int max, int error_fd) goto out; srunner_run_all(sr, CK_ENV); + if (xml_prefix) + litest_export_xml(sr, xml_prefix); + + failed = srunner_ntests_failed(sr); if (failed) { TestResult **trs; @@ -4050,6 +4117,7 @@ litest_parse_argv(int argc, char **argv) OPT_FILTER_DEVICE, OPT_FILTER_GROUP, OPT_FILTER_DEVICELESS, + OPT_XML_PREFIX, OPT_JOBS, OPT_LIST, OPT_VERBOSE, @@ -4059,6 +4127,7 @@ litest_parse_argv(int argc, char **argv) { "filter-device", 1, 0, OPT_FILTER_DEVICE }, { "filter-group", 1, 0, OPT_FILTER_GROUP }, { "filter-deviceless", 0, 0, OPT_FILTER_DEVICELESS }, + { "xml-output", 1, 0, OPT_XML_PREFIX }, { "jobs", 1, 0, OPT_JOBS }, { "list", 0, 0, OPT_LIST }, { "verbose", 0, 0, OPT_VERBOSE }, @@ -4111,6 +4180,10 @@ litest_parse_argv(int argc, char **argv) " Glob to filter on test groups\n" " --filter-deviceless=.... \n" " Glob to filter on tests that do not create test devices\n" + " --xml-output=/path/to/file-XXXXXXX.xml\n" + " Write test output in libcheck's XML format\n" + " to the given files. The file must match the format\n" + " prefix-XXXXXX.xml and only the prefix is your choice.\n" " --verbose\n" " Enable verbose output\n" " --jobs 8\n" @@ -4136,6 +4209,9 @@ litest_parse_argv(int argc, char **argv) case OPT_FILTER_GROUP: filter_group = optarg; break; + case OPT_XML_PREFIX: + xml_prefix = optarg; + break; case 'j': case OPT_JOBS: jobs = atoi(optarg);