tests: add a teamcity output format for V's test runner (#16681)

Author: Makhnev Petr

cmd/tools/modules/testing/common.v

@@ -292,6 +292,9 @@ fn (mut ts TestSession) handle_test_runner_option() {
 		'dump' {
 			ts.reporter = DumpReporter{}
 		}
+		'teamcity' {
+			ts.reporter = TeamcityReporter{}
+		}
 		else {
 			dump('just set ts.reporter to an instance of your own struct here')
 		}

cmd/tools/modules/testing/output_teamcity.v

@@ -0,0 +1,57 @@
+module testing
+
+// TeamcityReporter implements the interface `testing.Reporter`.
+// It is used by `v -test-runner teamcity test .`
+pub struct TeamcityReporter {
+}
+
+pub fn (r TeamcityReporter) session_start(message string, mut ts TestSession) {
+}
+
+pub fn (r TeamcityReporter) session_stop(message string, mut ts TestSession) {
+}
+
+pub fn (r TeamcityReporter) report(index int, message LogMessage) {
+	name := r.get_test_suite_name_by_file(message.file)
+	match message.kind {
+		.cmd_begin {
+			eprintln("##teamcity[testSuiteStarted name='${name}' flowId='${message.flow_id}']")
+		}
+		.cmd_end {
+			eprintln("##teamcity[testSuiteFinished name='${name}' flowId='${message.flow_id}' duration='${message.took}']")
+		}
+		else {}
+	}
+}
+
+pub fn (r TeamcityReporter) get_test_suite_name_by_file(path string) string {
+	file_name := path.replace('\\', '/').split('/').last()
+	return file_name.split('.').first()
+}
+
+pub fn (r TeamcityReporter) report_stop() {
+}
+
+pub fn (r TeamcityReporter) progress(index int, message string) {
+}
+
+pub fn (r TeamcityReporter) update_last_line(index int, message string) {
+}
+
+pub fn (r TeamcityReporter) update_last_line_and_move_to_next(index int, message string) {
+}
+
+pub fn (r TeamcityReporter) message(index int, message string) {
+}
+
+pub fn (r TeamcityReporter) divider() {
+}
+
+pub fn (r TeamcityReporter) worker_threads_start(files []string, mut ts TestSession) {
+}
+
+pub fn (r TeamcityReporter) worker_threads_finish(mut ts TestSession) {
+}
+
+pub fn (r TeamcityReporter) list_of_failed_commands(failed_cmds []string) {
+}

cmd/v/help/test.txt

@@ -35,9 +35,10 @@ You can use several alternative test result formats, using `-test-runner name`,
 or by setting VTEST_RUNNER (the command line option has higher priority).
 
 The names of the available test runners are:
-    `simple` Fastest, does not import additional modules, does no processing.
-    `tap`    Format the output as required by the Test Anything Protocol (TAP).
-    `normal` Supports color output, nicest/most human readable, the default.
+    `simple`   Fastest, does not import additional modules, does no processing.
+    `tap`      Format the output as required by the Test Anything Protocol (TAP).
+    `normal`   Supports color output, nicest/most human readable, the default.
+    `teamcity` Format the output as required by the Teamcity and JetBrains IDEs.
 
 You can also implement your own custom test runner, by providing the path to
 your .v file, that implements it to this option. For example, see:

vlib/v/pref/pref.v

@@ -90,7 +90,7 @@ pub enum Arch {
 pub const list_of_flags_with_param = ['o', 'd', 'define', 'b', 'backend', 'cc', 'os', 'cf', 'cflags',
 	'path', 'arch']
 
-pub const supported_test_runners = ['normal', 'simple', 'tap', 'dump']
+pub const supported_test_runners = ['normal', 'simple', 'tap', 'dump', 'teamcity']
 
 [heap; minify]
 pub struct Preferences {

vlib/v/preludes/test_runner_teamcity.v

@@ -0,0 +1,136 @@
+module main
+
+import time
+
+// Provide a Teamcity implementation of the TestRunner interface.
+// Used in Teamcity and JetBrains IDEs for nice test reporting.
+
+fn vtest_init() {
+	change_test_runner(&TestRunner(TeamcityTestRunner{}))
+}
+
+struct TeamcityTestRunner {
+mut:
+	fname      string
+	start_time time.Time
+
+	file_test_info   VTestFileMetaInfo
+	fn_test_info     VTestFnMetaInfo
+	fn_assert_passes u64
+	fn_passes        u64
+	fn_fails         u64
+
+	total_assert_passes u64
+	total_assert_fails  u64
+}
+
+fn (mut runner TeamcityTestRunner) free() {
+	unsafe {
+		runner.fname.free()
+		runner.fn_test_info.free()
+		runner.file_test_info.free()
+	}
+}
+
+fn normalise_fname(name string) string {
+	return name.replace('__', '.').replace('main.', '')
+}
+
+fn (mut runner TeamcityTestRunner) start(ntests int) {
+}
+
+fn (mut runner TeamcityTestRunner) finish() {
+}
+
+fn (mut runner TeamcityTestRunner) exit_code() int {
+	if runner.fn_fails > 0 {
+		return 1
+	}
+	if runner.total_assert_fails > 0 {
+		return 1
+	}
+	return 0
+}
+
+//
+
+fn (mut runner TeamcityTestRunner) fn_start() bool {
+	runner.fn_assert_passes = 0
+	runner.start_time = time.now()
+	runner.fname = normalise_fname(runner.fn_test_info.name)
+	println("Start '${runner.fname}'")
+	eprintln("##teamcity[testStarted name='${runner.fname}' locationHint='v_qn://${runner.file_test_info.file}:${runner.fname}']")
+	return true
+}
+
+fn (mut runner TeamcityTestRunner) fn_pass() {
+	runner.fn_passes++
+	duration := runner.test_duration()
+	eprintln("##teamcity[testFinished name='${runner.fname}' duration='${duration}']")
+	println('\n')
+}
+
+fn (mut runner TeamcityTestRunner) fn_fail() {
+	runner.fn_fails++
+	duration := runner.test_duration()
+	eprintln("##teamcity[testFailed name='${runner.fname}' duration='${duration}' message='assertion failed']")
+	println('\n')
+}
+
+fn (mut runner TeamcityTestRunner) fn_error(line_nr int, file string, mod string, fn_name string, errmsg string) {
+	eprintln('>>> TeamcityTestRunner fn_error ${runner.fname}, line_nr: ${line_nr}, file: ${file}, mod: ${mod}, fn_name: ${fn_name}, errmsg: ${errmsg}')
+}
+
+fn (mut runner TeamcityTestRunner) test_duration() i64 {
+	return time.now().unix_time_milli() - runner.start_time.unix_time_milli()
+}
+
+//
+
+fn (mut runner TeamcityTestRunner) assert_pass(i &VAssertMetaInfo) {
+	runner.total_assert_passes++
+	runner.fn_assert_passes++
+
+	filepath := i.fpath.clone()
+	println('>>> assert_pass ${filepath}:${i.line_nr + 1}')
+
+	unsafe { i.free() }
+}
+
+fn (mut runner TeamcityTestRunner) assert_fail(i &VAssertMetaInfo) {
+	runner.total_assert_fails++
+
+	filepath := i.fpath.clone()
+	mut final_filepath := filepath + ':${i.line_nr + 1}:'
+	mut final_funcname := 'fn ' + i.fn_name.replace('main.', '').replace('__', '.')
+	final_src := 'assert ' + i.src
+	eprintln('${final_filepath} ${final_funcname}')
+	if i.op.len > 0 && i.op != 'call' {
+		mut lvtitle := '    Left value:'
+		mut rvtitle := '    Right value:'
+		mut slvalue := '${i.lvalue}'
+		mut srvalue := '${i.rvalue}'
+		cutoff_limit := 30
+		if slvalue.len > cutoff_limit || srvalue.len > cutoff_limit {
+			eprintln('  > ${final_src}')
+			eprintln(lvtitle)
+			eprintln('      ${slvalue}')
+			eprintln(rvtitle)
+			eprintln('      ${srvalue}')
+		} else {
+			eprintln('   > ${final_src}')
+			eprintln(' ${lvtitle} ${slvalue}')
+			eprintln('${rvtitle} ${srvalue}')
+		}
+	} else {
+		eprintln('    ${final_src}')
+	}
+	if i.has_msg {
+		mut mtitle := '        Message:'
+		mut mvalue := '${i.message}'
+		eprintln('${mtitle} ${mvalue}')
+	}
+	eprintln('')
+
+	unsafe { i.free() }
+}