preludes,builder,cgen: add support for VTEST_RUNNER=tap and -test-runner tap (#12523)

Author: spytheman

cmd/tools/test_if_v_test_system_works.v

@@ -40,24 +40,31 @@ fn cleanup_tdir() {
 	os.rmdir_all(tdir) or { eprintln(err) }
 }
 
+fn create_test(tname string, tcontent string) ?string {
+	tpath := os.join_path(tdir, tname)
+	os.write_file(tpath, tcontent) ?
+	eprintln('>>>>>>>> tpath: $tpath | tcontent: $tcontent')
+	return tpath
+}
+
 fn main() {
 	defer {
 		os.chdir(os.wd_at_startup) or {}
 	}
 	println('> vroot: $vroot | vexe: $vexe | tdir: $tdir')
-	ok_fpath := os.join_path(tdir, 'single_test.v')
-	os.write_file(ok_fpath, 'fn test_ok(){ assert true }') ?
-	check_ok('"$vexe" $ok_fpath')
-	check_ok('"$vexe" test $ok_fpath')
-	fail_fpath := os.join_path(tdir, 'failing_test.v')
-	os.write_file(fail_fpath, 'fn test_fail(){ assert 1 == 2 }') ?
-	check_fail('"$vexe" $fail_fpath')
-	check_fail('"$vexe" test $fail_fpath')
-	check_fail('"$vexe" test $tdir')
+	ok_fpath := create_test('a_single_ok_test.v', 'fn test_ok(){ assert true }') ?
+	check_ok('"$vexe" "$ok_fpath"')
+	check_ok('"$vexe" test "$ok_fpath"')
+	check_ok('"$vexe" test "$tdir"')
+	fail_fpath := create_test('a_single_failing_test.v', 'fn test_fail(){ assert 1 == 2 }') ?
+	check_fail('"$vexe" "$fail_fpath"')
+	check_fail('"$vexe" test "$fail_fpath"')
+	check_fail('"$vexe" test "$tdir"')
 	rel_dir := os.join_path(tdir, rand.ulid())
 	os.mkdir(rel_dir) ?
 	os.chdir(rel_dir) ?
-	check_ok('"$vexe" test ..${os.path_separator + os.base(ok_fpath)}')
+	check_ok('"$vexe" test "..${os.path_separator + os.base(ok_fpath)}"')
+	println('> all done')
 }
 
 fn check_ok(cmd string) string {

cmd/v/help/test.txt

@@ -11,8 +11,23 @@ and then you can perform:
 ... to run all the module's '_test.v' files.
 
 NB 2: V builtin testing requires you to name your files with a _test.v
-suffix, and to name your test functions with test_ prefix. Each 'test_'
-function in a '_test.v' file will be called automatically by the test
-framework. You can use `assert condition` inside each 'test_' function.
-If the asserted condition fails, then v will record that and produce a
-more detailed error message about where the failure was.
+suffix, and to name your test functions with test_ prefix. Each function,
+that starts with 'fn test_', and that is in a '_test.v' file will be called
+automatically by the test framework.
+
+NB 3: You can use `assert condition` inside each 'test_' function. If the
+asserted condition fails, then v will record that, and produce a more detailed
+error message, about where the failure was.
+
+NB 4: Alternative test runners (for IDE integrations):
+You can use several alternative test result formats, using `-test-runner name`,
+or by setting VTEST_RUNNER (the command line option has higher priority).
+
+The names of the available test runners are:
+	`simple` Fastest, does not import additional modules, does no processing.
+	`tap`    Format the output as required by the Test Anything Protocol (TAP).
+	`normal` Supports color output, nicest/most human readable, the default.
+
+You can also implement your own custom test runner, by providing the path to
+your .v file, that implements it to this option. For example, see:
+vlib/v/preludes/test_runner_tap.v .

vlib/builtin/builtin.c.v

@@ -173,8 +173,25 @@ pub fn eprint(s string) {
 	}
 }
 
+pub fn flush_stdout() {
+	$if freestanding {
+		not_implemented := 'flush_stdout is not implemented\n'
+		bare_eprint(not_implemented.str, u64(not_implemented.len))
+	} $else {
+		C.fflush(C.stdout)
+	}
+}
+
+pub fn flush_stderr() {
+	$if freestanding {
+		not_implemented := 'flush_stderr is not implemented\n'
+		bare_eprint(not_implemented.str, u64(not_implemented.len))
+	} $else {
+		C.fflush(C.stderr)
+	}
+}
+
 // print prints a message to stdout. Unlike `println` stdout is not automatically flushed.
-// A call to `flush()` will flush the output buffer to stdout.
 [manualfree]
 pub fn print(s string) {
 	$if android {

vlib/builtin/builtin.v

@@ -41,8 +41,8 @@ fn __as_cast(obj voidptr, obj_type int, expected_type int) voidptr {
 	return obj
 }
 
-// VAssertMetaInfo is used during assertions. An instance of it
-// is filled in by compile time generated code, when an assertion fails.
+// VAssertMetaInfo is used during assertions. An instance of it is filled in by
+// compile time generated code, when an assertion fails.
 pub struct VAssertMetaInfo {
 pub:
 	fpath   string // the source file path of the assertion
@@ -56,9 +56,8 @@ pub:
 	rvalue  string // the stringified *actual value* of the right side of a failed assertion
 }
 
-// free is used to free the memory occupied by the assertion meta data.
-// It is called by cb_assertion_failed, and cb_assertion_ok in the preludes,
-// once they are done with reporting/formatting the meta data.
+// free frees the memory occupied by the assertion meta data. It is called automatically by
+// the code, that V's test framework generates, after all other callbacks have been called.
 [manualfree; unsafe]
 pub fn (ami &VAssertMetaInfo) free() {
 	unsafe {

vlib/builtin/map_test.v

@@ -1,8 +1,6 @@
 import rand
 
-const (
-	strings = unique_strings(20000, 10)
-)
+const strings = unique_strings(7000, 10)
 
 fn unique_strings(arr_len int, str_len int) []string {
 	mut arr := []string{cap: arr_len}
@@ -448,7 +446,7 @@ fn test_map_in() {
 		'Foo': 'bar'
 	}
 	if 'foo'.capitalize() in m {
-		println('ok')
+		assert true
 	} else {
 		assert false
 	}

vlib/builtin/sorting_test.v

@@ -7,14 +7,14 @@ const (
 fn test_sorting_simple() {
 	mut a := unsorted.clone()
 	a.sort()
-	eprintln(' a: $a')
+	println(' a: $a')
 	assert a == sorted_asc
 }
 
 fn test_sorting_with_condition_expression() {
 	mut a := unsorted.clone()
 	a.sort(a > b)
-	eprintln(' a: $a')
+	println(' a: $a')
 	assert a == sorted_desc
 }
 
@@ -44,25 +44,25 @@ fn mysort(mut a []int) {
 fn test_sorting_by_passing_a_mut_array_to_a_function() {
 	mut a := unsorted.clone()
 	mysort(mut a)
-	eprintln(' a: $a')
+	println(' a: $a')
 	assert a == sorted_asc
 }
 
 /*
 fn test_sorting_by_passing_an_anonymous_sorting_function() {
 	mut a := unsorted
 	a.sort(fn(a &int, b &int) int {	return *b - *a })
-	eprintln(' a: $a')
+	println(' a: $a')
 	assert a == sort_desc
 }
 */
 fn test_sorting_u64s() {
 	mut a := [u64(3), 2, 1, 9, 0, 8]
 	a.sort()
-	eprintln(' a: $a')
+	println(' a: $a')
 	assert a == [u64(0), 1, 2, 3, 8, 9]
 	a.sort(a > b)
-	eprintln(' a: $a')
+	println(' a: $a')
 	assert a == [u64(9), 8, 3, 2, 1, 0]
 }
 

vlib/builtin/string.v

@@ -1444,6 +1444,7 @@ pub fn (s &string) free() {
 		return
 	}
 	unsafe {
+		// C.printf(c's: %x %s\n', s.str, s.str)
 		free(s.str)
 	}
 	s.is_lit = -98761234
@@ -1712,8 +1713,8 @@ pub fn (s string) strip_margin() string {
 pub fn (s string) strip_margin_custom(del byte) string {
 	mut sep := del
 	if sep.is_space() {
-		eprintln('Warning: `strip_margin` cannot use white-space as a delimiter')
-		eprintln('    Defaulting to `|`')
+		println('Warning: `strip_margin` cannot use white-space as a delimiter')
+		println('    Defaulting to `|`')
 		sep = `|`
 	}
 	// don't know how much space the resulting string will be, but the max it

vlib/v/ast/table.v

@@ -91,6 +91,7 @@ pub:
 	is_keep_alive   bool // passed memory must not be freed (by GC) before function returns
 	no_body         bool // a pure declaration like `fn abc(x int)`; used in .vh files, C./JS. fns.
 	mod             string
+	file            string
 	file_mode       Language
 	pos             token.Position
 	return_type_pos token.Position

vlib/v/builder/compile.v

@@ -271,7 +271,24 @@ pub fn (v &Builder) get_user_files() []string {
 		user_files << os.join_path(preludes_path, 'live_shared.v')
 	}
 	if v.pref.is_test {
-		user_files << os.join_path(preludes_path, 'tests_assertions.v')
+		user_files << os.join_path(preludes_path, 'test_runner.v')
+		//
+		mut v_test_runner_prelude := os.getenv('VTEST_RUNNER')
+		if v.pref.test_runner != '' {
+			v_test_runner_prelude = v.pref.test_runner
+		}
+		if v_test_runner_prelude == '' {
+			v_test_runner_prelude = 'normal'
+		}
+		if !v_test_runner_prelude.contains('/') && !v_test_runner_prelude.contains('\\')
+			&& !v_test_runner_prelude.ends_with('.v') {
+			v_test_runner_prelude = os.join_path(preludes_path, 'test_runner_${v_test_runner_prelude}.v')
+		}
+		if !os.is_file(v_test_runner_prelude) || !os.is_readable(v_test_runner_prelude) {
+			eprintln('test runner error: File $v_test_runner_prelude should be readable.')
+			verror('supported test runners are: tap, json, simple, normal')
+		}
+		user_files << v_test_runner_prelude
 	}
 	if v.pref.is_test && v.pref.is_stats {
 		user_files << os.join_path(preludes_path, 'tests_with_stats.v')

vlib/v/checker/checker.v

@@ -1431,7 +1431,9 @@ fn (mut c Checker) fail_if_immutable(expr ast.Expr) (string, token.Position) {
 					}
 				}
 			} else if expr.obj is ast.ConstField && expr.name in c.const_names {
-				c.error('cannot modify constant `$expr.name`', expr.pos)
+				if !c.inside_unsafe {
+					c.error('cannot modify constant `$expr.name`', expr.pos)
+				}
 			}
 		}
 		ast.IndexExpr {