cgen,pref: add -coverage support + vcover tool

cmd/tools/vbuild-tools.v

@@ -11,7 +11,7 @@ import v.util
 // should be compiled (v folder).
 // To implement that, these folders are initially skipped, then added
 // as a whole *after the testing.prepare_test_session call*.
-const tools_in_subfolders = ['vast', 'vcreate', 'vdoc', 'vpm', 'vsymlink', 'vvet', 'vwhere']
+const tools_in_subfolders = ['vast', 'vcreate', 'vdoc', 'vpm', 'vsymlink', 'vvet', 'vwhere', 'vcover']
 
 // non_packaged_tools are tools that should not be packaged with
 // prebuild versions of V, to keep the size smaller.

cmd/tools/vcover/cover_test.v

@@ -0,0 +1,104 @@
+import os
+
+const vexe = @VEXE
+const vroot = os.dir(vexe)
+const tfolder = os.join_path(os.vtmp_dir(), 'cover_test')
+
+fn testsuite_begin() {
+	os.setenv('VCOLORS', 'never', true)
+	os.chdir(vroot)!
+	os.rmdir_all(tfolder) or {}
+	os.mkdir(tfolder) or {}
+}
+
+fn testsuite_end() {
+	os.rmdir_all(tfolder) or {}
+}
+
+fn test_help() {
+	res := os.execute('${os.quoted_path(vexe)} cover -h')
+	assert res.exit_code == 0
+	assert res.output.contains('Usage: v cover')
+	assert res.output.contains('Description: Analyze & make reports')
+	assert res.output.contains('Options:')
+	assert res.output.contains('-h, --help                Show this help text.')
+	assert res.output.contains('-v, --verbose             Be more verbose while processing the coverages.')
+	assert res.output.contains('-H, --hotspots            Show most frequently executed covered lines.')
+	assert res.output.contains('-P, --percentages         Show coverage percentage per file.')
+	assert res.output.contains('-S, --show_test_files     Show `_test.v` files as well (normally filtered).')
+	assert res.output.contains('-A, --absolute            Use absolute paths for all files')
+}
+
+fn np(path string) string {
+	return path.replace('\\', '/')
+}
+
+fn test_simple() {
+	t1 := np(os.join_path(tfolder, 't1'))
+	t2 := np(os.join_path(tfolder, 't2'))
+	t3 := np(os.join_path(tfolder, 't3'))
+	assert !os.exists(t1), t1
+	assert !os.exists(t2), t2
+	assert !os.exists(t3), t3
+
+	r1 := os.execute('${os.quoted_path(vexe)} -coverage ${os.quoted_path(t1)} cmd/tools/vcover/testdata/simple/t1_test.v')
+	assert r1.exit_code == 0, r1.str()
+	assert r1.output.trim_space() == '10', r1.str()
+	assert os.exists(t1), t1
+	cmd := '${os.quoted_path(vexe)} cover ${os.quoted_path(t1)} --filter vcover/testdata/simple/'
+	filter1 := os.execute(cmd)
+	assert filter1.exit_code == 0, filter1.output
+	assert filter1.output.contains('cmd/tools/vcover/testdata/simple/simple.v'), filter1.output
+	assert filter1.output.trim_space().ends_with('|      4 |      9 |  44.44%'), filter1.output
+	hfilter1 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t1)} --filter vcover/testdata/simple/ -H -P false')
+	assert hfilter1.exit_code == 0, hfilter1.output
+	assert !hfilter1.output.contains('%'), hfilter1.output
+	houtput1 := hfilter1.output.trim_space().split_into_lines()
+	zeros1 := houtput1.filter(it.starts_with('0 '))
+	nzeros1 := houtput1.filter(!it.starts_with('0 '))
+	assert zeros1.len > 0
+	assert zeros1.any(it.contains('simple.v:12')), zeros1.str()
+	assert zeros1.any(it.contains('simple.v:14')), zeros1.str()
+	assert zeros1.any(it.contains('simple.v:17')), zeros1.str()
+	assert zeros1.any(it.contains('simple.v:18')), zeros1.str()
+	assert zeros1.any(it.contains('simple.v:19')), zeros1.str()
+	assert nzeros1.len > 0
+	assert nzeros1.any(it.contains('simple.v:4')), nzeros1.str()
+	assert nzeros1.any(it.contains('simple.v:6')), nzeros1.str()
+	assert nzeros1.any(it.contains('simple.v:8')), nzeros1.str()
+	assert nzeros1.any(it.contains('simple.v:25')), nzeros1.str()
+
+	r2 := os.execute('${os.quoted_path(vexe)} -coverage ${os.quoted_path(t2)} cmd/tools/vcover/testdata/simple/t2_test.v')
+	assert r2.exit_code == 0, r2.str()
+	assert r2.output.trim_space() == '24', r2.str()
+	assert os.exists(t2), t2
+	filter2 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t2)} --filter vcover/testdata/simple')
+	assert filter2.exit_code == 0, filter2.output
+	assert filter2.output.contains('cmd/tools/vcover/testdata/simple/simple.v')
+	assert filter2.output.trim_space().ends_with('|      6 |      9 |  66.67%'), filter2.output
+	hfilter2 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t2)} --filter testdata/simple -H -P false')
+	assert hfilter2.exit_code == 0, hfilter2.output
+	assert !hfilter2.output.contains('%'), hfilter2.output
+	houtput2 := hfilter2.output.trim_space().split_into_lines()
+	zeros2 := houtput2.filter(it.starts_with('0 '))
+	nzeros2 := houtput2.filter(!it.starts_with('0 '))
+	assert zeros2.len > 0
+	assert zeros2.any(it.contains('simple.v:4')), zeros2.str()
+	assert zeros2.any(it.contains('simple.v:6')), zeros2.str()
+	assert zeros2.any(it.contains('simple.v:8')), zeros2.str()
+	assert nzeros2.len > 0
+	assert nzeros2.any(it.contains('simple.v:17')), nzeros2.str()
+	assert nzeros2.any(it.contains('simple.v:18')), nzeros2.str()
+	assert nzeros2.any(it.contains('simple.v:19')), nzeros2.str()
+	assert nzeros2.any(it.contains('simple.v:25')), nzeros2.str()
+
+	// Run both tests. The coverage should be combined and == 100%
+	r3 := os.execute('${os.quoted_path(vexe)} -coverage ${os.quoted_path(t3)} test cmd/tools/vcover/testdata/simple/')
+	assert r3.exit_code == 0, r3.str()
+	assert r3.output.trim_space().contains('Summary for all V _test.v files: 2 passed'), r3.str()
+	assert os.exists(t3), t3
+	filter3 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t3)} --filter simple/')
+	assert filter3.exit_code == 0, filter3.str()
+	assert filter3.output.contains('cmd/tools/vcover/testdata/simple/simple.v'), filter3.str()
+	assert filter3.output.trim_space().match_glob('*cmd/tools/vcover/testdata/simple/simple.v *|      9 |      9 | 100.00%'), filter3.str()
+}

cmd/tools/vcover/data.v

@@ -0,0 +1,34 @@
+// Copyright (c) 2024 Felipe Pena and Delyan Angelov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module main
+
+// vcounter_*.csv files contain counter lines in a CSV format. They can get quite large,
+// for big programs, since they contain all non zero coverage counters.
+// Their names are timestamp (rand.ulid + clock_gettime) based, to minimise the chance that parallel runs
+// will overwrite each other, but without the overhead of additional synchronisation/locks.
+struct CounterLine {
+mut:
+	file string // retrieved based on the loaded meta
+	line int    // retrieved based on the loaded meta
+	//
+	meta  string // A filename in the sibling meta/ folder, should exist, to match the value from this field. The filename is a hash of both the path and the used build options, to facilitate merging coverage data from different builds/programs
+	point int    // The index of a source point. Note that it is not a line number, but an index in the meta data file, keyed by the field `meta` above.
+	hits  u64    // How many times the coverage point was executed. Only counters that are != 0 are recorded.
+}
+
+// Source metadata files in meta/*.txt, contain JSON encoded fields (mappings from v source files to point line numbers).
+// Their names are a result of a hashing function, applied over both the source file name, and the build options.
+// This has several benefits:
+//   a) it makes sure, that the resulting path is normalised
+//   b) the meta data is deduplicated between runs that use the same source files
+//   c) coverage data from different runs can be merged by simply reusing the same -coverage folder,
+//      or by copy/pasting all files from 1 run, to the folder of another.
+struct MetaData {
+	file          string // V source file path
+	fhash         string // fhash is the name of the meta file
+	v_version     string // the V version, used to generate the coverage meta data file
+	build_options string // the build options for the program
+	npoints       int    // the number of stored coverage points
+	points        []int  // the line numbers corresponding to each point
+}

cmd/tools/vcover/main.v

@@ -0,0 +1,208 @@
+// Copyright (c) 2024 Felipe Pena and Delyan Angelov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module main
+
+import os
+import log
+import flag
+import json
+import arrays
+import encoding.csv
+
+// program options, storage etc
+struct Context {
+mut:
+	show_help          bool
+	show_hotspots      bool
+	show_percentages   bool
+	show_test_files    bool
+	use_absolute_paths bool
+	be_verbose         bool
+	filter             string
+	working_folder     string
+	//
+	targets            []string
+	meta               map[string]MetaData // aggregated meta data, read from all .json files
+	all_lines_per_file map[string][]int    // aggregated by load_meta
+	//
+	counters         map[string]u64 // incremented by process_target, based on each .csv file
+	lines_per_file   map[string]map[int]int // incremented by process_target, based on each .csv file
+	processed_points u64
+}
+
+const metadata_extension = '.json'
+const vcounter_glob_pattern = 'vcounters_*.csv'
+
+fn (mut ctx Context) load_meta(folder string) {
+	for omfile in os.walk_ext(folder, metadata_extension) {
+		mfile := omfile.replace('\\', '/')
+		content := os.read_file(mfile) or { '' }
+		meta := os.file_name(mfile.replace(metadata_extension, ''))
+		data := json.decode(MetaData, content) or {
+			log.error('${@METHOD} failed to load ${mfile}')
+			continue
+		}
+		ctx.meta[meta] = data
+		mut lines_per_file := ctx.all_lines_per_file[data.file]
+		lines_per_file << data.points
+		ctx.all_lines_per_file[data.file] = arrays.distinct(lines_per_file)
+	}
+}
+
+fn (mut ctx Context) post_process_all_metas() {
+	ctx.verbose('${@METHOD}')
+	for _, m in ctx.meta {
+		lines_per_file := ctx.all_lines_per_file[m.file]
+		for line in lines_per_file {
+			ctx.counters['${m.file}:${line}:'] = 0
+		}
+	}
+}
+
+fn (mut ctx Context) post_process_all_targets() {
+	ctx.verbose('${@METHOD}')
+	ctx.verbose('ctx.processed_points: ${ctx.processed_points}')
+}
+
+fn (ctx &Context) verbose(msg string) {
+	if ctx.be_verbose {
+		log.info(msg)
+	}
+}
+
+fn (mut ctx Context) process_target(tfile string) ! {
+	ctx.verbose('${@METHOD} ${tfile}')
+	mut reader := csv.new_reader_from_file(tfile)!
+	header := reader.read()!
+	if header != ['meta', 'point', 'hits'] {
+		return error('invalid header in .csv file')
+	}
+	for {
+		row := reader.read() or { break }
+		mut cline := CounterLine{
+			meta: row[0]
+			point: row[1].int()
+			hits: row[2].u64()
+		}
+		m := ctx.meta[cline.meta] or {
+			ctx.verbose('> skipping invalid meta: ${cline.meta} in file: ${cline.file}, csvfile: ${tfile}')
+			continue
+		}
+		cline.file = m.file
+		cline.line = m.points[cline.point] or {
+			ctx.verbose('> skipping invalid point: ${cline.point} in file: ${cline.file}, meta: ${cline.meta}, csvfile: ${tfile}')
+			continue
+		}
+		ctx.counters['${cline.file}:${cline.line}:'] += cline.hits
+		mut lines := ctx.lines_per_file[cline.file].move()
+		lines[cline.line]++
+		ctx.lines_per_file[cline.file] = lines.move()
+		// dump( ctx.lines_per_file[cline.meta][cline.point] )
+		ctx.processed_points++
+	}
+}
+
+fn (mut ctx Context) show_report() ! {
+	filters := ctx.filter.split(',').filter(it != '')
+	if ctx.show_hotspots {
+		for location, hits in ctx.counters {
+			if filters.len > 0 {
+				if !filters.any(location.contains(it)) {
+					continue
+				}
+			}
+			mut final_path := normalize_path(location)
+			if !ctx.use_absolute_paths {
+				final_path = location.all_after_first('${ctx.working_folder}/')
+			}
+			println('${hits:-8} ${final_path}')
+		}
+	}
+	if ctx.show_percentages {
+		for file, lines in ctx.lines_per_file {
+			if !ctx.show_test_files {
+				if file.ends_with('_test.v') || file.ends_with('_test.c.v') {
+					continue
+				}
+			}
+			if filters.len > 0 {
+				if !filters.any(file.contains(it)) {
+					continue
+				}
+			}
+			total_lines := ctx.all_lines_per_file[file].len
+			executed_points := lines.len
+			coverage_percent := 100.0 * f64(executed_points) / f64(total_lines)
+			mut final_path := normalize_path(file)
+			if !ctx.use_absolute_paths {
+				final_path = file.all_after_first('${ctx.working_folder}/')
+			}
+			println('${final_path:-80s} | ${executed_points:6} | ${total_lines:6} | ${coverage_percent:6.2f}%')
+		}
+	}
+}
+
+fn normalize_path(path string) string {
+	return path.replace(os.path_separator, '/')
+}
+
+fn main() {
+	mut ctx := Context{}
+	ctx.working_folder = normalize_path(os.real_path(os.getwd()))
+	mut fp := flag.new_flag_parser(os.args#[1..])
+	fp.application('v cover')
+	fp.version('0.0.2')
+	fp.description('Analyze & make reports, based on cover files, produced by running programs and tests, compiled with `-coverage folder/`')
+	fp.arguments_description('[folder1/ file2 ...]')
+	fp.skip_executable()
+	ctx.show_help = fp.bool('help', `h`, false, 'Show this help text.')
+	ctx.be_verbose = fp.bool('verbose', `v`, false, 'Be more verbose while processing the coverages.')
+	ctx.show_hotspots = fp.bool('hotspots', `H`, false, 'Show most frequently executed covered lines.')
+	ctx.show_percentages = fp.bool('percentages', `P`, true, 'Show coverage percentage per file.')
+	ctx.show_test_files = fp.bool('show_test_files', `S`, false, 'Show `_test.v` files as well (normally filtered).')
+	ctx.use_absolute_paths = fp.bool('absolute', `A`, false, 'Use absolute paths for all files, no matter the current folder. By default, files inside the current folder, are shown with a relative path.')
+	ctx.filter = fp.string('filter', `f`, '', 'Filter only the matching source path patterns.')
+	if ctx.show_help {
+		println(fp.usage())
+		exit(0)
+	}
+	targets := fp.finalize() or {
+		log.error(fp.usage())
+		exit(1)
+	}
+	ctx.verbose('Targets: ${targets}')
+	for t in targets {
+		if !os.exists(t) {
+			log.error('Skipping ${t}, since it does not exist')
+			continue
+		}
+		if os.is_dir(t) {
+			found_counter_files := os.walk_ext(t, '.csv')
+			if found_counter_files.len == 0 {
+				log.error('Skipping ${t}, since there are 0 ${vcounter_glob_pattern} files in it')
+				continue
+			}
+			for counterfile in found_counter_files {
+				ctx.targets << counterfile
+				ctx.load_meta(t)
+			}
+		} else {
+			ctx.targets << t
+			ctx.load_meta(os.dir(t))
+		}
+	}
+	ctx.post_process_all_metas()
+	ctx.verbose('Final ctx.targets.len: ${ctx.targets.len}')
+	ctx.verbose('Final ctx.meta.len: ${ctx.meta.len}')
+	ctx.verbose('Final ctx.filter: ${ctx.filter}')
+	if ctx.targets.len == 0 {
+		log.error('0 cover targets')
+		exit(1)
+	}
+	for t in ctx.targets {
+		ctx.process_target(t)!
+	}
+	ctx.post_process_all_targets()
+	ctx.show_report()!
+}