Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cgen,pref: add -coverage support + vcover tool #21154

Merged
merged 42 commits into from
May 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
5cad749
fix
felipensp Mar 31, 2024
c255914
fix
felipensp Apr 1, 2024
f9ed550
improvement
felipensp Apr 1, 2024
a1c524c
fix
felipensp Apr 1, 2024
68bd90d
improvement
felipensp Apr 1, 2024
f05811a
improve
felipensp May 13, 2024
322d2f4
Merge remote-tracking branch 'origin/master' into v_coverage
felipensp May 13, 2024
5a8ecb8
fix merge
felipensp May 13, 2024
8344d02
Merge remote-tracking branch 'origin/master' into v_coverage
felipensp May 23, 2024
0adcd12
wip
felipensp May 23, 2024
b5d9fa6
wip
felipensp May 24, 2024
cf6bb6f
fix
felipensp May 24, 2024
0f4f4a7
Merge remote-tracking branch 'origin/master' into v_coverage
felipensp May 24, 2024
c223eb9
wip
felipensp May 24, 2024
791d7ba
wip
felipensp May 24, 2024
a561426
wip
felipensp May 24, 2024
01ed4d7
improve
felipensp May 25, 2024
1096fb5
fix
felipensp May 25, 2024
c80b978
added -filter
felipensp May 25, 2024
73c3878
ignore test funcs
felipensp May 25, 2024
0d6665f
improve filtering
felipensp May 25, 2024
d16866f
change format to csv, add more details, add cmd/tools/vcover/testdata…
spytheman May 25, 2024
2f4dbb1
make test runs of cmd/tools/vcover/testdata/example1/ silent for now
spytheman May 25, 2024
622ed4a
cleanup output format for both the csv counters, and the json meta da…
spytheman May 25, 2024
def6f9b
add the build options to the .csv file too, as a comment (for easier …
spytheman May 25, 2024
44b0c08
move condition_test.v to a separate folder example2/
spytheman May 25, 2024
0b03e87
fix `# build_options` in cgen for `v -cc clang -cstrict -gc none -cov…
spytheman May 25, 2024
729ecc9
escape the paths and build options involved in coverage reports, when…
spytheman May 25, 2024
f08eafb
update copyright
spytheman May 25, 2024
7f9c0d6
draft of `v cover` working with the updated .csv and .json coverage data
spytheman May 25, 2024
f450a55
Support relative locations in the --hotspots listing too
spytheman May 26, 2024
19229fa
ci: fix `v build-tools`, since now `v cover` has multiple .v files.
spytheman May 26, 2024
07499c8
ci: fix `./v -autofree -o v2 cmd/v` in the misc-tooling job
spytheman May 26, 2024
bceb116
add more tests
spytheman May 26, 2024
26f1b65
fix `v test cmd/tools/`
spytheman May 26, 2024
189d3ca
ci: fix `v -os windows -coverage folder file.v` failure, use GetTickC…
spytheman May 26, 2024
dbfb62e
ci: use just os.walk_ext/2, instead of os.glob/1 (the windows impleme…
spytheman May 26, 2024
adf49d2
fix
felipensp May 26, 2024
ec57f10
ci: fix windows failure (unescaped path in the generated source)
spytheman May 26, 2024
a7c0570
ci: normalise paths on windows to the unix ones for uniformity and co…
spytheman May 26, 2024
2f76f7b
Merge branch 'master' into v_coverage
spytheman May 27, 2024
9f34dec
fix case where assert breaks
felipensp May 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/tools/vbuild-tools.v
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import v.util
// should be compiled (v folder).
// To implement that, these folders are initially skipped, then added
// as a whole *after the testing.prepare_test_session call*.
const tools_in_subfolders = ['vast', 'vcreate', 'vdoc', 'vpm', 'vsymlink', 'vvet', 'vwhere']
const tools_in_subfolders = ['vast', 'vcreate', 'vdoc', 'vpm', 'vsymlink', 'vvet', 'vwhere', 'vcover']

// non_packaged_tools are tools that should not be packaged with
// prebuild versions of V, to keep the size smaller.
Expand Down
104 changes: 104 additions & 0 deletions cmd/tools/vcover/cover_test.v
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import os

const vexe = @VEXE
const vroot = os.dir(vexe)
const tfolder = os.join_path(os.vtmp_dir(), 'cover_test')

fn testsuite_begin() {
os.setenv('VCOLORS', 'never', true)
os.chdir(vroot)!
os.rmdir_all(tfolder) or {}
os.mkdir(tfolder) or {}
}

fn testsuite_end() {
os.rmdir_all(tfolder) or {}
}

fn test_help() {
res := os.execute('${os.quoted_path(vexe)} cover -h')
assert res.exit_code == 0
assert res.output.contains('Usage: v cover')
assert res.output.contains('Description: Analyze & make reports')
assert res.output.contains('Options:')
assert res.output.contains('-h, --help Show this help text.')
assert res.output.contains('-v, --verbose Be more verbose while processing the coverages.')
assert res.output.contains('-H, --hotspots Show most frequently executed covered lines.')
assert res.output.contains('-P, --percentages Show coverage percentage per file.')
assert res.output.contains('-S, --show_test_files Show `_test.v` files as well (normally filtered).')
assert res.output.contains('-A, --absolute Use absolute paths for all files')
}

fn np(path string) string {
return path.replace('\\', '/')
}

fn test_simple() {
t1 := np(os.join_path(tfolder, 't1'))
t2 := np(os.join_path(tfolder, 't2'))
t3 := np(os.join_path(tfolder, 't3'))
assert !os.exists(t1), t1
assert !os.exists(t2), t2
assert !os.exists(t3), t3

r1 := os.execute('${os.quoted_path(vexe)} -coverage ${os.quoted_path(t1)} cmd/tools/vcover/testdata/simple/t1_test.v')
assert r1.exit_code == 0, r1.str()
assert r1.output.trim_space() == '10', r1.str()
assert os.exists(t1), t1
cmd := '${os.quoted_path(vexe)} cover ${os.quoted_path(t1)} --filter vcover/testdata/simple/'
filter1 := os.execute(cmd)
assert filter1.exit_code == 0, filter1.output
assert filter1.output.contains('cmd/tools/vcover/testdata/simple/simple.v'), filter1.output
assert filter1.output.trim_space().ends_with('| 4 | 9 | 44.44%'), filter1.output
hfilter1 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t1)} --filter vcover/testdata/simple/ -H -P false')
assert hfilter1.exit_code == 0, hfilter1.output
assert !hfilter1.output.contains('%'), hfilter1.output
houtput1 := hfilter1.output.trim_space().split_into_lines()
zeros1 := houtput1.filter(it.starts_with('0 '))
nzeros1 := houtput1.filter(!it.starts_with('0 '))
assert zeros1.len > 0
assert zeros1.any(it.contains('simple.v:12')), zeros1.str()
assert zeros1.any(it.contains('simple.v:14')), zeros1.str()
assert zeros1.any(it.contains('simple.v:17')), zeros1.str()
assert zeros1.any(it.contains('simple.v:18')), zeros1.str()
assert zeros1.any(it.contains('simple.v:19')), zeros1.str()
assert nzeros1.len > 0
assert nzeros1.any(it.contains('simple.v:4')), nzeros1.str()
assert nzeros1.any(it.contains('simple.v:6')), nzeros1.str()
assert nzeros1.any(it.contains('simple.v:8')), nzeros1.str()
assert nzeros1.any(it.contains('simple.v:25')), nzeros1.str()

r2 := os.execute('${os.quoted_path(vexe)} -coverage ${os.quoted_path(t2)} cmd/tools/vcover/testdata/simple/t2_test.v')
assert r2.exit_code == 0, r2.str()
assert r2.output.trim_space() == '24', r2.str()
assert os.exists(t2), t2
filter2 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t2)} --filter vcover/testdata/simple')
assert filter2.exit_code == 0, filter2.output
assert filter2.output.contains('cmd/tools/vcover/testdata/simple/simple.v')
assert filter2.output.trim_space().ends_with('| 6 | 9 | 66.67%'), filter2.output
hfilter2 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t2)} --filter testdata/simple -H -P false')
assert hfilter2.exit_code == 0, hfilter2.output
assert !hfilter2.output.contains('%'), hfilter2.output
houtput2 := hfilter2.output.trim_space().split_into_lines()
zeros2 := houtput2.filter(it.starts_with('0 '))
nzeros2 := houtput2.filter(!it.starts_with('0 '))
assert zeros2.len > 0
assert zeros2.any(it.contains('simple.v:4')), zeros2.str()
assert zeros2.any(it.contains('simple.v:6')), zeros2.str()
assert zeros2.any(it.contains('simple.v:8')), zeros2.str()
assert nzeros2.len > 0
assert nzeros2.any(it.contains('simple.v:17')), nzeros2.str()
assert nzeros2.any(it.contains('simple.v:18')), nzeros2.str()
assert nzeros2.any(it.contains('simple.v:19')), nzeros2.str()
assert nzeros2.any(it.contains('simple.v:25')), nzeros2.str()

// Run both tests. The coverage should be combined and == 100%
r3 := os.execute('${os.quoted_path(vexe)} -coverage ${os.quoted_path(t3)} test cmd/tools/vcover/testdata/simple/')
assert r3.exit_code == 0, r3.str()
assert r3.output.trim_space().contains('Summary for all V _test.v files: 2 passed'), r3.str()
assert os.exists(t3), t3
filter3 := os.execute('${os.quoted_path(vexe)} cover ${os.quoted_path(t3)} --filter simple/')
assert filter3.exit_code == 0, filter3.str()
assert filter3.output.contains('cmd/tools/vcover/testdata/simple/simple.v'), filter3.str()
assert filter3.output.trim_space().match_glob('*cmd/tools/vcover/testdata/simple/simple.v *| 9 | 9 | 100.00%'), filter3.str()
}
34 changes: 34 additions & 0 deletions cmd/tools/vcover/data.v
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright (c) 2024 Felipe Pena and Delyan Angelov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module main

// vcounter_*.csv files contain counter lines in a CSV format. They can get quite large,
// for big programs, since they contain all non zero coverage counters.
// Their names are timestamp (rand.ulid + clock_gettime) based, to minimise the chance that parallel runs
// will overwrite each other, but without the overhead of additional synchronisation/locks.
struct CounterLine {
mut:
file string // retrieved based on the loaded meta
line int // retrieved based on the loaded meta
//
meta string // A filename in the sibling meta/ folder, should exist, to match the value from this field. The filename is a hash of both the path and the used build options, to facilitate merging coverage data from different builds/programs
point int // The index of a source point. Note that it is not a line number, but an index in the meta data file, keyed by the field `meta` above.
hits u64 // How many times the coverage point was executed. Only counters that are != 0 are recorded.
}

// Source metadata files in meta/*.txt, contain JSON encoded fields (mappings from v source files to point line numbers).
// Their names are a result of a hashing function, applied over both the source file name, and the build options.
// This has several benefits:
// a) it makes sure, that the resulting path is normalised
// b) the meta data is deduplicated between runs that use the same source files
// c) coverage data from different runs can be merged by simply reusing the same -coverage folder,
// or by copy/pasting all files from 1 run, to the folder of another.
struct MetaData {
file string // V source file path
fhash string // fhash is the name of the meta file
v_version string // the V version, used to generate the coverage meta data file
build_options string // the build options for the program
npoints int // the number of stored coverage points
points []int // the line numbers corresponding to each point
}
208 changes: 208 additions & 0 deletions cmd/tools/vcover/main.v
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
// Copyright (c) 2024 Felipe Pena and Delyan Angelov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module main

import os
import log
import flag
import json
import arrays
import encoding.csv

// program options, storage etc
struct Context {
mut:
show_help bool
show_hotspots bool
show_percentages bool
show_test_files bool
use_absolute_paths bool
be_verbose bool
filter string
working_folder string
//
targets []string
meta map[string]MetaData // aggregated meta data, read from all .json files
all_lines_per_file map[string][]int // aggregated by load_meta
//
counters map[string]u64 // incremented by process_target, based on each .csv file
lines_per_file map[string]map[int]int // incremented by process_target, based on each .csv file
processed_points u64
}

const metadata_extension = '.json'
const vcounter_glob_pattern = 'vcounters_*.csv'

fn (mut ctx Context) load_meta(folder string) {
for omfile in os.walk_ext(folder, metadata_extension) {
mfile := omfile.replace('\\', '/')
content := os.read_file(mfile) or { '' }
meta := os.file_name(mfile.replace(metadata_extension, ''))
data := json.decode(MetaData, content) or {
log.error('${@METHOD} failed to load ${mfile}')
continue
}
ctx.meta[meta] = data
mut lines_per_file := ctx.all_lines_per_file[data.file]
lines_per_file << data.points
ctx.all_lines_per_file[data.file] = arrays.distinct(lines_per_file)
}
}

fn (mut ctx Context) post_process_all_metas() {
ctx.verbose('${@METHOD}')
for _, m in ctx.meta {
lines_per_file := ctx.all_lines_per_file[m.file]
for line in lines_per_file {
ctx.counters['${m.file}:${line}:'] = 0
}
}
}

fn (mut ctx Context) post_process_all_targets() {
ctx.verbose('${@METHOD}')
ctx.verbose('ctx.processed_points: ${ctx.processed_points}')
}

fn (ctx &Context) verbose(msg string) {
if ctx.be_verbose {
log.info(msg)
}
}

fn (mut ctx Context) process_target(tfile string) ! {
ctx.verbose('${@METHOD} ${tfile}')
mut reader := csv.new_reader_from_file(tfile)!
header := reader.read()!
if header != ['meta', 'point', 'hits'] {
return error('invalid header in .csv file')
}
for {
row := reader.read() or { break }
mut cline := CounterLine{
meta: row[0]
point: row[1].int()
hits: row[2].u64()
}
m := ctx.meta[cline.meta] or {
ctx.verbose('> skipping invalid meta: ${cline.meta} in file: ${cline.file}, csvfile: ${tfile}')
continue
}
cline.file = m.file
cline.line = m.points[cline.point] or {
ctx.verbose('> skipping invalid point: ${cline.point} in file: ${cline.file}, meta: ${cline.meta}, csvfile: ${tfile}')
continue
}
ctx.counters['${cline.file}:${cline.line}:'] += cline.hits
mut lines := ctx.lines_per_file[cline.file].move()
lines[cline.line]++
ctx.lines_per_file[cline.file] = lines.move()
// dump( ctx.lines_per_file[cline.meta][cline.point] )
ctx.processed_points++
}
}

fn (mut ctx Context) show_report() ! {
filters := ctx.filter.split(',').filter(it != '')
if ctx.show_hotspots {
for location, hits in ctx.counters {
if filters.len > 0 {
if !filters.any(location.contains(it)) {
continue
}
}
mut final_path := normalize_path(location)
if !ctx.use_absolute_paths {
final_path = location.all_after_first('${ctx.working_folder}/')
}
println('${hits:-8} ${final_path}')
}
}
if ctx.show_percentages {
for file, lines in ctx.lines_per_file {
if !ctx.show_test_files {
if file.ends_with('_test.v') || file.ends_with('_test.c.v') {
continue
}
}
if filters.len > 0 {
if !filters.any(file.contains(it)) {
continue
}
}
total_lines := ctx.all_lines_per_file[file].len
executed_points := lines.len
coverage_percent := 100.0 * f64(executed_points) / f64(total_lines)
mut final_path := normalize_path(file)
if !ctx.use_absolute_paths {
final_path = file.all_after_first('${ctx.working_folder}/')
}
println('${final_path:-80s} | ${executed_points:6} | ${total_lines:6} | ${coverage_percent:6.2f}%')
}
}
}

fn normalize_path(path string) string {
return path.replace(os.path_separator, '/')
}

fn main() {
mut ctx := Context{}
ctx.working_folder = normalize_path(os.real_path(os.getwd()))
mut fp := flag.new_flag_parser(os.args#[1..])
fp.application('v cover')
fp.version('0.0.2')
fp.description('Analyze & make reports, based on cover files, produced by running programs and tests, compiled with `-coverage folder/`')
fp.arguments_description('[folder1/ file2 ...]')
fp.skip_executable()
ctx.show_help = fp.bool('help', `h`, false, 'Show this help text.')
ctx.be_verbose = fp.bool('verbose', `v`, false, 'Be more verbose while processing the coverages.')
ctx.show_hotspots = fp.bool('hotspots', `H`, false, 'Show most frequently executed covered lines.')
ctx.show_percentages = fp.bool('percentages', `P`, true, 'Show coverage percentage per file.')
ctx.show_test_files = fp.bool('show_test_files', `S`, false, 'Show `_test.v` files as well (normally filtered).')
ctx.use_absolute_paths = fp.bool('absolute', `A`, false, 'Use absolute paths for all files, no matter the current folder. By default, files inside the current folder, are shown with a relative path.')
ctx.filter = fp.string('filter', `f`, '', 'Filter only the matching source path patterns.')
if ctx.show_help {
println(fp.usage())
exit(0)
}
targets := fp.finalize() or {
log.error(fp.usage())
exit(1)
}
ctx.verbose('Targets: ${targets}')
for t in targets {
if !os.exists(t) {
log.error('Skipping ${t}, since it does not exist')
continue
}
if os.is_dir(t) {
found_counter_files := os.walk_ext(t, '.csv')
if found_counter_files.len == 0 {
log.error('Skipping ${t}, since there are 0 ${vcounter_glob_pattern} files in it')
continue
}
for counterfile in found_counter_files {
ctx.targets << counterfile
ctx.load_meta(t)
}
} else {
ctx.targets << t
ctx.load_meta(os.dir(t))
}
}
ctx.post_process_all_metas()
ctx.verbose('Final ctx.targets.len: ${ctx.targets.len}')
ctx.verbose('Final ctx.meta.len: ${ctx.meta.len}')
ctx.verbose('Final ctx.filter: ${ctx.filter}')
if ctx.targets.len == 0 {
log.error('0 cover targets')
exit(1)
}
for t in ctx.targets {
ctx.process_target(t)!
}
ctx.post_process_all_targets()
ctx.show_report()!
}
Loading
Loading