Skip to content
Permalink
Browse files

sanitycheck: report tests that never build/run

Add new option --report-excluded to list all those tests with bad
filtering that never build or run. This option produces accurate results
with --all but can be used with default sanitycheck options to see what
does not run/build in CI for example. (limited coverage).

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
  • Loading branch information...
nashif committed Jun 14, 2019
1 parent ff443c1 commit 49b22d4fc38f2047d55d22b090fbde0a93de0fae
Showing with 32 additions and 3 deletions.
  1. +32 −3 scripts/sanitycheck
@@ -163,7 +163,6 @@ Most everyday users will run with no arguments.
"""

import os

if os.name == 'nt':
print("Running sanitycheck on Windows is not supported yet.")
print("https://github.com/zephyrproject-rtos/zephyr/issues/2664")
@@ -2382,6 +2381,7 @@ class TestSuite:
case.create_overlay(case.platform.name)

self.discards = discards

return discards

def add_instances(self, ti_list):
@@ -2800,6 +2800,12 @@ Artificially long but functional example:
parser.add_argument("--list-tags", action="store_true",
help="list all tags in selected tests")

parser.add_argument("--report-excluded",
action="store_true",
help="""List all tests that are never run based on current scope and
coverage. If you are looking for accurate results, run this with
--all, but this will take a while...""")

case_select.add_argument("--list-tests", action="store_true",
help="""List of all sub-test functions recursively found in
all --testcase-root arguments. Note different sub-tests can share
@@ -3298,8 +3304,8 @@ def main():
if options.test:
run_individual_tests = options.test

if options.list_tests or options.sub_test:
cnt = 0

def get_unique_tests(ts):
unq = []
run_individual_tests = []
for n,tc in ts.testcases.items():
@@ -3309,6 +3315,12 @@ def main():
run_individual_tests.append(tc.name)
unq.append(c)

return unq

if options.list_tests or options.sub_test:
cnt = 0
unq = get_unique_tests(ts)

if options.sub_test:
if run_individual_tests:
info("Running the following tests:")
@@ -3333,6 +3345,7 @@ def main():
else:
discards = ts.apply_filters()


if options.discard_report:
ts.discard_report(options.discard_report)

@@ -3373,6 +3386,22 @@ def main():
ts.instances = OrderedDict(sorted(ts.instances.items(),
key=cmp_to_key(native_and_unit_first)))



if options.report_excluded:
all_tests = set(get_unique_tests(ts))
to_be_run = set()
for i,p in ts.instances.items():
to_be_run.update(p.test.cases)

if (all_tests - to_be_run):
print("Tests that never build or run:")
for not_run in (all_tests - to_be_run):
print("- {}".format(not_run))

return


if options.save_tests:
ts.save_tests(options.save_tests)
return

0 comments on commit 49b22d4

Please sign in to comment.
You can’t perform that action at this time.