Skip to content

Commit

Permalink
[LIT] Print discovered tests and percentages (#66057) (#69831)
Browse files Browse the repository at this point in the history
This patch adds "nice-to-have" feature in lit.
it prints the total number of discovered tests at the beginning. It is
covenient to see the total number of tests and avoid scrolling up to the
beginning of log.

Further, this patch also prints %ge of tests.

This patch fixes tests pointed by previous attempt of landing this
patch.

Reviewed By: RoboTux, jdenny-ornl

Co-authored-by: Madhur A <madhura@nvidia.com>
  • Loading branch information
madhur13490 and madhur13490 committed Oct 22, 2023
1 parent 00c8da6 commit 0b7ae41
Show file tree
Hide file tree
Showing 58 changed files with 78 additions and 65 deletions.
8 changes: 5 additions & 3 deletions llvm/utils/lit/lit/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ def print_histogram(tests):

def print_results(tests, elapsed, opts):
tests_by_code = {code: [] for code in lit.Test.ResultCode.all_codes()}
total_tests = len(tests)
for test in tests:
tests_by_code[test.result.code].append(test)

Expand All @@ -321,7 +322,7 @@ def print_results(tests, elapsed, opts):
opts.shown_codes,
)

print_summary(tests_by_code, opts.quiet, elapsed)
print_summary(total_tests, tests_by_code, opts.quiet, elapsed)


def print_group(tests, code, shown_codes):
Expand All @@ -336,10 +337,11 @@ def print_group(tests, code, shown_codes):
sys.stdout.write("\n")


def print_summary(tests_by_code, quiet, elapsed):
def print_summary(total_tests, tests_by_code, quiet, elapsed):
if not quiet:
print("\nTesting Time: %.2fs" % elapsed)

print("\nTotal Discovered Tests: %s" % (total_tests))
codes = [c for c in lit.Test.ResultCode.all_codes() if not quiet or c.isFailure]
groups = [(c.label, len(tests_by_code[c])) for c in codes]
groups = [(label, count) for label, count in groups if count]
Expand All @@ -352,4 +354,4 @@ def print_summary(tests_by_code, quiet, elapsed):
for (label, count) in groups:
label = label.ljust(max_label_len)
count = str(count).rjust(max_count_len)
print(" %s: %s" % (label, count))
print(" %s: %s (%.2f%%)" % (label, count, float(count) / total_tests * 100))
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name 'foo %{name}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK: Substitution name '%{name}:' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name '%{}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK: Substitution name '%{foo.bar}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK: Substitution name '%{foo' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK: Substitution name '%{foo bar}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK: Substitution name '%{3}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name '%{foo bar}' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: Substitution name '' is malformed {{.*}}
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
# CHECK: Substitution's continuation is empty
# CHECK-NEXT: in DEFINE: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
# CHECK: Substitution's continuation is empty
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{REDEFINE}}: directive on test line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') from line [[#@LINE-3]] to [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
# CHECK: Substitution's definition does not contain '='
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') from line [[#@LINE-3]] to [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-2]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{RUN}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@

# CHECK: Test has unterminated '{{RUN}}:' directive (with '\') at line [[#@LINE-4]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
# CHECK: Substitution's continuation is empty
# CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-5]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: %{global:greeting}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
# CHECK-NEXT: Existing pattern: %{foo}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: <%{global:inside}>
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-once-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: %{global:prefix}\((.*)\)
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# CHECK-NEXT: Existing pattern: @%{global:suffix}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: <%{global:inside}>
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK-NEXT: Existing pattern: %{global:multiple-once-exact}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# CHECK: ValueError: No substitution for '%{local}' is defined before '{{REDEFINE}}:' directive at line [[#@LINE-3]]
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: %{global:prefix}\((.*)\)
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# CHECK-NEXT: Existing pattern: @%{global:suffix}
# CHECK-NOT: Existing pattern:

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: ValueError: Substitution whose pattern contains '%{foo}' is already defined before '{{DEFINE}}:' directive from line [[#@LINE-4]] to [[#@LINE-3]]

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

# CHECK: Test has no '{{RUN}}:' line

# CHECK: Unresolved: 1
# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,4 @@
; RUN: %{check}
; CHECK:# | %clang_cc1 -verify -fopenmp -fopenmp-version=51 -triple x86_64-unknown-linux-gnu -emit-llvm -o - {{.*}} | FileCheck -check-prefix=NO-SIMD {{.*}}

; CHECK: Passed: 1
; CHECK: Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@
# RUN: %{local:echo}
# CHECK:# | LOCAL: So Long World World

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@
# RUN: echo '%{lines-rel}'
# CHECK:# | [[#@LINE+1]], [[#@LINE-3]], [[#@LINE-12]]

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
2 changes: 1 addition & 1 deletion llvm/utils/lit/tests/Inputs/shtest-define/name-chars.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@
# RUN: echo '%{_foo_bar_}'
# CHECK:# | ok at [[#@LINE - 2]]

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
# CHECK-NON-RECUR:# | %{inner}
# CHECK-RECUR:# | expanded

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
# DEFINE: %{local:echo}=echo LOCAL0: subst
# RUN: %{local:echo}

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
# DEFINE: %{local:echo}=echo LOCAL1: subst
# RUN: %{local:echo}

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
2 changes: 1 addition & 1 deletion llvm/utils/lit/tests/Inputs/shtest-define/value-equals.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
# RUN: echo '%{continue-equals}'
# CHECK:# | FileCheck -input-file=test.txt -implicit-check-not=foobar -check-prefixes=FOO,BAR

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# RUN: echo '%{escape}'
# CHECK:# | {{\\?}}\n {{\\?}}\g<param>

# CHECK: Passed: 1
# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
Original file line number Diff line number Diff line change
Expand Up @@ -132,4 +132,4 @@
# RUN: %{deceptive-continue}
# CHECK:# | x=3

# CHECK:{{ *}}Passed: 1
# CHECK:{{ *}}Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
7 changes: 7 additions & 0 deletions llvm/utils/lit/tests/discovery.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@
# CHECK-BASIC-OUT: top-level-suite :: test-one
# CHECK-BASIC-OUT: top-level-suite :: test-two

# RUN: %{lit} %{inputs}/discovery \
# RUN: -v > %t.out 2> %t.err
# RUN: FileCheck --check-prefix=CHECK-PERCENTAGES-OUT < %/t.out %s
#
# CHECK-PERCENTAGES-OUT: Total Discovered Tests: {{[0-9]*}}
# CHECK-PERCENTAGES-OUT: Passed: {{[0-9]*}} {{\([0-9]*\.[0-9]*%\)}}

# Check discovery when providing the special builtin 'config_map'
# RUN: %{python} %{inputs}/config-map-discovery/driver.py \
# RUN: %{inputs}/config-map-discovery/main-config/lit.cfg \
Expand Down
9 changes: 5 additions & 4 deletions llvm/utils/lit/tests/ignore-fail.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
# CHECK-DAG: XPASS: ignore-fail :: xpass.txt

# CHECK: Testing Time:
# CHECK-NEXT: Expectedly Failed : 1
# CHECK-NEXT: Unresolved : 1
# CHECK-NEXT: Failed : 1
# CHECK-NEXT: Unexpectedly Passed: 1
# CHECK: Total Discovered Tests:
# CHECK-NEXT: Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NOT: {{.}}
5 changes: 3 additions & 2 deletions llvm/utils/lit/tests/shtest-env.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@
# CHECK-NOT: {{^[^#]}}
# CHECK: --

# CHECK: Passed: 4
# CHECK: Failed: 12
# CHECK: Total Discovered Tests: 16
# CHECK: Passed: 4 {{\([0-9]*\.[0-9]*%\)}}
# CHECK: Failed: 12 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NOT: {{.}}
5 changes: 3 additions & 2 deletions llvm/utils/lit/tests/shtest-not.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@
# CHECK: # | Error: 'not --crash' cannot call 'rm'
# CHECK: # error: command failed with exit status: {{.*}}

# CHECK: Passed: 1
# CHECK: Failed: 16
# CHECK: Total Discovered Tests: 17
# CHECK: Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
# CHECK: Failed: 16 {{\([0-9]*\.[0-9]*%\)}}
# CHECK-NOT: {{.}}

0 comments on commit 0b7ae41

Please sign in to comment.