Skip to content

Commit

Permalink
Continued progress toward feature parity with new test-runner
Browse files Browse the repository at this point in the history 
- Expanded test defines to allow for lists of configurations

  These are useful for changing multi-dimensional test configurations
  without leading to extremely large and less useful configuration
  combinations.

- Made warnings more visible durring test parsing

- Add lfs_testbd.h to implicit test includes

- Fixed issue with not closing files in ./scripts/explode_asserts.py

- Add `make test_runner` and `make test_list` build rules for
  convenience
  • Loading branch information
@geky
geky committed Jun 6, 2022
1 parent 5ee4b05 commit 4962829
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 72 deletions.
11 changes: 9 additions & 2 deletions Makefile
View file
Expand Up @@ -119,9 +119,16 @@ test:
test%: tests/test$$(firstword $$(subst \#, ,%)).toml
./scripts/test.py $@ $(TESTFLAGS)

.PHONY: test_runner
test_runner: $(BUILDDIR)runners/test_runner

.PHONY: test_
test_: $(BUILDDIR)runners/test_runner
./scripts/test_.py --runner=$< $(TESTFLAGS_)
test_: test_runner
./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_)

.PHONY: test_list
test_list: test_runner
./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) -l

.PHONY: code
code: $(OBJ)
Expand Down
60 changes: 34 additions & 26 deletions scripts/explode_asserts.py
View file
Expand Up @@ -134,6 +134,15 @@
}
}

def openio(path, mode='r'):
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
else:
return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
else:
return open(path, mode)

def mkdecls(outf, maxwidth=16):
outf.write("#include <stdio.h>\n")
outf.write("#include <stdbool.h>\n")
Expand Down Expand Up @@ -341,32 +350,31 @@ def pstmt(p):


def main(args):
inf = open(args.input, 'r') if args.input else sys.stdin
outf = open(args.output, 'w') if args.output else sys.stdout

lexemes = LEX.copy()
if args.pattern:
lexemes['assert'] = args.pattern
p = Parse(inf, lexemes)

# write extra verbose asserts
mkdecls(outf, maxwidth=args.maxwidth)
if args.input:
outf.write("#line %d \"%s\"\n" % (1, args.input))

# parse and write out stmt at a time
try:
while True:
outf.write(pstmt(p))
if p.accept('sep'):
outf.write(p.m)
else:
break
except ParseFailure as f:
pass

for i in range(p.off, len(p.tokens)):
outf.write(p.tokens[i][1])
with openio(args.input or '-', 'r') as inf:
with openio(args.output or '-', 'w') as outf:
lexemes = LEX.copy()
if args.pattern:
lexemes['assert'] = args.pattern
p = Parse(inf, lexemes)

# write extra verbose asserts
mkdecls(outf, maxwidth=args.maxwidth)
if args.input:
outf.write("#line %d \"%s\"\n" % (1, args.input))

# parse and write out stmt at a time
try:
while True:
outf.write(pstmt(p))
if p.accept('sep'):
outf.write(p.m)
else:
break
except ParseFailure as f:
pass

for i in range(p.off, len(p.tokens)):
outf.write(p.tokens[i][1])

if __name__ == "__main__":
import argparse
Expand Down
100 changes: 56 additions & 44 deletions scripts/test_.py
View file
Expand Up @@ -24,6 +24,7 @@

SUITE_PROLOGUE = """
#include "runners/test_runner.h"
#include "bd/lfs_testbd.h"
#include <stdio.h>
"""
CASE_PROLOGUE = """
Expand Down Expand Up @@ -78,20 +79,30 @@ def __init__(self, config, args={}):
self.valgrind = config.pop('valgrind',
config.pop('suite_valgrind', True))

# figure out defines and the number of resulting permutations
self.defines = {}
for k, v in (
config.pop('suite_defines', {})
| config.pop('defines', {})).items():
if not isinstance(v, list):
v = [v]

self.defines[k] = v

self.permutations = m.prod(len(v) for v in self.defines.values())
# figure out defines and build possible permutations
self.defines = set()
self.permutations = []

suite_defines = config.pop('suite_defines', {})
if not isinstance(suite_defines, list):
suite_defines = [suite_defines]
defines = config.pop('defines', {})
if not isinstance(defines, list):
defines = [defines]

# build possible permutations
for suite_defines_ in suite_defines:
self.defines |= suite_defines_.keys()
for defines_ in defines:
self.defines |= defines_.keys()
self.permutations.extend(map(dict, it.product(*(
[(k, v) for v in (vs if isinstance(vs, list) else [vs])]
for k, vs in sorted(
(suite_defines_ | defines_).items())))))

for k in config.keys():
print('warning: in %s, found unused key %r' % (self.id(), k),
print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
% (self.id(), k),
file=sys.stderr)

def id(self):
Expand Down Expand Up @@ -130,7 +141,7 @@ def __init__(self, path, args={}):
# sort in case toml parsing did not retain order
case_linenos.sort()

cases = config.pop('cases', [])
cases = config.pop('cases')
for (lineno, name), (nlineno, _) in it.zip_longest(
case_linenos, case_linenos[1:],
fillvalue=(float('inf'), None)):
Expand Down Expand Up @@ -179,16 +190,17 @@ def __init__(self, path, args={}):
**case}))

# combine per-case defines
self.defines = sorted(
set.union(*(set(case.defines) for case in self.cases)))
self.defines = set.union(*(
set(case.defines) for case in self.cases))

# combine other per-case things
self.normal = any(case.normal for case in self.cases)
self.reentrant = any(case.reentrant for case in self.cases)
self.valgrind = any(case.valgrind for case in self.cases)

for k in config.keys():
print('warning: in %s, found unused key %r' % (self.id(), k),
print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
% (self.id(), k),
file=sys.stderr)

def id(self):
Expand Down Expand Up @@ -266,7 +278,7 @@ def writeln(s=''):
% (f.lineno+1, args['output']))
f.writeln()

for i, define in enumerate(suite.defines):
for i, define in enumerate(sorted(suite.defines)):
f.writeln('#ifndef %s' % define)
f.writeln('#define %-24s test_define(%d)' % (define, i))
f.writeln('#endif')
Expand All @@ -275,16 +287,13 @@ def writeln(s=''):
for case in suite.cases:
# create case defines
if case.defines:
sorted_defines = sorted(case.defines.items())

f.writeln('const test_define_t *const '
'__test__%s__%s__defines[] = {'
% (suite.name, case.name))
for defines in it.product(*(
[(k, v) for v in vs]
for k, vs in sorted_defines)):
for permutation in case.permutations:
f.writeln(4*' '+'(const test_define_t[]){%s},'
% ', '.join('%s' % v for _, v in defines))
% ', '.join(str(v) for _, v in sorted(
permutation.items())))
f.writeln('};')
f.writeln()

Expand All @@ -293,9 +302,9 @@ def writeln(s=''):
% (suite.name, case.name))
f.writeln(4*' '+'%s,'
% ', '.join(
'%s' % [k for k, _ in sorted_defines].index(k)
str(sorted(case.defines).index(k))
if k in case.defines else '0xff'
for k in suite.defines))
for k in sorted(suite.defines)))
f.writeln('};')
f.writeln()

Expand Down Expand Up @@ -365,7 +374,8 @@ def writeln(s=''):
'TEST_NORMAL' if case.normal else None,
'TEST_REENTRANT' if case.reentrant else None,
'TEST_VALGRIND' if case.valgrind else None])))
f.writeln(4*' '+'.permutations = %d,' % case.permutations)
f.writeln(4*' '+'.permutations = %d,'
% len(case.permutations))
if case.defines:
f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
% (suite.name, case.name))
Expand All @@ -381,12 +391,13 @@ def writeln(s=''):
f.writeln()

# create suite define names
f.writeln('const char *const __test__%s__define_names[] = {'
% suite.name)
for k in suite.defines:
f.writeln(4*' '+'"%s",' % k)
f.writeln('};')
f.writeln()
if suite.defines:
f.writeln('const char *const __test__%s__define_names[] = {'
% suite.name)
for k in sorted(suite.defines):
f.writeln(4*' '+'"%s",' % k)
f.writeln('};')
f.writeln()

# create suite struct
f.writeln('const struct test_suite __test__%s__suite = {'
Expand All @@ -399,8 +410,9 @@ def writeln(s=''):
'TEST_NORMAL' if suite.normal else None,
'TEST_REENTRANT' if suite.reentrant else None,
'TEST_VALGRIND' if suite.valgrind else None])))
f.writeln(4*' '+'.define_names = __test__%s__define_names,'
% suite.name)
if suite.defines:
f.writeln(4*' '+'.define_names = __test__%s__define_names,'
% suite.name)
f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
f.writeln(4*' '+'.cases = (const struct test_case *const []){')
for case in suite.cases:
Expand Down Expand Up @@ -726,7 +738,7 @@ def run_job(runner, start=None, step=None):

if not args.get('verbose'):
sys.stdout.write('\r\x1b[K'
'running \x1b[%dm%s\x1b[m: '
'running \x1b[%dm%s:\x1b[m '
'%d/%d suites, %d/%d cases, %d/%d perms%s '
% (32 if not failures else 31,
name,
Expand Down Expand Up @@ -779,12 +791,12 @@ def run(**args):
expected = 0
passed = 0
failures = []
if args.get('by_suites'):
if args.get('by_cases'):
for type in ['normal', 'reentrant', 'valgrind']:
for suite in expected_suite_perms.keys():
for case in expected_case_perms.keys():
expected_, passed_, failures_, killed = run_stage(
'%s %s' % (type, suite),
runner_ + ['--%s' % type, suite],
'%s %s' % (type, case),
runner_ + ['--%s' % type, case],
**args)
expected += expected_
passed += passed_
Expand All @@ -793,12 +805,12 @@ def run(**args):
break
if (failures and not args.get('keep_going')) or killed:
break
elif args.get('by_cases'):
elif args.get('by_suites'):
for type in ['normal', 'reentrant', 'valgrind']:
for case in expected_case_perms.keys():
for suite in expected_suite_perms.keys():
expected_, passed_, failures_, killed = run_stage(
'%s %s' % (type, case),
runner_ + ['--%s' % type, case],
'%s %s' % (type, suite),
runner_ + ['--%s' % type, suite],
**args)
expected += expected_
passed += passed_
Expand All @@ -821,7 +833,7 @@ def run(**args):

# show summary
print()
print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs'
print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs'
% (32 if not failures else 31,
passed, expected, len(failures), expected,
time.time()-start))
Expand Down

0 comments on commit 4962829

Please sign in to comment.