Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
scripts: Add regression test runner/checker
- Loading branch information
Showing
2 changed files
with
152 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
import argparse | ||
import glob | ||
import sys | ||
import os | ||
import re | ||
import hashlib | ||
|
||
from pathlib import Path | ||
|
||
def compare_frames(path1, path2): | ||
try: | ||
with open(path1, "rb") as f: | ||
hash1 = hashlib.md5(f.read()).digest() | ||
with open(path2, "rb") as f: | ||
hash2 = hashlib.md5(f.read()).digest() | ||
|
||
#print(hash1, hash2) | ||
return hash1 == hash2 | ||
except: | ||
return False | ||
|
||
|
||
def check_regression_test(baselinedir, testdir, name): | ||
#print("Checking '%s'..." % name) | ||
|
||
dir1 = os.path.join(baselinedir, name) | ||
dir2 = os.path.join(testdir, name) | ||
if not os.path.isdir(dir2): | ||
#print("*** %s is missing in test set" % name) | ||
return False | ||
|
||
images = glob.glob(os.path.join(dir1, "frame_*.png")) | ||
for imagepath in images: | ||
imagename = Path(imagepath).name | ||
matches = re.match("frame_([0-9]+).png", imagename) | ||
if matches is None: | ||
continue | ||
|
||
framenum = int(matches[1]) | ||
|
||
path1 = os.path.join(dir1, imagename) | ||
path2 = os.path.join(dir2, imagename) | ||
if not os.path.isfile(path2): | ||
print("--- Frame %u for %s is missing in test set" % (framenum, name)) | ||
return False | ||
|
||
if not compare_frames(path1, path2): | ||
print("*** Difference in frame %u for %s" % (framenum, name)) | ||
return False | ||
|
||
return True | ||
|
||
|
||
def check_regression_tests(baselinedir, testdir): | ||
gamedirs = glob.glob(baselinedir + "/*", recursive=False) | ||
|
||
success = 0 | ||
failure = 0 | ||
|
||
for gamedir in gamedirs: | ||
name = Path(gamedir).name | ||
if check_regression_test(baselinedir, testdir, name): | ||
success += 1 | ||
else: | ||
failure += 1 | ||
|
||
return (failure == 0) | ||
|
||
|
||
if __name__ == "__main__": | ||
parser = argparse.ArgumentParser(description="Check frame dump images for regression tests") | ||
parser.add_argument("-baselinedir", action="store", required=True, help="Directory containing baseline frames to check against") | ||
parser.add_argument("-testdir", action="store", required=True, help="Directory containing frames to check") | ||
|
||
args = parser.parse_args() | ||
|
||
if not check_regression_tests(os.path.realpath(args.baselinedir), os.path.realpath(args.testdir)): | ||
sys.exit(1) | ||
else: | ||
sys.exit(0) | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
import argparse | ||
import glob | ||
import sys | ||
import os | ||
import subprocess | ||
import multiprocessing | ||
from functools import partial | ||
|
||
def is_game_path(path): | ||
idx = path.rfind('.') | ||
if idx < 0: | ||
return False | ||
|
||
extension = path[idx + 1:].strip().lower() | ||
return extension in ["cue", "chd"] | ||
|
||
|
||
def run_regression_test(runner, destdir, dump_interval, frames, gamepath): | ||
args = [runner, | ||
"-renderer", "software", | ||
"-log", "verbose", | ||
"-dumpdir", destdir, | ||
"-dumpinterval", str(dump_interval), | ||
"-frames", str(frames), | ||
"--", gamepath | ||
] | ||
|
||
print("Running '%s'" % (" ".join(args))) | ||
subprocess.run(args) | ||
|
||
|
||
def run_regression_tests(runner, gamedir, destdir, dump_interval, frames, parallel=1): | ||
paths = glob.glob(gamedir + "/*.*", recursive=True) | ||
gamepaths = list(filter(is_game_path, paths)) | ||
|
||
if not os.path.isdir(destdir) and not os.mkdir(destdir): | ||
print("Failed to create directory") | ||
return False | ||
|
||
print("Found %u games" % len(gamepaths)) | ||
|
||
if parallel <= 1: | ||
for game in gamepaths: | ||
run_regression_test(runner, destdir, dump_interval, frames, game) | ||
else: | ||
print("Processing %u games on %u processors" % (len(gamepaths), parallel)) | ||
func = partial(run_regression_test, runner, destdir, dump_interval, frames) | ||
pool = multiprocessing.Pool(parallel) | ||
pool.map(func, gamepaths) | ||
pool.close() | ||
|
||
|
||
return True | ||
|
||
|
||
if __name__ == "__main__": | ||
parser = argparse.ArgumentParser(description="Generate frame dump images for regression tests") | ||
parser.add_argument("-runner", action="store", required=True, help="Path to DuckStation regression test runner") | ||
parser.add_argument("-gamedir", action="store", required=True, help="Directory containing game images") | ||
parser.add_argument("-destdir", action="store", required=True, help="Base directory to dump frames to") | ||
parser.add_argument("-dumpinterval", action="store", type=int, required=True, help="Interval to dump frames at") | ||
parser.add_argument("-frames", action="store", type=int, default=3600, help="Number of frames to run") | ||
parser.add_argument("-parallel", action="store", type=int, default=1, help="Number of proceeses to run") | ||
|
||
args = parser.parse_args() | ||
|
||
if not run_regression_tests(args.runner, os.path.realpath(args.gamedir), os.path.realpath(args.destdir), args.dumpinterval, args.frames, args.parallel): | ||
sys.exit(1) | ||
else: | ||
sys.exit(0) | ||
|