Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add word-by-word comparison with tolerance and threshold to file diff. #2031

Merged
merged 4 commits into from
May 15, 2023
Merged

Add word-by-word comparison with tolerance and threshold to file diff. #2031

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bb67427
Add word-by-word comparison with tolerance and threshold to file diff.
danielmayer May 12, 2023
8d3a0c1
Merge branch 'develop' into feature_tolerance_for_file_diff
danielmayer May 13, 2023
5462786
Remove some white-space to make CodeFactor happy.
danielmayer May 15, 2023
757cf5d
Merge remote-tracking branch 'github/feature_tolerance_for_file_diff'…
danielmayer May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 110 additions & 9 deletions TestCases/TestCase.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ def print_vals(vals, name="Values"):
"""Print an array of floats."""
print(name + ': ' + ', '.join('{:f}'.format(v) for v in vals))

def is_float(test_string):
try:
float(test_string)
return True
except ValueError:
return False

class TestCase:

Expand Down Expand Up @@ -104,6 +110,8 @@ def __init__(self,tag_in):
self.command = self.Command()
self.timeout = 0
self.tol = 0.0
self.tol_file_percent = 0.0
self.comp_threshold = 0.0

# Options for file-comparison tests
self.reference_file = "of_grad.dat.ref"
Expand Down Expand Up @@ -313,6 +321,7 @@ def run_filediff(self):
print("Output from the failed case:")
subprocess.call(["cat", logfilename])

diff_time_start = datetime.datetime.now()
if not timed_out and passed:
# Compare files
fromfile = self.reference_file
Expand All @@ -325,7 +334,90 @@ def run_filediff(self):
try:
todate = time.ctime(os.stat(tofile).st_mtime)
tolines = open(tofile, 'U').readlines()
diff = list(difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate))

# If file tolerance is set to 0, make regular diff
if self.tol_file_percent == 0.0:
diff = list(difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate))

# Else test word by word with given tolerance
else:

diff = []
max_delta = 0
compare_counter = 0
ignore_counter = 0

# Assert that both files have the same number of lines
if len(fromlines) != len(tolines):
diff = ["ERROR: Number of lines in " + fromfile + " and " + tofile + " differ."]
passed = False

# Loop through all lines
for i_line in range(0, len(fromlines)):

if passed == False: break

# Extract next line and split it
from_line = fromlines[i_line].split()
to_line = tolines[i_line].split()

# Assert that both lines have the same number of words
if len(from_line) != len(to_line):
diff = ["ERROR: Number of words in line " + str(i_line+1) + " differ."]
passed = False
break

# Loop through all words of one line
for i_word in range(0, len(from_line)):

# Extract next word and strip whitespace and commas
from_word = from_line[i_word].strip().strip(',')
to_word = to_line[i_word].strip().strip(',')

# Assert that words are either both numeric or both non-numeric
from_isfloat = is_float(from_word)
to_isfloat = is_float(to_word)
if from_isfloat != to_isfloat:
diff = ["ERROR: File entries '" + from_word + "' and '" + to_word + "' in line " + str(i_line+1) + ", word " + str(i_word+1) + " differ."]
passed = False
delta = 0.0
max_delta = "Not applicable"
break

# Make actual comparison
# Compare floats
if from_isfloat:
try:
# Only do a relative comparison when the threshold is met.
# This is to prevent large relative differences for very small numbers.
if (abs(float(from_word)) > self.comp_threshold):
delta = abs( (float(from_word) - float(to_word)) / float(from_word) ) * 100
compare_counter += 1
else:
delta = 0.0
pcarruscag marked this conversation as resolved.
Show resolved Hide resolved
ignore_counter += 1

max_delta = max(max_delta, delta)

except ZeroDivisionError:
ignore_counter += 1
continue

# Compare non-floats
else:
delta = 0.0
compare_counter += 1
if from_word != to_word:
diff = ["ERROR: File entries '" + from_word + "' and '" + to_word + "' in line " + str(i_line+1) + ", word " + str(i_word+1) + " differ."]
passed = False
max_delta = "Not applicable"
break

if delta > self.tol_file_percent:
diff = ["ERROR: File entries '" + from_word + "' and '" + to_word + "' in line " + str(i_line+1) + ", word " + str(i_word+1) + " differ."]
passed = False
break

except OSError:
print("OS error, most likely from missing reference file:", fromfile)
print("Current working directory contents:")
Expand All @@ -335,10 +427,6 @@ def run_filediff(self):
print("Current working directory contents:")
print(os.listdir("."))





if (diff==[]):
passed=True
else:
Expand All @@ -349,8 +437,21 @@ def run_filediff(self):
else:
passed = False

print('CPU architecture=%s'%self.cpu_arch)
print('test duration: %.2f min'%(running_time/60.0))
# Report results
diff_time_stop = datetime.datetime.now()
diff_time = (diff_time_stop - diff_time_start).microseconds

print('CPU architecture: %s'%self.cpu_arch)
print('Test duration: %.2f min'%(running_time/60.0))
print('Diff duration: %.2f sec'%(diff_time/1e6))
print('Specified tolerance: ' + str(self.tol_file_percent) + '%')
print('Specified threshold: ' + str(self.comp_threshold))

if self.tol_file_percent != 0.0:
print('Compared entries: ' + str(compare_counter))
print('Ignored entries: ' + str(ignore_counter))
print('Maximum difference: ' + str(max_delta) + '%')

print('==================== End Test: %s ====================\n'%self.tag)

sys.stdout.flush()
Expand Down Expand Up @@ -654,7 +755,7 @@ def run_def(self):
except AttributeError: # popen.kill apparently fails on some versions of subprocess... the killall command should take care of things!
pass
timed_out = True
passed = False
passed = False

# Examine the output
f = open(logfilename,'r')
Expand Down Expand Up @@ -688,7 +789,7 @@ def run_def(self):
delta_vals.append( abs(float(data[j])-self.test_vals[j]) )
if delta_vals[j] > self.tol:
exceed_tol = True
passed = False
passed = False
break
else:
iter_missing = True
Expand Down