Skip to content

Commit

Permalink
Merge pull request #652 from Stonemason5040/expect-test-failure-argument
Browse files Browse the repository at this point in the history 
Adding '-x' option
  • Loading branch information
@tomato42
tomato42 committed Apr 29, 2020
2 parents 7f890e3 + ffd7a1b commit 2b72f3f
Show file tree
Hide file tree
Showing 139 changed files with 8,158 additions and 2,044 deletions.
70 changes: 58 additions & 12 deletions scripts/test-SSLv3-padding.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from tlsfuzzer.utils.lists import natural_sort_keys


version = 3
version = 4


def help_msg():
Expand All @@ -35,6 +35,12 @@ def help_msg():
print(" names and not all of them, e.g \"sanity\"")
print(" -e probe-name exclude the probe from the list of the ones run")
print(" may be specified multiple times")
print(" -x probe-name expect the probe to fail. When such probe passes despite being marked like this")
print(" it will be reported in the test summary and the whole script will fail.")
print(" May be specified multiple times.")
print(" -X message expect the `message` substring in exception raised during")
print(" execution of preceding expected failure probe")
print(" usage: [-x probe-name] [-X exception], order is compulsory!")
print(" -n num only run `num` random tests instead of a full set")
print(" (excluding \"sanity\" tests)")
print(" --help this message")
Expand All @@ -46,16 +52,25 @@ def main():
port = 4433
num_limit = None
run_exclude = set()
expected_failures = {}
last_exp_tmp = None

argv = sys.argv[1:]
opts, args = getopt.getopt(argv, "h:p:e:n:", ["help"])
opts, args = getopt.getopt(argv, "h:p:e:x:X:n:", ["help"])
for opt, arg in opts:
if opt == '-h':
host = arg
elif opt == '-p':
port = int(arg)
elif opt == '-e':
run_exclude.add(arg)
elif opt == '-x':
expected_failures[arg] = None
last_exp_tmp = str(arg)
elif opt == '-X':
if not last_exp_tmp:
raise ValueError("-x has to be specified before -X")
expected_failures[last_exp_tmp] = str(arg)
elif opt == '-n':
num_limit = int(arg)
elif opt == '--help':
Expand Down Expand Up @@ -243,7 +258,10 @@ def main():
# run the conversation
good = 0
bad = 0
xfail = 0
xpass = 0
failed = []
xpassed = []
if not num_limit:
num_limit = len(conversations)

Expand All @@ -262,29 +280,57 @@ def main():
runner = Runner(c_test)

res = True
exception = None
try:
runner.run()
except:
except Exception as exp:
exception = exp
print("Error while processing")
print(traceback.format_exc())
res = False

if res:
good += 1
print("OK\n")
if c_name in expected_failures:
if res:
xpass += 1
xpassed.append(c_name)
print("XPASS: expected failure but test passed\n")
else:
if expected_failures[c_name] is not None and \
expected_failures[c_name] not in str(exception):
bad += 1
failed.append(c_name)
print("Expected error message: {0}\n"
.format(expected_failures[c_name]))
else:
xfail += 1
print("OK-expected failure\n")
else:
bad += 1
failed.append(c_name)
if res:
good += 1
print("OK\n")
else:
bad += 1
failed.append(c_name)

print("Script to verify different padding scenarios in SSLv3.")
print("Check if the padding in SSLv3 follows the RFC6101\n")
print("version: {0}\n".format(version))

print("Test end")
print("successful: {0}".format(good))
print("failed: {0}".format(bad))
failed_sorted = sorted(failed, key=natural_sort_keys)
print(" {0}".format('\n '.join(repr(i) for i in failed_sorted)))
print(20 * '=')
print("TOTAL: {0}".format(len(sampled_tests) + 2*len(sanity_tests)))
print("SKIP: {0}".format(len(run_exclude.intersection(conversations.keys()))))
print("PASS: {0}".format(good))
print("XFAIL: {0}".format(xfail))
print("FAIL: {0}".format(bad))
print("XPASS: {0}".format(xpass))
print(20 * '=')
sort = sorted(xpassed ,key=natural_sort_keys)
if len(sort):
print("XPASSED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
sort = sorted(failed, key=natural_sort_keys)
if len(sort):
print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))

if bad > 0:
sys.exit(1)
Expand Down
70 changes: 58 additions & 12 deletions scripts/test-TLSv1_2-rejected-without-TLSv1_2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from tlsfuzzer.utils.lists import natural_sort_keys


version = 2
version = 3


def help_msg():
Expand All @@ -36,6 +36,12 @@ def help_msg():
print(" names and not all of them, e.g \"sanity\"")
print(" -e probe-name exclude the probe from the list of the ones run")
print(" may be specified multiple times")
print(" -x probe-name expect the probe to fail. When such probe passes despite being marked like this")
print(" it will be reported in the test summary and the whole script will fail.")
print(" May be specified multiple times.")
print(" -X message expect the `message` substring in exception raised during")
print(" execution of preceding expected failure probe")
print(" usage: [-x probe-name] [-X exception], order is compulsory!")
print(" -n num only run `num` random tests instead of a full set")
print(" (\"sanity\" tests are always executed)")
print(" --help this message")
Expand All @@ -47,16 +53,25 @@ def main():
port = 4433
num_limit = None
run_exclude = set()
expected_failures = {}
last_exp_tmp = None

argv = sys.argv[1:]
opts, args = getopt.getopt(argv, "h:p:e:n:", ["help"])
opts, args = getopt.getopt(argv, "h:p:e:x:X:n:", ["help"])
for opt, arg in opts:
if opt == '-h':
host = arg
elif opt == '-p':
port = int(arg)
elif opt == '-e':
run_exclude.add(arg)
elif opt == '-x':
expected_failures[arg] = None
last_exp_tmp = str(arg)
elif opt == '-X':
if not last_exp_tmp:
raise ValueError("-x has to be specified before -X")
expected_failures[last_exp_tmp] = str(arg)
elif opt == '-n':
num_limit = int(arg)
elif opt == '--help':
Expand Down Expand Up @@ -148,7 +163,10 @@ def main():
# run the conversation
good = 0
bad = 0
xfail = 0
xpass = 0
failed = []
xpassed = []
if not num_limit:
num_limit = len(conversations)

Expand All @@ -167,29 +185,57 @@ def main():
runner = Runner(c_test)

res = True
exception = None
try:
runner.run()
except Exception:
except Exception as exp:
exception = exp
print("Error while processing")
print(traceback.format_exc())
res = False

if res:
good += 1
print("OK\n")
if c_name in expected_failures:
if res:
xpass += 1
xpassed.append(c_name)
print("XPASS: expected failure but test passed\n")
else:
if expected_failures[c_name] is not None and \
expected_failures[c_name] not in str(exception):
bad += 1
failed.append(c_name)
print("Expected error message: {0}\n"
.format(expected_failures[c_name]))
else:
xfail += 1
print("OK-expected failure\n")
else:
bad += 1
failed.append(c_name)
if res:
good += 1
print("OK\n")
else:
bad += 1
failed.append(c_name)

print("Script to verify that server does not negotiate TLS1.2 ciphers")
print("in TLS1.1 or earlier protocol")
print("version: {0}\n".format(version))

print("Test end")
print("successful: {0}".format(good))
print("failed: {0}".format(bad))
failed_sorted = sorted(failed, key=natural_sort_keys)
print(" {0}".format('\n '.join(repr(i) for i in failed_sorted)))
print(20 * '=')
print("TOTAL: {0}".format(len(sampled_tests) + 2*len(sanity_tests)))
print("SKIP: {0}".format(len(run_exclude.intersection(conversations.keys()))))
print("PASS: {0}".format(good))
print("XFAIL: {0}".format(xfail))
print("FAIL: {0}".format(bad))
print("XPASS: {0}".format(xpass))
print(20 * '=')
sort = sorted(xpassed ,key=natural_sort_keys)
if len(sort):
print("XPASSED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
sort = sorted(failed, key=natural_sort_keys)
if len(sort):
print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))

if bad > 0:
sys.exit(1)
Expand Down
69 changes: 57 additions & 12 deletions scripts/test-aes-gcm-nonces.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ def help_msg():
print(" names and not all of them, e.g \"sanity\"")
print(" -e probe-name exclude the probe from the list of the ones run")
print(" may be specified multiple times")
print(" -x probe-name expect the probe to fail. When such probe passes despite being marked like this")
print(" it will be reported in the test summary and the whole script will fail.")
print(" May be specified multiple times.")
print(" -X message expect the `message` substring in exception raised during")
print(" execution of preceding expected failure probe")
print(" usage: [-x probe-name] [-X exception], order is compulsory!")
print(" -n num only run `num` random tests instead of a full set")
print(" (excluding \"sanity\" tests)")
print(" --help this message")
Expand All @@ -43,16 +49,25 @@ def main():
port = 4433
num_limit = None
run_exclude = set()
expected_failures = {}
last_exp_tmp = None

argv = sys.argv[1:]
opts, args = getopt.getopt(argv, "h:p:e:n:", ["help"])
opts, args = getopt.getopt(argv, "h:p:e:x:X:n:", ["help"])
for opt, arg in opts:
if opt == '-h':
host = arg
elif opt == '-p':
port = int(arg)
elif opt == '-e':
run_exclude.add(arg)
elif opt == '-x':
expected_failures[arg] = None
last_exp_tmp = str(arg)
elif opt == '-X':
if not last_exp_tmp:
raise ValueError("-x has to be specified before -X")
expected_failures[last_exp_tmp] = str(arg)
elif opt == '-n':
num_limit = int(arg)
elif opt == '--help':
Expand Down Expand Up @@ -144,7 +159,10 @@ def main():

good = 0
bad = 0
xfail = 0
xpass = 0
failed = []
xpassed = []
if not num_limit:
num_limit = len(conversations)

Expand All @@ -163,19 +181,37 @@ def main():
runner = Runner(c_test)

res = True
exception = None
try:
runner.run()
except:
except Exception as exp:
exception = exp
print("Error while processing")
print(traceback.format_exc())
res = False

if res:
good += 1
print("OK\n")
if c_name in expected_failures:
if res:
xpass += 1
xpassed.append(c_name)
print("XPASS: expected failure but test passed\n")
else:
if expected_failures[c_name] is not None and \
expected_failures[c_name] not in str(exception):
bad += 1
failed.append(c_name)
print("Expected error message: {0}\n"
.format(expected_failures[c_name]))
else:
xfail += 1
print("OK-expected failure\n")
else:
bad += 1
failed.append(c_name)
if res:
good += 1
print("OK\n")
else:
bad += 1
failed.append(c_name)

print("aes-128-gcm Nonce monotonicity...")
if len(nonces) < 2:
Expand Down Expand Up @@ -207,12 +243,21 @@ def main():
print("OK\n")
good += 1


print("Test end")
print("successful: {0}".format(good))
print("failed: {0}".format(bad))
failed_sorted = sorted(failed, key=natural_sort_keys)
print(" {0}".format('\n '.join(repr(i) for i in failed_sorted)))
print(20 * '=')
print("TOTAL: {0}".format(len(sampled_tests) + 2*len(sanity_tests)))
print("SKIP: {0}".format(len(run_exclude.intersection(conversations.keys()))))
print("PASS: {0}".format(good))
print("XFAIL: {0}".format(xfail))
print("FAIL: {0}".format(bad))
print("XPASS: {0}".format(xpass))
print(20 * '=')
sort = sorted(xpassed ,key=natural_sort_keys)
if len(sort):
print("XPASSED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
sort = sorted(failed, key=natural_sort_keys)
if len(sort):
print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))

if bad > 0:
sys.exit(1)
Expand Down

0 comments on commit 2b72f3f

Please sign in to comment.