Merge pull request #674 from tomato42/speed-up-tests

Speed up tests
tlsfuzzer · Jun 18, 2020 · 7e3aed4 · 7e3aed4
2 parents 54a1350 + 09be636
commit 7e3aed4
Show file tree

Hide file tree

Showing 11 changed files with 47 additions and 45 deletions.
diff --git a/scripts/test-aesccm.py b/scripts/test-aesccm.py
@@ -43,7 +43,7 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(100)")
     print("                (excluding \"sanity\" tests)")
     print(" -d             negotiate (EC)DHE instead of RSA key exchange")
     print(" --help         this message")
@@ -52,7 +52,7 @@ def help_msg():
 def main():
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 100
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None

diff --git a/scripts/test-certificate-malformed.py b/scripts/test-certificate-malformed.py
@@ -33,7 +33,7 @@
 from tlsfuzzer.utils.lists import natural_sort_keys
 
 
-version = 2
+version = 3
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -44,7 +44,7 @@ def help_msg():
     print("                names and not all of them, e.g \"sanity\"")
     print(" -e probe-name  exclude the probe from the list of the ones run")
     print("                may be specified multiple times")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(1000)")
     print("                (excluding \"sanity\" tests)")
     print(" -x probe-name  expect the probe to fail. When such probe passes despite being marked like this")
     print("                it will be reported in the test summary and the whole script will fail.")
@@ -63,15 +63,15 @@ def main():
     conversations = {}
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 1000
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None
     private_key = None
     cert = None
 
     argv = sys.argv[1:]
-    opts, args = getopt.getopt(argv, "h:p:e:x:X:k:c:", ["help"])
+    opts, args = getopt.getopt(argv, "h:p:e:x:X:k:c:n:", ["help"])
     for opt, arg in opts:
         if opt == '-h':
             host = arg

diff --git a/scripts/test-dhe-no-shared-secret-padding.py b/scripts/test-dhe-no-shared-secret-padding.py
@@ -44,7 +44,7 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(1)")
     print("                (excluding \"sanity\" tests)")
     print(" --min-zeros m  minimal number of zeros that have to be cut from")
     print("                shared secret for test case to be valid,")
@@ -57,7 +57,7 @@ def main():
     """Verify correct DHE shared secret handling."""
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 1
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None

diff --git a/scripts/test-dhe-rsa-key-exchange-signatures.py b/scripts/test-dhe-rsa-key-exchange-signatures.py
@@ -27,7 +27,7 @@
 from tlsfuzzer.helpers import RSA_SIG_ALL
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 2
+version = 4
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -38,7 +38,7 @@ def help_msg():
     print("                names and not all of them, e.g \"sanity\"")
     print(" -e probe-name  exclude the probe from the list of the ones run")
     print("                may be specified multiple times")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(10)")
     print("                (excluding \"sanity\" tests)")
     print(" -x probe-name  expect the probe to fail. When such probe passes despite being marked like this")
     print("                it will be reported in the test summary and the whole script will fail.")
@@ -53,7 +53,7 @@ def main():
     """Test if server supports all common hash algorithms in DHE_RSA kex"""
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 10
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None
@@ -123,6 +123,7 @@ def main():
                                          AlertDescription.close_notify))
     node = node.add_child(ExpectAlert())
     node.next_sibling = ExpectClose()
+    node = node.add_child(ExpectClose())
 
     conversations["sanity"] = conversation
 
@@ -160,6 +161,7 @@ def main():
                                                  AlertDescription.close_notify))
             node = node.add_child(ExpectAlert())
             node.next_sibling = ExpectClose()
+            node = node.add_child(ExpectClose())
 
             conversations[CipherSuite.ietfNames[cipher] + " " + hash_alg
                           + " signature"] = conversation

diff --git a/scripts/test-ecdhe-padded-shared-secret.py b/scripts/test-ecdhe-padded-shared-secret.py
@@ -27,7 +27,7 @@
         SupportedGroupsExtension
 
 
-version = 3
+version = 4
 
 
 def help_msg():
@@ -45,7 +45,7 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(1)")
     print("                (excluding \"sanity\" tests)")
     print(" --min-zeros m  minimal number of zeros that have to be cut from")
     print("                shared secret for test case to be valid,")
@@ -58,7 +58,7 @@ def main():
     """Verify correct ECDHE shared secret handling."""
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 1
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None
@@ -262,6 +262,7 @@ def main():
                     else:
                         xfail += 1
                         print("OK-expected failure\n")
+                break_loop = True
             else:
                 if res:
                     good += 1
@@ -270,12 +271,12 @@ def main():
                         print("Got premaster secret with {0} most significant "
                             "bytes equal to zero."
                             .format(min_zeros))
-                    break_loop = True
+                        break_loop = True
                     print("OK\n")
                 else:
                     bad += 1
                     failed.append(c_name)
-                    break
+                    break_loop = True
             if break_loop:
                 break
 

diff --git a/scripts/test-heartbeat.py b/scripts/test-heartbeat.py
@@ -28,7 +28,7 @@
         HeartbeatExtension
 
 
-version = 3
+version = 4
 
 
 def help_msg():
@@ -46,7 +46,7 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(120)")
     print("                (\"sanity\" tests are always executed)")
     print(" -d             negotiate (EC)DHE instead of RSA key exchange")
     print(" --help         this message")
@@ -66,7 +66,7 @@ def add_dhe_extensions(extensions):
 def main():
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 120
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None

diff --git a/scripts/test-serverhello-random.py b/scripts/test-serverhello-random.py
@@ -26,7 +26,7 @@
 from tlsfuzzer.helpers import uniqueness_check
 
 
-version = 3
+version = 4
 
 
 def help_msg():
@@ -44,7 +44,7 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(4)")
     print("                (excluding \"sanity\" tests)")
     print(" --repeat num   repeat every test num times to collect the values")
     print("                32 by default")
@@ -56,7 +56,7 @@ def main():
     """Test if server provides unique random values in Server Hello."""
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 4
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None

diff --git a/scripts/test-tls13-dhe-shared-secret-padding.py b/scripts/test-tls13-dhe-shared-secret-padding.py
@@ -30,7 +30,7 @@
 """Script to verify that the DH keys are computed correctly."""
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -48,7 +48,7 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(1)")
     print("                (excluding \"sanity\" tests)")
     print(" --min-zeros num number of zeros that need to be found in the")
     print("                shared secret for the test case to be considered")
@@ -59,7 +59,7 @@ def help_msg():
 def main():
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 1
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None

diff --git a/scripts/test-tls13-version-negotiation.py b/scripts/test-tls13-version-negotiation.py
@@ -29,7 +29,7 @@
 from tlsfuzzer.helpers import key_share_gen, RSA_SIG_ALL, key_share_ext_gen
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -47,15 +47,15 @@ def help_msg():
     print(" -X message     expect the `message` substring in exception raised during")
     print("                execution of preceding expected failure probe")
     print("                usage: [-x probe-name] [-X exception], order is compulsory!")
-    print(" -n num         run 'num' or all(if 0) tests instead of default(all)")
+    print(" -n num         run 'num' or all(if 0) tests instead of default(200)")
     print("                (excluding \"sanity\" tests)")
     print(" --help         this message")
 
 
 def main():
     host = "localhost"
     port = 4433
-    num_limit = None
+    num_limit = 200
     run_exclude = set()
     expected_failures = {}
     last_exp_tmp = None

diff --git a/tests/tlslite-ng-random-subset.json b/tests/tlslite-ng-random-subset.json
@@ -50,10 +50,9 @@
          {"name" : "test-client-compatibility.py",
           "arguments" : ["-x", "18: IE 6 on XP",
                          "-x", "52: YandexBot 3.0 on unknown",
-                         "-x", "100: IE 6 on XP",
-                         "-n", "50"
+                         "-x", "100: IE 6 on XP"
           ],
-          "comment": "SSLv3 is not supported by tlslite-ng by default, there are a lot of test cases"
+          "comment": "SSLv3 is not supported by tlslite-ng by default"
          },
          {"name" : "test-client-compatibility.py",
           "arguments" : ["-d", "sanity"],
@@ -71,8 +70,9 @@
                          "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello"],
           "comment": "SSLv3 is not supported by tlslite-ng by default"},
          {"name" : "test-dhe-no-shared-secret-padding.py",
-          "arguments" : ["Protocol (3, 3)"],
-          "comment": "test is non-deterministic and long, run just one case"},
+          "arguments" : ["-e", "Protocol (3, 0)",
+                         "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello"],
+          "comment": "SSLv3 is not supported by tlslite-ng by default"},
          {"name" : "test-dhe-rsa-key-exchange.py"},
          {"name" : "test-dhe-rsa-key-exchange-signatures.py"},
          {"name" : "test-dhe-rsa-key-exchange-with-bad-messages.py"},
@@ -135,11 +135,10 @@
          {"name" : "test-fuzzed-padding.py"},
          {"name" : "test-fuzzed-padding.py",
           "arguments" : ["-d"]},
-         {"name" : "test-fuzzed-plaintext.py",
-          "comment" : "--random to speed up initial test case creation"},
+         {"name" : "test-fuzzed-plaintext.py"},
          {"name" : "test-fuzzed-plaintext.py",
           "arguments" : ["-d"],
-          "comment" : "-d to test DHE key exchange, which is slower, so lower -n value"},
+          "comment" : "-d to test DHE key exchange"},
          {"name" : "test-fuzzed-plaintext.py",
           "arguments" : ["-C", "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384"]},
          {"name" : "test-heartbeat.py"},
@@ -220,8 +219,9 @@
                          "-c", "tests/serverX509Cert.pem"]},
          {"name" : "test-resumption-with-wrong-ciphers.py"},
          {"name" : "test-serverhello-random.py",
-          "arguments" : ["-e", "Protocol (3, 0)",
-                         "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello"],
+          "arguments" : ["-x", "Protocol (3, 0)", "-X", "protocol_version",
+                         "-x", "Protocol (3, 0) in SSLv2 compatible ClientHello",
+                         "-X", "protocol_version"],
           "comment" : "tlslite-ng does not support SSLv3 by default"},
          {"name" : "test-sessionID-resumption.py"},
          {"name" : "test-sig-algs.py",
@@ -234,13 +234,13 @@
           "arguments" : ["--alerts", "0",
                          "--alert-level", "warning",
                          "--alert-description", "close_notify"],
-          "comment" : "tlslite-ng does not allow client to send warning alerts, thus the number of allowed alerts (-n) is 0"
+          "comment" : "tlslite-ng does not allow client to send warning alerts, thus the number of allowed alerts (--alerts) is 0"
          },
          {"name" : "test-ssl-death-alert.py",
           "arguments" : ["-d", "--alerts", "0",
                          "--alert-level", "warning",
                          "--alert-description", "close_notify"],
-          "comment" : "tlslite-ng does not allow client to send warning alerts, thus the number of allowed alerts (-n) is 0"
+          "comment" : "tlslite-ng does not allow client to send warning alerts, thus the number of allowed alerts (--alerts) is 0"
          },
          {"name" : "test-sslv2-connection.py"},
          {"name" : "test-sslv2-force-cipher-3des.py"},
@@ -259,9 +259,7 @@
          {"name" : "test-tls13-count-tickets.py",
           "arguments" : ["-t", "2"]},
          {"name" : "test-tls13-crfg-curves.py"},
-         {"name" : "test-tls13-dhe-shared-secret-padding.py",
-          "arguments" : ["TLS 1.3 with ffdhe2048"],
-          "comment" : "test is non-deterministic and long, run just one case"},
+         {"name" : "test-tls13-dhe-shared-secret-padding.py"},
          {"name" : "test-tls13-ecdhe-curves.py"},
          {"name" : "test-tls13-empty-alert.py"},
          {"name" : "test-tls13-ffdhe-groups.py"},

diff --git a/tests/tlslite-ng.json b/tests/tlslite-ng.json
@@ -220,8 +220,9 @@
                          "-c", "tests/serverX509Cert.pem"]},
          {"name" : "test-resumption-with-wrong-ciphers.py"},
          {"name" : "test-serverhello-random.py",
-          "arguments" : ["-e", "Protocol (3, 0)",
-                         "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello"],
+          "arguments" : ["-x", "Protocol (3, 0)", "-X", "protocol_version",
+                         "-x", "Protocol (3, 0) in SSLv2 compatible ClientHello",
+                         "-X", "protocol_version"],
           "comment" : "tlslite-ng does not support SSLv3 by default"},
          {"name" : "test-sessionID-resumption.py"},
          {"name" : "test-sig-algs.py",