Merge pull request #764 from tlsfuzzer/correct-xpass

return error code in scripts on unexpected phase passes

.github/workflows/ci.yml

@@ -334,9 +334,15 @@ jobs:
       - name: Linter
         # pylint doesn't work on 2.6: https://bitbucket.org/logilab/pylint/issue/390/py26-compatiblity-broken
         if: ${{ matrix.python-version != 2.6 }}
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
+          opt=""
+          if ! [[ $PYTHON_VERSION == 2.6 || $PYTHON_VERSION == 2.7 || $PYTHON_VERSION == 3.3 || $PYTHON_VERSION == 3.4 || $PYTHON_VERSION == 3.5 ]]; then
+            opt="--compare-branch origin/master"
+          fi
           pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" tlsfuzzer > pylint_report.txt || :
-          diff-quality --violations=pylint --fail-under=90 pylint_report.txt
+          diff-quality $opt --violations=pylint --fail-under=90 pylint_report.txt
       - name: Verify that intermediate commits are testable
         # for some reason this takes a lot of time on Python 2.6, on the other
         # hand, it's highly unlikely that anybody will bisect code on

scripts/test-SSLv3-padding.py

@@ -23,7 +23,7 @@
 from tlsfuzzer.utils.lists import natural_sort_keys
 
 
-version = 5
+version = 6
 
 
 def help_msg():
@@ -340,7 +340,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-TLSv1_2-rejected-without-TLSv1_2.py

@@ -24,7 +24,7 @@
 from tlsfuzzer.utils.lists import natural_sort_keys
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -245,7 +245,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-aes-gcm-nonces.py

@@ -22,7 +22,9 @@
 from tlslite.utils.cryptomath import bytesToNumber
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 2
+
+version = 3
+
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -269,7 +271,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-aesccm.py

@@ -25,7 +25,8 @@
 from tlsfuzzer.helpers import SIG_ALL
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 3
+
+version = 4
 
 
 def help_msg():
@@ -862,7 +863,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-alpn-negotiation.py

@@ -28,7 +28,7 @@
 from tlsfuzzer.utils.ordered_dict import OrderedDict
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -831,7 +831,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-atypical-padding.py

@@ -28,7 +28,7 @@
 from tlslite.utils.compat import compatAscii2Bytes
 
 
-version = 5
+version = 6
 
 
 def help_msg():
@@ -622,7 +622,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-bleichenbacher-timing.py

@@ -29,7 +29,7 @@
 from tlsfuzzer.helpers import SIG_ALL, RSA_PKCS1_ALL
 
 
-version = 12
+version = 13
 
 
 def help_msg():
@@ -1099,7 +1099,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
     elif timing:
         # if regular tests passed, run timing collection and analysis

scripts/test-bleichenbacher-workaround.py

@@ -23,7 +23,9 @@
 from tlslite.extensions import SNIExtension
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 2
+
+version = 3
+
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -1209,7 +1211,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-certificate-malformed.py

@@ -33,7 +33,8 @@
 from tlsfuzzer.utils.lists import natural_sort_keys
 
 
-version = 3
+version = 4
+
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -558,7 +559,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 

scripts/test-certificate-request.py

@@ -33,7 +33,7 @@
 from tlslite.x509certchain import X509CertChain
 
 
-version = 7
+version = 8
 
 
 def help_msg():
@@ -391,7 +391,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-certificate-verify-malformed-sig.py

@@ -31,7 +31,7 @@
 from tlsfuzzer.helpers import RSA_SIG_ALL
 
 
-version = 5
+version = 6
 
 
 def help_msg():
@@ -366,7 +366,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-certificate-verify-malformed.py

@@ -32,7 +32,8 @@
 from tlsfuzzer.helpers import RSA_SIG_ALL
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 4
+
+version = 5
 
 
 def help_msg():
@@ -400,7 +401,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-certificate-verify.py

@@ -30,7 +30,7 @@
 from tlsfuzzer.helpers import RSA_SIG_ALL
 
 
-version = 8
+version = 9
 
 
 def help_msg():
@@ -389,7 +389,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-chacha20.py

@@ -27,7 +27,7 @@
 from tlsfuzzer.helpers import SIG_ALL
 
 
-version = 3
+version = 4
 
 
 def help_msg():
@@ -455,7 +455,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-client-compatibility.py

@@ -32,7 +32,7 @@
 from tlsfuzzer.utils.lists import natural_sort_keys
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -5821,8 +5821,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-client-hello-max-size.py

@@ -27,7 +27,7 @@
 from tlsfuzzer.helpers import RSA_SIG_ALL
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -305,7 +305,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-clienthello-md5.py

@@ -34,7 +34,9 @@
 from tlsfuzzer.helpers import RSA_SIG_ALL
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 2
+
+version = 3
+
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -283,7 +285,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-conversation.py

@@ -26,7 +26,7 @@
 from tlsfuzzer.helpers import SIG_ALL
 
 
-version = 6
+version = 7
 
 
 def help_msg():
@@ -238,7 +238,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-cve-2016-2107.py

@@ -23,7 +23,9 @@
 
 from tlsfuzzer.utils.lists import natural_sort_keys
 
-version = 2
+
+version = 3
+
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -217,7 +219,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":

scripts/test-cve-2016-6309.py

@@ -23,7 +23,10 @@
         ExtensionType
 from tlslite.extensions import PaddingExtension
 from tlsfuzzer.utils.lists import natural_sort_keys
-version = 2
+
+
+version = 3
+
 
 def help_msg():
     print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
@@ -227,7 +230,7 @@ def main():
     if len(sort):
         print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
 
-    if bad > 0:
+    if bad or xpass:
         sys.exit(1)
 
 if __name__ == "__main__":