From 4ae435ad3bd7bdd44e044a8d5a03c3ab36cae8f4 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 19:26:13 +0200 Subject: [PATCH 1/9] don't do byte based copy in PRF calculation --- tlslite/mathtls.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tlslite/mathtls.py b/tlslite/mathtls.py index e40563a88..0275f66dc 100644 --- a/tlslite/mathtls.py +++ b/tlslite/mathtls.py @@ -677,18 +677,17 @@ def paramStrength(param): def P_hash(macFunc, secret, seed, length): - bytes = bytearray(length) + ret = bytearray(length) A = seed index = 0 - while 1: + while index < length: A = macFunc(secret, A) output = macFunc(secret, A + seed) - for c in output: - if index >= length: - return bytes - bytes[index] = c - index += 1 - return bytes + how_many = min(length - index, len(output)) + ret[index:index+how_many] = output[:how_many] + index += how_many + return ret + def PRF(secret, label, seed, length): #Split the secret into left and right halves From c52b72b0cbf1baad3a30c60fe8576253c1349621 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 24 Sep 2020 13:59:11 +0200 Subject: [PATCH 2/9] reuse the key stretching in HMAC for multiple invocations every time we call a HMAC_SHA256 it needs to create a new instance and calculate the i_key and o_key, even if the key used is the same use one instance of the HMAC and just copy state also use multiple update calls, don't concatenate inputs --- tlslite/mathtls.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tlslite/mathtls.py b/tlslite/mathtls.py index 0275f66dc..db8f5adfa 100644 --- a/tlslite/mathtls.py +++ b/tlslite/mathtls.py @@ -676,13 +676,22 @@ def paramStrength(param): return 256 # NIST SP 800-57 -def P_hash(macFunc, secret, seed, length): +def P_hash(mac_name, secret, seed, length): + """Internal method for calculation the PRF in TLS.""" ret = bytearray(length) + seed = compatHMAC(seed) A = seed index = 0 + mac = hmac.HMAC(compatHMAC(secret), digestmod=mac_name) while index < length: - A = macFunc(secret, A) - output = macFunc(secret, A + seed) + a_fun = mac.copy() + a_fun.update(A) + A = a_fun.digest() + out_fun = mac.copy() + out_fun.update(A) + out_fun.update(seed) + output = out_fun.digest() + how_many = min(length - index, len(output)) ret[index:index+how_many] = output[:how_many] index += how_many @@ -696,8 +705,8 @@ def PRF(secret, label, seed, length): S2 = secret[ int(math.floor(len(secret)/2.0)) : ] #Run the left half through P_MD5 and the right half through P_SHA1 - p_md5 = P_hash(HMAC_MD5, S1, label + seed, length) - p_sha1 = P_hash(HMAC_SHA1, S2, label + seed, length) + p_md5 = P_hash("md5", S1, label + seed, length) + p_sha1 = P_hash("sha1", S2, label + seed, length) #XOR the output values and return the result for x in range(length): @@ -706,11 +715,11 @@ def PRF(secret, label, seed, length): def PRF_1_2(secret, label, seed, length): """Pseudo Random Function for TLS1.2 ciphers that use SHA256""" - return P_hash(HMAC_SHA256, secret, label + seed, length) + return P_hash("sha256", secret, label + seed, length) def PRF_1_2_SHA384(secret, label, seed, length): """Pseudo Random Function for TLS1.2 ciphers that use SHA384""" - return P_hash(HMAC_SHA384, secret, label + seed, length) + return P_hash("sha384", secret, label + seed, length) def PRF_SSL(secret, seed, length): bytes = bytearray(length) From 693bc32b1acd11b82d1d16f7bcefc41ab112e819 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 20:12:47 +0200 Subject: [PATCH 3/9] use built-in HMAC, if it allows MD5 --- tlslite/utils/tlshmac.py | 114 +++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/tlslite/utils/tlshmac.py b/tlslite/utils/tlshmac.py index afa048d90..789df7227 100644 --- a/tlslite/utils/tlshmac.py +++ b/tlslite/utils/tlshmac.py @@ -17,64 +17,72 @@ except ImportError: __all__ = ["new", "HMAC"] +try: + from hmac import HMAC, new + # if we can calculate HMAC on MD5, then use the built-in HMAC + # implementation + _val = HMAC(b'some key', b'msg', 'md5') + _val.digest() + del _val +except Exception: + # fallback only when MD5 doesn't work + class HMAC(object): + """Hacked version of HMAC that works in FIPS mode even with MD5.""" -class HMAC(object): - """Hacked version of HMAC that works in FIPS mode even with MD5.""" + def __init__(self, key, msg=None, digestmod=None): + """ + Initialise the HMAC and hash first portion of data. - def __init__(self, key, msg=None, digestmod=None): - """ - Initialise the HMAC and hash first portion of data. + msg: data to hash + digestmod: name of hash or object that be used as a hash and be cloned + """ + self.key = key + if digestmod is None: + digestmod = 'md5' + if callable(digestmod): + digestmod = digestmod() + if not hasattr(digestmod, 'digest_size'): + digestmod = tlshashlib.new(digestmod) + self.block_size = digestmod.block_size + self.digest_size = digestmod.digest_size + self.digestmod = digestmod + if len(key) > self.block_size: + k_hash = digestmod.copy() + k_hash.update(compatHMAC(key)) + key = k_hash.digest() + if len(key) < self.block_size: + key = key + b'\x00' * (self.block_size - len(key)) + key = bytearray(key) + ipad = bytearray(b'\x36' * self.block_size) + opad = bytearray(b'\x5c' * self.block_size) + i_key = bytearray(i ^ j for i, j in zip(key, ipad)) + self._o_key = bytearray(i ^ j for i, j in zip(key, opad)) + self._context = digestmod.copy() + self._context.update(compatHMAC(i_key)) + if msg: + self._context.update(compatHMAC(msg)) - msg: data to hash - digestmod: name of hash or object that be used as a hash and be cloned - """ - self.key = key - if digestmod is None: - digestmod = 'md5' - if callable(digestmod): - digestmod = digestmod() - if not hasattr(digestmod, 'digest_size'): - digestmod = tlshashlib.new(digestmod) - self.block_size = digestmod.block_size - self.digest_size = digestmod.digest_size - self.digestmod = digestmod - if len(key) > self.block_size: - k_hash = digestmod.copy() - k_hash.update(compatHMAC(key)) - key = k_hash.digest() - if len(key) < self.block_size: - key = key + b'\x00' * (self.block_size - len(key)) - key = bytearray(key) - ipad = bytearray(b'\x36' * self.block_size) - opad = bytearray(b'\x5c' * self.block_size) - i_key = bytearray(i ^ j for i, j in zip(key, ipad)) - self._o_key = bytearray(i ^ j for i, j in zip(key, opad)) - self._context = digestmod.copy() - self._context.update(compatHMAC(i_key)) - if msg: + def update(self, msg): self._context.update(compatHMAC(msg)) - def update(self, msg): - self._context.update(compatHMAC(msg)) - - def digest(self): - i_digest = self._context.digest() - o_hash = self.digestmod.copy() - o_hash.update(compatHMAC(self._o_key)) - o_hash.update(compatHMAC(i_digest)) - return o_hash.digest() + def digest(self): + i_digest = self._context.digest() + o_hash = self.digestmod.copy() + o_hash.update(compatHMAC(self._o_key)) + o_hash.update(compatHMAC(i_digest)) + return o_hash.digest() - def copy(self): - new = HMAC.__new__(HMAC) - new.key = self.key - new.digestmod = self.digestmod - new.block_size = self.block_size - new.digest_size = self.digest_size - new._o_key = self._o_key - new._context = self._context.copy() - return new + def copy(self): + new = HMAC.__new__(HMAC) + new.key = self.key + new.digestmod = self.digestmod + new.block_size = self.block_size + new.digest_size = self.digest_size + new._o_key = self._o_key + new._context = self._context.copy() + return new -def new(*args, **kwargs): - """General constructor that works in FIPS mode.""" - return HMAC(*args, **kwargs) + def new(*args, **kwargs): + """General constructor that works in FIPS mode.""" + return HMAC(*args, **kwargs) From 5c536bf404bff7b6e5a1bd25496dc4add81c820b Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 20:14:46 +0200 Subject: [PATCH 4/9] =?UTF-8?q?use=20built-in=20int.from=5Fbytes=20on=20py?= =?UTF-8?q?thon3=20for=20bytes=E2=86=92int=20conversion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tlslite/utils/compat.py | 12 ++++++++++++ tlslite/utils/cryptomath.py | 14 ++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tlslite/utils/compat.py b/tlslite/utils/compat.py index ff4d35759..1e6cfb46c 100644 --- a/tlslite/utils/compat.py +++ b/tlslite/utils/compat.py @@ -80,6 +80,8 @@ def remove_whitespace(text): """Removes all whitespace from passed in string""" return re.sub(r"\s+", "", text, flags=re.UNICODE) + bytes_to_int = int.from_bytes + else: # Python 2.6 requires strings instead of bytearrays in a couple places, # so we define this function so it does the conversion if needed. @@ -147,6 +149,16 @@ def time_stamp(): """Returns system time as a float""" return time.clock() + def bytes_to_int(val, byteorder): + """Convert bytes to an int.""" + if not val: + return 0 + if byteorder == "big": + return int(b2a_hex(val), 16) + if byteorder == "little": + return int(b2a_hex(val[::-1]), 16) + raise ValueError("Only 'big' and 'little' endian supported") + try: # Fedora and Red Hat Enterprise Linux versions have small curves removed getattr(ecdsa, 'NIST192p') diff --git a/tlslite/utils/cryptomath.py b/tlslite/utils/cryptomath.py index 3831df31b..b91a6011b 100644 --- a/tlslite/utils/cryptomath.py +++ b/tlslite/utils/cryptomath.py @@ -15,7 +15,7 @@ import binascii import sys -from .compat import compat26Str, compatHMAC, compatLong, b2a_hex +from .compat import compat26Str, compatHMAC, compatLong, bytes_to_int from .codec import Writer from . import tlshashlib as hashlib @@ -204,18 +204,8 @@ def bytesToNumber(b, endian="big"): By default assumes big-endian encoding of the number. """ - # if string is empty, consider it to be representation of zero - # while it may be a bit unorthodox, it is the inverse of numberToByteArray - # with default parameters - if not b: - return 0 + return bytes_to_int(b, endian) - if endian == "big": - return int(b2a_hex(b), 16) - elif endian == "little": - return int(b2a_hex(b[::-1]), 16) - else: - raise ValueError("Only 'big' and 'little' endian supported") def numberToByteArray(n, howManyBytes=None, endian="big"): """ From 3990851880b22da9e02d611228aadb5d02c0ff6c Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 20:17:58 +0200 Subject: [PATCH 5/9] don't loop in Parser.get() --- tlslite/utils/codec.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tlslite/utils/codec.py b/tlslite/utils/codec.py index 2b03043bf..8645280c1 100644 --- a/tlslite/utils/codec.py +++ b/tlslite/utils/codec.py @@ -8,6 +8,7 @@ import sys import struct from struct import pack +from .compat import bytes_to_int class DecodeError(SyntaxError): @@ -305,14 +306,8 @@ def get(self, length): :rtype: int """ - if self.index + length > len(self.bytes): - raise DecodeError("Read past end of buffer") - x = 0 - for _ in range(length): - x <<= 8 - x |= self.bytes[self.index] - self.index += 1 - return x + ret = self.getFixBytes(length) + return bytes_to_int(ret, 'big') def getFixBytes(self, lengthBytes): """ From 2d53c14c8ff84eed014b934ec792d84f1c399e31 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 21:00:04 +0200 Subject: [PATCH 6/9] don't copy values in asn1parse if not necessary --- tlslite/utils/asn1parser.py | 8 ++++---- tlslite/utils/codec.py | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tlslite/utils/asn1parser.py b/tlslite/utils/asn1parser.py index 0e0416ab8..6edfa7e37 100644 --- a/tlslite/utils/asn1parser.py +++ b/tlslite/utils/asn1parser.py @@ -85,9 +85,9 @@ def getChildCount(self): while True: if p.getRemainingLength() == 0: break - p.get(1) # skip Type + p.skip_bytes(1) # skip Type length = self._getASN1Length(p) - p.getFixBytes(length) # skip value + p.skip_bytes(length) # skip value count += 1 return count @@ -104,9 +104,9 @@ def getChildBytes(self, which): p = Parser(self.value) for _ in range(which+1): markIndex = p.index - p.get(1) #skip Type + p.skip_bytes(1) # skip Type length = self._getASN1Length(p) - p.getFixBytes(length) + p.skip_bytes(length) return p.bytes[markIndex : p.index] @staticmethod diff --git a/tlslite/utils/codec.py b/tlslite/utils/codec.py index 8645280c1..f36400e47 100644 --- a/tlslite/utils/codec.py +++ b/tlslite/utils/codec.py @@ -324,6 +324,12 @@ def getFixBytes(self, lengthBytes): self.index += lengthBytes return bytes + def skip_bytes(self, length): + """Move the internal pointer ahead length bytes.""" + if self.index + length > len(self.bytes): + raise DecodeError("Read past end of buffer") + self.index += length + def getVarBytes(self, lengthLength): """ Read a variable length string with a fixed length. From 39981a5c11ce8f3c66d7f42174169ca141c63394 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 23:53:40 +0200 Subject: [PATCH 7/9] make the buffered socket buffer reads too --- tlslite/bufferedsocket.py | 12 +++++++++++- unit_tests/test_tlslite_bufferedsocket.py | 6 ++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tlslite/bufferedsocket.py b/tlslite/bufferedsocket.py index ec579bd77..c7dc7bb35 100644 --- a/tlslite/bufferedsocket.py +++ b/tlslite/bufferedsocket.py @@ -25,6 +25,7 @@ def __init__(self, socket): self.socket = socket self._write_queue = deque() self.buffer_writes = False + self._read_buffer = bytearray() def send(self, data): """Send data to the socket""" @@ -51,7 +52,16 @@ def flush(self): def recv(self, bufsize): """Receive data from socket (socket emulation)""" - return self.socket.recv(bufsize) + if self._read_buffer: # and len(self._read_buffer) < bufsize: + ret = self._read_buffer[:bufsize] + self._read_buffer = self._read_buffer[bufsize:] + return ret + read = bufsize - len(self._read_buffer) + read_bytes = self.socket.recv(max(4096, read)) + self._read_buffer += read_bytes + ret = self._read_buffer[:bufsize] + self._read_buffer = self._read_buffer[bufsize:] + return ret def getsockname(self): """Return the socket's own address (socket emulation).""" diff --git a/unit_tests/test_tlslite_bufferedsocket.py b/unit_tests/test_tlslite_bufferedsocket.py index f625e4f2d..bbfab4c9e 100644 --- a/unit_tests/test_tlslite_bufferedsocket.py +++ b/unit_tests/test_tlslite_bufferedsocket.py @@ -89,11 +89,9 @@ def test_flush_with_data_and_multiple_messages(self): self.raw_sock.sendall.assert_called_once_with(bytearray(b'abcdefg')) def test_recv(self): - value = mock.Mock() - ret = self.sock.recv(value) + ret = self.sock.recv(10) - self.raw_sock.recv.assert_called_once_with(value) - self.assertIs(ret, self.raw_sock.recv.return_value) + self.raw_sock.recv.assert_called_once_with(4096) def test_getsockname(self): ret = self.sock.getsockname() From 5fdf7aa089b26622e1135a601b6b2badbc1e4853 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 23:54:36 +0200 Subject: [PATCH 8/9] use native methods for number to bytes conversion --- tlslite/utils/compat.py | 65 ++++++++++++++++++++++++++++++++++--- tlslite/utils/cryptomath.py | 47 +++++++++++---------------- 2 files changed, 79 insertions(+), 33 deletions(-) diff --git a/tlslite/utils/compat.py b/tlslite/utils/compat.py index 1e6cfb46c..979060e25 100644 --- a/tlslite/utils/compat.py +++ b/tlslite/utils/compat.py @@ -16,12 +16,21 @@ if sys.version_info >= (3,0): def compat26Str(x): return x - - # Python 3 requires bytes instead of bytearrays for HMAC - + + # Python 3.3 requires bytes instead of bytearrays for HMAC # So, python 2.6 requires strings, python 3 requires 'bytes', - # and python 2.7 can handle bytearrays... - def compatHMAC(x): return bytes(x) + # and python 2.7 and 3.5 can handle bytearrays... + # pylint: disable=invalid-name + # we need to keep compatHMAC and `x` for API compatibility + if sys.version_info < (3, 4): + def compatHMAC(x): + """Convert bytes-like input to format acceptable for HMAC.""" + return bytes(x) + else: + def compatHMAC(x): + """Convert bytes-like input to format acceptable for HMAC.""" + return x + # pylint: enable=invalid-name def compatAscii2Bytes(val): """Convert ASCII string to bytes.""" @@ -80,7 +89,24 @@ def remove_whitespace(text): """Removes all whitespace from passed in string""" return re.sub(r"\s+", "", text, flags=re.UNICODE) + # pylint: disable=invalid-name + # pylint is stupid here and deson't notice it's a function, not + # constant bytes_to_int = int.from_bytes + # pylint: enable=invalid-name + + def bit_length(val): + """Return number of bits necessary to represent an integer.""" + return val.bit_length() + + def int_to_bytes(val, length=None, byteorder="big"): + """Return number converted to bytes""" + if length is None: + length = byte_length(val) + # for gmpy we need to convert back to native int + if type(val) != int: + val = int(val) + return bytearray(val.to_bytes(length=length, byteorder=byteorder)) else: # Python 2.6 requires strings instead of bytearrays in a couple places, @@ -94,6 +120,12 @@ def compat26Str(x): return str(x) def remove_whitespace(text): """Removes all whitespace from passed in string""" return re.sub(r"\s+", "", text) + + def bit_length(val): + """Return number of bits necessary to represent an integer.""" + if val == 0: + return 0 + return len(bin(val))-2 else: def compat26Str(x): return x @@ -101,6 +133,10 @@ def remove_whitespace(text): """Removes all whitespace from passed in string""" return re.sub(r"\s+", "", text, flags=re.UNICODE) + def bit_length(val): + """Return number of bytes necessary to represent an integer.""" + return val.bit_length() + def compatAscii2Bytes(val): """Convert ASCII string to bytes.""" return val @@ -159,6 +195,25 @@ def bytes_to_int(val, byteorder): return int(b2a_hex(val[::-1]), 16) raise ValueError("Only 'big' and 'little' endian supported") + def int_to_bytes(val, length=None, byteorder="big"): + """Return number converted to bytes""" + if length is None: + length = byte_length(val) + if byteorder == "big": + return bytearray((val >> i) & 0xff + for i in reversed(range(0, length*8, 8))) + if byteorder == "little": + return bytearray((val >> i) & 0xff + for i in range(0, length*8, 8)) + raise ValueError("Only 'big' or 'little' endian supported") + + +def byte_length(val): + """Return number of bytes necessary to represent an integer.""" + length = bit_length(val) + return (length + 7) // 8 + + try: # Fedora and Red Hat Enterprise Linux versions have small curves removed getattr(ecdsa, 'NIST192p') diff --git a/tlslite/utils/cryptomath.py b/tlslite/utils/cryptomath.py index b91a6011b..eae171d7e 100644 --- a/tlslite/utils/cryptomath.py +++ b/tlslite/utils/cryptomath.py @@ -13,9 +13,9 @@ import math import base64 import binascii -import sys -from .compat import compat26Str, compatHMAC, compatLong, bytes_to_int +from .compat import compat26Str, compatHMAC, compatLong, \ + bytes_to_int, int_to_bytes, bit_length, byte_length from .codec import Writer from . import tlshashlib as hashlib @@ -215,16 +215,14 @@ def numberToByteArray(n, howManyBytes=None, endian="big"): not be larger. The returned bytearray will contain a big- or little-endian encoding of the input integer (n). Big endian encoding is used by default. """ - if howManyBytes == None: - howManyBytes = numBytes(n) - if endian == "big": - return bytearray((n >> i) & 0xff - for i in reversed(range(0, howManyBytes*8, 8))) - elif endian == "little": - return bytearray((n >> i) & 0xff - for i in range(0, howManyBytes*8, 8)) - else: - raise ValueError("Only 'big' and 'little' endian supported") + if howManyBytes is not None: + length = byte_length(n) + if howManyBytes < length: + ret = int_to_bytes(n, length, endian) + if endian == "big": + return ret[length-howManyBytes:length] + return ret[:howManyBytes] + return int_to_bytes(n, howManyBytes, endian) def mpiToNumber(mpi): @@ -255,23 +253,16 @@ def numberToMPI(n): # Misc. Utility Functions # ************************************************************************** -def numBits(n): - """Return number of bits necessary to represent the integer in binary""" - if n==0: - return 0 - if sys.version_info < (2, 7): - # bit_length() was introduced in 2.7, and it is an order of magnitude - # faster than the below code - return len(bin(n))-2 - else: - return n.bit_length() -def numBytes(n): - """Return number of bytes necessary to represent the integer in bytes""" - if n==0: - return 0 - bits = numBits(n) - return (bits + 7) // 8 +# pylint: disable=invalid-name +# pylint recognises them as constants, not function names, also +# we can't change their names without API change +numBits = bit_length + + +numBytes = byte_length +# pylint: enable=invalid-name + # ************************************************************************** # Big Number Math From 7df73e30d7eaf738ea46daf2689d070178c046a8 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 17 Sep 2020 23:55:17 +0200 Subject: [PATCH 9/9] other micro optimisations --- tlslite/utils/codec.py | 7 ++++--- tlslite/utils/rsakey.py | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tlslite/utils/codec.py b/tlslite/utils/codec.py index f36400e47..71fb3063c 100644 --- a/tlslite/utils/codec.py +++ b/tlslite/utils/codec.py @@ -318,11 +318,12 @@ def getFixBytes(self, lengthBytes): :rtype: bytearray """ - if self.index + lengthBytes > len(self.bytes): + end = self.index + lengthBytes + if end > len(self.bytes): raise DecodeError("Read past end of buffer") - bytes = self.bytes[self.index : self.index+lengthBytes] + ret = self.bytes[self.index : end] self.index += lengthBytes - return bytes + return ret def skip_bytes(self, length): """Move the internal pointer ahead length bytes.""" diff --git a/tlslite/utils/rsakey.py b/tlslite/utils/rsakey.py index b3163835e..4f11da62d 100644 --- a/tlslite/utils/rsakey.py +++ b/tlslite/utils/rsakey.py @@ -534,11 +534,10 @@ def _addPKCS1Padding(self, bytes, blockType): pad = bytearray(0) while len(pad) < padLength: padBytes = getRandomBytes(padLength * 2) - pad = [b for b in padBytes if b != 0] + pad = [b for b in padBytes if b] pad = pad[:padLength] else: raise AssertionError() padding = bytearray([0,blockType] + pad + [0]) - paddedBytes = padding + bytes - return paddedBytes + return padding + bytes