From 63641235a16b87b0e8f8ebb54144e9aaf38b27e1 Mon Sep 17 00:00:00 2001 From: Uxio Fuentefria Date: Wed, 15 Jun 2022 16:31:12 +0200 Subject: [PATCH 1/2] Optimize `to_checksum_address` function While developing Safe Tx Service we found out that a lot of time was spent on converting addresses from `bytes` to `ChecksumAddress`. Biggest part of this time was overhead added on `eth_utils` library to `keccak256` function. So we decided to use `pysha3` library (way faster than `pycryptodome`) directly. That's why we implemented new methods: - fast_keccak - fast_keccak_hex - fast_keccak_hex - fast_to_checksum_address - fast_bytes_to_checksum_address - fast_is_checksum_address This is also very relevant for the `EthereumAddressV2Field`, as it stores addresses as `bytes` on database and returns them as `ChecksumAddress` Related to: - https://github.com/ethereum/eth-utils/issues/95 - https://github.com/ethereum/eth-hash/issues/35 --- README.rst | 2 +- docs/source/quickstart.rst | 2 +- gnosis/eth/clients/sourcify.py | 4 +- gnosis/eth/django/filters.py | 5 +- gnosis/eth/django/models.py | 21 ++- gnosis/eth/django/serializers.py | 4 +- gnosis/eth/django/tests/test_forms.py | 2 +- gnosis/eth/django/tests/test_models.py | 5 +- gnosis/eth/django/validators.py | 4 +- gnosis/eth/ethereum_client.py | 26 ++-- gnosis/eth/oracles/oracles.py | 8 +- gnosis/eth/tests/test_ethereum_client.py | 4 +- gnosis/eth/tests/test_utils.py | 95 +++++++++++- gnosis/eth/utils.py | 175 +++++++++++++++++++---- gnosis/safe/multi_send.py | 7 +- gnosis/safe/proxy_factory.py | 5 +- gnosis/safe/safe.py | 23 +-- gnosis/safe/safe_create2_tx.py | 12 +- gnosis/safe/safe_creation_tx.py | 17 ++- gnosis/safe/safe_signature.py | 6 +- gnosis/safe/safe_tx.py | 4 +- gp_cli.py | 7 +- requirements.txt | 1 + setup.py | 3 +- 24 files changed, 335 insertions(+), 107 deletions(-) diff --git a/README.rst b/README.rst index 04fc9e25d..8a51c3a9e 100644 --- a/README.rst +++ b/README.rst @@ -97,7 +97,7 @@ Contains utils for ethereum operations: - ``get_eth_address_with_key() -> Tuple[str, bytes]``: Returns a tuple of a valid public ethereum checksumed address with the private key. -- ``generate_address_2(from_: Union[str, bytes], salt: Union[str, bytes], init_code: [str, bytes]) -> str``: +- ``mk_contract_address_2(from_: Union[str, bytes], salt: Union[str, bytes], init_code: [str, bytes]) -> str``: Calculates the address of a new contract created using the new CREATE2 opcode. Ethereum django (REST) utils diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index b7ba88b3e..4c30876bf 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -100,7 +100,7 @@ Contains utils for ethereum operations: - ``get_eth_address_with_key() -> Tuple[str, bytes]``: Returns a tuple of a valid public ethereum checksumed address with the private key. -- ``generate_address_2(from_: Union[str, bytes], salt: Union[str, bytes], init_code: [str, bytes]) -> str``: +- ``mk_contract_address_2(from_: Union[str, bytes], salt: Union[str, bytes], init_code: [str, bytes]) -> str``: Calculates the address of a new contract created using the new CREATE2 opcode. Ethereum django (REST) utils diff --git a/gnosis/eth/clients/sourcify.py b/gnosis/eth/clients/sourcify.py index 137db88de..1566c2bd7 100644 --- a/gnosis/eth/clients/sourcify.py +++ b/gnosis/eth/clients/sourcify.py @@ -2,9 +2,9 @@ from urllib.parse import urljoin import requests -from web3 import Web3 from .. import EthereumNetwork +from ..utils import fast_is_checksum_address from .contract_metadata import ContractMetadata @@ -47,7 +47,7 @@ def _do_request(self, url: str) -> Optional[Dict[str, Any]]: def get_contract_metadata( self, contract_address: str ) -> Optional[ContractMetadata]: - assert Web3.isChecksumAddress( + assert fast_is_checksum_address( contract_address ), "Expecting a checksummed address" diff --git a/gnosis/eth/django/filters.py b/gnosis/eth/django/filters.py index e742ce1e0..235627bd2 100644 --- a/gnosis/eth/django/filters.py +++ b/gnosis/eth/django/filters.py @@ -6,7 +6,8 @@ import django_filters from hexbytes import HexBytes -from web3 import Web3 + +from ..utils import fast_is_checksum_address class EthereumAddressFieldForm(CharFieldForm): @@ -21,7 +22,7 @@ def to_python(self, value): value = super().to_python(value) if value in self.empty_values: return None - elif not Web3.isChecksumAddress(value): + elif not fast_is_checksum_address(value): raise ValidationError(self.error_messages["invalid"], code="invalid") return value diff --git a/gnosis/eth/django/models.py b/gnosis/eth/django/models.py index 1a0755f7b..fcc3f8008 100644 --- a/gnosis/eth/django/models.py +++ b/gnosis/eth/django/models.py @@ -6,9 +6,10 @@ from django.utils.translation import gettext_lazy as _ from eth_typing import ChecksumAddress -from eth_utils import to_checksum_address +from eth_utils import to_normalized_address from hexbytes import HexBytes +from ..utils import fast_bytes_to_checksum_address, fast_to_checksum_address from .filters import EthereumAddressFieldForm, Keccak256FieldForm from .validators import validate_checksumed_address @@ -45,7 +46,7 @@ def to_python(self, value): value = super().to_python(value) if value: try: - return to_checksum_address(value) + return fast_to_checksum_address(value) except ValueError: raise exceptions.ValidationError( self.error_messages["invalid"], @@ -74,16 +75,23 @@ def from_db_value( self, value: memoryview, expression, connection ) -> Optional[ChecksumAddress]: if value: - return to_checksum_address(value.hex()) + return fast_bytes_to_checksum_address(value) def get_prep_value(self, value: ChecksumAddress) -> Optional[bytes]: if value: - return HexBytes(self.to_python(value)) + try: + return HexBytes(to_normalized_address(value)) + except (TypeError, ValueError): + raise exceptions.ValidationError( + self.error_messages["invalid"], + code="invalid", + params={"value": value}, + ) def to_python(self, value) -> Optional[ChecksumAddress]: if value is not None: try: - return to_checksum_address(value) + return fast_to_checksum_address(value) except ValueError: raise exceptions.ValidationError( self.error_messages["invalid"], @@ -101,12 +109,13 @@ def formfield(self, **kwargs): class Uint256Field(models.DecimalField): - description = _("Ethereum uint256 number") """ Field to store ethereum uint256 values. Uses Decimal db type without decimals to store in the database, but retrieve as `int` instead of `Decimal` (https://docs.python.org/3/library/decimal.html) """ + description = _("Ethereum uint256 number") + def __init__(self, *args, **kwargs): kwargs["max_digits"] = 79 # 2 ** 256 is 78 digits kwargs["decimal_places"] = 0 diff --git a/gnosis/eth/django/serializers.py b/gnosis/eth/django/serializers.py index 501fa9293..0cec143b4 100644 --- a/gnosis/eth/django/serializers.py +++ b/gnosis/eth/django/serializers.py @@ -5,7 +5,6 @@ from hexbytes import HexBytes from rest_framework import serializers from rest_framework.exceptions import ValidationError -from web3 import Web3 from ..constants import ( SIGNATURE_R_MAX_VALUE, @@ -15,6 +14,7 @@ SIGNATURE_V_MAX_VALUE, SIGNATURE_V_MIN_VALUE, ) +from ..utils import fast_is_checksum_address logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ def to_representation(self, obj): def to_internal_value(self, data): # Check if address is valid try: - if Web3.toChecksumAddress(data) != data: + if not fast_is_checksum_address(data): raise ValueError elif int(data, 16) == 0 and not self.allow_zero_address: raise ValidationError("0x0 address is not allowed") diff --git a/gnosis/eth/django/tests/test_forms.py b/gnosis/eth/django/tests/test_forms.py index eacaa8063..a3cf8bfdd 100644 --- a/gnosis/eth/django/tests/test_forms.py +++ b/gnosis/eth/django/tests/test_forms.py @@ -16,7 +16,7 @@ class Keccak256Form(forms.Form): class TestForms(TestCase): def test_ethereum_address_field_form(self): - form = EthereumAddressForm(data={"value": "not a ethereum address"}) + form = EthereumAddressForm(data={"value": "not an ethereum address"}) self.assertFalse(form.is_valid()) self.assertEqual( form.errors["value"], ["Enter a valid checksummed Ethereum Address."] diff --git a/gnosis/eth/django/tests/test_models.py b/gnosis/eth/django/tests/test_models.py index 7f3a676fb..5a3b17141 100644 --- a/gnosis/eth/django/tests/test_models.py +++ b/gnosis/eth/django/tests/test_models.py @@ -7,6 +7,7 @@ from web3 import Web3 from ...constants import NULL_ADDRESS, SENTINEL_ADDRESS +from ...utils import fast_is_checksum_address from .models import EthereumAddress, EthereumAddressV2, Keccak256Hash, Sha3Hash, Uint256 faker = Faker() @@ -17,10 +18,10 @@ def test_ethereum_address_field(self): for EthereumAddressModel in (EthereumAddress, EthereumAddressV2): with self.subTest(EthereumAddressModel=EthereumAddressModel): address = Account.create().address - self.assertTrue(Web3.isChecksumAddress(address)) + self.assertTrue(fast_is_checksum_address(address)) ethereum_address = EthereumAddressModel.objects.create(value=address) ethereum_address.refresh_from_db() - self.assertTrue(Web3.isChecksumAddress(ethereum_address.value)) + self.assertTrue(fast_is_checksum_address(ethereum_address.value)) self.assertEqual(address, ethereum_address.value) # Test addresses diff --git a/gnosis/eth/django/validators.py b/gnosis/eth/django/validators.py index 65ee6418f..467d9c4e4 100644 --- a/gnosis/eth/django/validators.py +++ b/gnosis/eth/django/validators.py @@ -1,10 +1,10 @@ from django.core.exceptions import ValidationError -from web3 import Web3 +from ..utils import fast_is_checksum_address def validate_checksumed_address(address): - if not Web3.isChecksumAddress(address): + if not fast_is_checksum_address(address): raise ValidationError( "%(address)s has an invalid checksum", params={"address": address}, diff --git a/gnosis/eth/ethereum_client.py b/gnosis/eth/ethereum_client.py index bffa3b0b6..ed5be48fa 100644 --- a/gnosis/eth/ethereum_client.py +++ b/gnosis/eth/ethereum_client.py @@ -54,7 +54,11 @@ Wei, ) -from gnosis.eth.utils import mk_contract_address +from gnosis.eth.utils import ( + fast_is_checksum_address, + fast_to_checksum_address, + mk_contract_address, +) from .constants import ( ERC20_721_TRANSFER_TOPIC, @@ -434,7 +438,7 @@ def _decode_transfer_log( try: from_to_data = b"".join(topics[1:]) _from, to = ( - Web3.toChecksumAddress(address) + fast_to_checksum_address(address) for address in eth_abi.decode_abi( ["address", "address"], from_to_data ) @@ -452,7 +456,9 @@ def _decode_transfer_log( _from, to, token_id = eth_abi.decode_abi( ["address", "address", "uint256"], b"".join(topics[1:]) ) - _from, to = [Web3.toChecksumAddress(address) for address in (_from, to)] + _from, to = [ + fast_to_checksum_address(address) for address in (_from, to) + ] return {"from": _from, "to": to, "tokenId": token_id} return None @@ -892,7 +898,7 @@ def _decode_trace_action(self, action: Dict[str, Any]) -> Dict[str, Any]: # CALL, DELEGATECALL, CREATE or CREATE2 if "from" in action: - decoded["from"] = self.w3.toChecksumAddress(action["from"]) + decoded["from"] = fast_to_checksum_address(action["from"]) if "gas" in action: decoded["gas"] = int(action["gas"], 16) if "value" in action: @@ -904,7 +910,7 @@ def _decode_trace_action(self, action: Dict[str, Any]) -> Dict[str, Any]: if "input" in action: decoded["input"] = HexBytes(action["input"]) if "to" in action: - decoded["to"] = self.w3.toChecksumAddress(action["to"]) + decoded["to"] = fast_to_checksum_address(action["to"]) # CREATE or CREATE2 if "init" in action: @@ -912,13 +918,11 @@ def _decode_trace_action(self, action: Dict[str, Any]) -> Dict[str, Any]: # SELF-DESTRUCT if "address" in action: - decoded["address"] = self.w3.toChecksumAddress(action["address"]) + decoded["address"] = fast_to_checksum_address(action["address"]) if "balance" in action: decoded["balance"] = int(action["balance"], 16) if "refundAddress" in action: - decoded["refundAddress"] = self.w3.toChecksumAddress( - action["refundAddress"] - ) + decoded["refundAddress"] = fast_to_checksum_address(action["refundAddress"]) return decoded @@ -935,7 +939,7 @@ def _decode_trace_result(self, result: Dict[str, Any]) -> Dict[str, Any]: if "code" in result: decoded["code"] = HexBytes(result["code"]) if "address" in result: - decoded["address"] = self.w3.toChecksumAddress(result["address"]) + decoded["address"] = fast_to_checksum_address(result["address"]) return decoded @@ -1898,7 +1902,7 @@ def send_eth_to( :return: tx_hash """ - assert Web3.isChecksumAddress(to) + assert fast_is_checksum_address(to) account = Account.from_key(private_key) diff --git a/gnosis/eth/oracles/oracles.py b/gnosis/eth/oracles/oracles.py index 2d5c06e28..dbfdc09d2 100644 --- a/gnosis/eth/oracles/oracles.py +++ b/gnosis/eth/oracles/oracles.py @@ -9,7 +9,6 @@ from eth_abi.packed import encode_abi_packed from eth_typing import ChecksumAddress from hexbytes import HexBytes -from web3 import Web3 from web3.contract import Contract from web3.exceptions import BadFunctionCallOutput @@ -23,6 +22,7 @@ get_uniswap_v2_pair_contract, get_uniswap_v2_router_contract, ) +from ..utils import fast_bytes_to_checksum_address, fast_keccak from .abis.aave_abis import AAVE_ATOKEN_ABI from .abis.balancer_abis import balancer_pool_abi from .abis.cream_abis import cream_ctoken_abi @@ -380,16 +380,16 @@ def calculate_pair_address(self, token_address: str, token_address_2: str): """ if token_address.lower() > token_address_2.lower(): token_address, token_address_2 = token_address_2, token_address - salt = Web3.keccak( + salt = fast_keccak( encode_abi_packed(["address", "address"], [token_address, token_address_2]) ) - address = Web3.keccak( + address = fast_keccak( encode_abi_packed( ["bytes", "address", "bytes", "bytes"], [HexBytes("ff"), self.factory_address, salt, self.pair_init_code], ) )[-20:] - return Web3.toChecksumAddress(address) + return fast_bytes_to_checksum_address(address) def get_decimals(self, token_address: str, token_address_2: str) -> Tuple[int, int]: if not ( diff --git a/gnosis/eth/tests/test_ethereum_client.py b/gnosis/eth/tests/test_ethereum_client.py index aef914b94..73a1bae62 100644 --- a/gnosis/eth/tests/test_ethereum_client.py +++ b/gnosis/eth/tests/test_ethereum_client.py @@ -22,7 +22,7 @@ SenderAccountNotFoundInNode, ) from ..exceptions import BatchCallException, InvalidERC20Info -from ..utils import get_eth_address_with_key +from ..utils import fast_to_checksum_address, get_eth_address_with_key from .ethereum_test_case import EthereumTestCaseMixin from .mocks.mock_internal_txs import creation_internal_txs, internal_txs_errored from .mocks.mock_log_receipts import invalid_log_receipt, log_receipts @@ -531,7 +531,7 @@ def test_decode_trace(self): self.assertEqual(decoded_traces[0]["result"]["output"], HexBytes("")) self.assertEqual( decoded_traces[1]["result"]["address"], - self.w3.toChecksumAddress(example_traces[1]["result"]["address"]), + fast_to_checksum_address(example_traces[1]["result"]["address"]), ) self.assertEqual( decoded_traces[1]["result"]["code"], diff --git a/gnosis/eth/tests/test_utils.py b/gnosis/eth/tests/test_utils.py index 39c362bef..e37e58b7d 100644 --- a/gnosis/eth/tests/test_utils.py +++ b/gnosis/eth/tests/test_utils.py @@ -1,27 +1,44 @@ +import os + from django.test import TestCase from eth_abi.packed import encode_abi_packed from eth_account import Account +from eth_utils import to_checksum_address from hexbytes import HexBytes from ..contracts import get_proxy_1_0_0_deployed_bytecode, get_proxy_factory_contract -from ..utils import compare_byte_code, decode_string_or_bytes32, generate_address_2 +from ..utils import ( + compare_byte_code, + decode_string_or_bytes32, + fast_bytes_to_checksum_address, + fast_is_checksum_address, + fast_keccak, + fast_to_checksum_address, + generate_address_2, + mk_contract_address, + mk_contract_address_2, +) from .ethereum_test_case import EthereumTestCaseMixin class TestUtils(EthereumTestCaseMixin, TestCase): - def test_generate_address_2(self): + def test_mk_contract_address_2(self): from_ = "0x8942595A2dC5181Df0465AF0D7be08c8f23C93af" salt = self.w3.keccak(text="aloha") init_code = "0x00abcd" expected = "0x8D02C796Dd019916F65EBa1C9D65a7079Ece00E0" - address2 = generate_address_2(from_, salt, init_code) + address2 = mk_contract_address_2(from_, salt, init_code) self.assertEqual(address2, expected) from_ = HexBytes("0x8942595A2dC5181Df0465AF0D7be08c8f23C93af") salt = self.w3.keccak(text="aloha").hex() init_code = HexBytes("0x00abcd") expected = "0x8D02C796Dd019916F65EBa1C9D65a7079Ece00E0" + address2 = mk_contract_address_2(from_, salt, init_code) + self.assertEqual(address2, expected) + + # Make sure deprecated function is working address2 = generate_address_2(from_, salt, init_code) self.assertEqual(address2, expected) @@ -72,7 +89,7 @@ def test_generate_address2_with_proxy(self): deployment_data = encode_abi_packed( ["bytes", "uint256"], [proxy_creation_code, int(master_copy, 16)] ) - address2 = generate_address_2( + address2 = mk_contract_address_2( proxy_factory_contract.address, salt, deployment_data ) self.assertEqual(proxy_address, address2) @@ -104,3 +121,73 @@ def test_compare_byte_code(self): self.assertTrue( compare_byte_code(proxy_with_metadata, proxy_with_different_metadata) ) + + def test_fast_keccak(self): + text = "chidori" + self.assertEqual( + fast_keccak(text.encode()), + HexBytes( + "0xd20148e42186a9e7698e652e255c809851633d62bad625a55d05efd7f718449c" + ), + ) + + binary = b"" + self.assertEqual( + fast_keccak(binary), + HexBytes( + "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + ), + ) + + binary = b"1234" + self.assertEqual( + fast_keccak(binary), + HexBytes( + "0x387a8233c96e1fc0ad5e284353276177af2186e7afa85296f106336e376669f7" + ), + ) + + def test_fast_to_checksum_address(self): + for _ in range(10): + address = os.urandom(20).hex() + self.assertEqual( + fast_to_checksum_address(address), to_checksum_address(address) + ) + + def test_fast_bytes_to_checksum_address(self): + with self.assertRaises(ValueError): + fast_bytes_to_checksum_address(os.urandom(19)) + + with self.assertRaises(ValueError): + fast_bytes_to_checksum_address(os.urandom(21)) + + for _ in range(10): + address = os.urandom(20) + self.assertEqual( + fast_bytes_to_checksum_address(address), to_checksum_address(address) + ) + + def test_fast_is_checksum_address(self): + self.assertFalse(fast_is_checksum_address(None)) + self.assertFalse(fast_is_checksum_address("")) + self.assertFalse( + fast_is_checksum_address(0x6ED857DC1DA2C41470A95589BB482152000773E9) + ) + self.assertFalse(fast_is_checksum_address(2)) + self.assertFalse(fast_is_checksum_address("2")) + self.assertFalse( + fast_is_checksum_address("0x6ed857dc1da2c41470A95589bB482152000773e9") + ) + self.assertTrue( + fast_is_checksum_address("0x6ED857dc1da2c41470A95589bB482152000773e9") + ) + + def test_mk_contract_address(self): + self.assertEqual( + mk_contract_address("0x76E2cFc1F5Fa8F6a5b3fC4c8F4788F0116861F9B", 129), + "0x5A317285cD83092fD40153C4B7c3Df64d8482Da8", + ) + self.assertEqual( + mk_contract_address("0x76E2cFc1F5Fa8F6a5b3fC4c8F4788F0116861F9B", 399), + "0xF03b503CC9Ee8aAA3B17856942a440be0c77Cd84", + ) diff --git a/gnosis/eth/utils.py b/gnosis/eth/utils.py index 045008fc5..283e8bafa 100644 --- a/gnosis/eth/utils.py +++ b/gnosis/eth/utils.py @@ -1,50 +1,116 @@ +import warnings from secrets import token_bytes from typing import Tuple, Union import eth_abi from eth._utils.address import generate_contract_address from eth_keys import keys -from eth_utils import to_canonical_address, to_checksum_address +from eth_typing import AnyAddress, ChecksumAddress, HexStr +from eth_utils import to_normalized_address from hexbytes import HexBytes -from web3 import Web3 +from sha3 import keccak_256 -def get_eth_address_with_key() -> Tuple[str, bytes]: - private_key = keys.PrivateKey(token_bytes(32)) - address = private_key.public_key.to_checksum_address() - return address, private_key.to_bytes() +def fast_keccak(value: bytes) -> bytes: + """ + Calculates ethereum keccak256 using fast library `pysha3` + :param value: + :return: Keccak256 used by ethereum as `bytes` + """ + return keccak_256(value).digest() -def get_eth_address_with_invalid_checksum() -> str: - address, _ = get_eth_address_with_key() - return "0x" + "".join( - [c.lower() if c.isupper() else c.upper() for c in address[2:]] +def fast_keccak_hex(value: bytes) -> HexStr: + """ + Same as `fast_keccak`, but it's a little more optimal calling `hexdigest()` + than calling `digest()` and then `hex()` + + :param value: + :return: Keccak256 used by ethereum as an hex string (not 0x prefixed) + """ + return HexStr(keccak_256(value).hexdigest()) + + +def _build_checksum_address( + norm_address: HexStr, address_hash: HexStr +) -> ChecksumAddress: + """ + https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md + + :param norm_address: address in lowercase (not 0x prefixed) + :param address_hash: keccak256 of `norm_address` (not 0x prefixed) + :return: + """ + return ChecksumAddress( + "0x" + + ( + "".join( + ( + norm_address[i].upper() + if int(address_hash[i], 16) > 7 + else norm_address[i] + ) + for i in range(0, 40) + ) + ) ) -def generate_address_2( - from_: Union[str, bytes], salt: Union[str, bytes], init_code: Union[str, bytes] -) -> str: +def fast_to_checksum_address(value: Union[AnyAddress, str, bytes]) -> ChecksumAddress: """ - Generates an address for a contract created using CREATE2. + Converts to checksum_address. Uses more optimal `pysha3` instead of `eth_utils` for keccak256 calculation - :param from_: The address which is creating this new address (need to be 20 bytes) - :param salt: A salt (32 bytes) - :param init_code: A init code of the contract being created - :return: Address of the new contract + :param value: + :return: """ + norm_address = to_normalized_address(value)[2:] + address_hash = fast_keccak_hex(norm_address.encode()) + return _build_checksum_address(norm_address, address_hash) - from_ = HexBytes(from_) - salt = HexBytes(salt) - init_code = HexBytes(init_code) - assert len(from_) == 20, f"Address {from_.hex()} is not valid. Must be 20 bytes" - assert len(salt) == 32, f"Salt {salt.hex()} is not valid. Must be 32 bytes" - assert len(init_code) > 0, f"Init code {init_code.hex()} is not valid" +def fast_bytes_to_checksum_address(value: bytes) -> ChecksumAddress: + """ + Converts to checksum_address. Uses more optimal `pysha3` instead of `eth_utils` for keccak256 calculation. + As input is already in bytes, some checks and conversions can be skipped, providing a speedup of ~50% - init_code_hash = Web3.keccak(init_code) - contract_address = Web3.keccak(HexBytes("ff") + from_ + salt + init_code_hash) - return Web3.toChecksumAddress(contract_address[12:]) + :param value: + :return: + """ + if len(value) != 20: + raise ValueError( + "Cannot convert %s to a checksum address, 20 bytes were expected" + ) + norm_address = bytes(value).hex() + address_hash = fast_keccak_hex(norm_address.encode()) + return _build_checksum_address(norm_address, address_hash) + + +def fast_is_checksum_address(value: Union[AnyAddress, str, bytes]) -> bool: + """ + Fast version to check if an address is a checksum_address + + :param value: + :return: `True` if checksummed, `False` otherwise + """ + if not isinstance(value, str) or len(value) != 42 or not value.startswith("0x"): + return False + try: + return fast_to_checksum_address(value) == value + except ValueError: + return False + + +def get_eth_address_with_key() -> Tuple[str, bytes]: + private_key = keys.PrivateKey(token_bytes(32)) + address = private_key.public_key.to_checksum_address() + return address, private_key.to_bytes() + + +def get_eth_address_with_invalid_checksum() -> str: + address, _ = get_eth_address_with_key() + return "0x" + "".join( + [c.lower() if c.isupper() else c.upper() for c in address[2:]] + ) def decode_string_or_bytes32(data: bytes) -> str: @@ -94,7 +160,56 @@ def compare_byte_code(code_1: bytes, code_2: bytes) -> bool: return codes[0] == codes[1] -def mk_contract_address(address: Union[str, bytes], nonce: int) -> str: - return to_checksum_address( - generate_contract_address(to_canonical_address(address), nonce) +def mk_contract_address(address: Union[str, bytes], nonce: int) -> ChecksumAddress: + """ + Generate expected contract address when using EVM CREATE + + :param address: + :param nonce: + :return: + """ + return fast_to_checksum_address(generate_contract_address(HexBytes(address), nonce)) + + +def mk_contract_address_2( + from_: Union[str, bytes], salt: Union[str, bytes], init_code: Union[str, bytes] +) -> ChecksumAddress: + + """ + Generate expected contract address when using EVM CREATE2. + + :param from_: The address which is creating this new address (need to be 20 bytes) + :param salt: A salt (32 bytes) + :param init_code: A init code of the contract being created + :return: Address of the new contract + """ + + from_ = HexBytes(from_) + salt = HexBytes(salt) + init_code = HexBytes(init_code) + + assert len(from_) == 20, f"Address {from_.hex()} is not valid. Must be 20 bytes" + assert len(salt) == 32, f"Salt {salt.hex()} is not valid. Must be 32 bytes" + assert len(init_code) > 0, f"Init code {init_code.hex()} is not valid" + + init_code_hash = fast_keccak(init_code) + contract_address = fast_keccak(HexBytes("ff") + from_ + salt + init_code_hash) + return fast_bytes_to_checksum_address(contract_address[12:]) + + +def generate_address_2( + from_: Union[str, bytes], salt: Union[str, bytes], init_code: Union[str, bytes] +) -> ChecksumAddress: + """ + .. deprecated:: use mk_contract_address_2 + + :param from_: + :param salt: + :param init_code: + :return: + """ + warnings.warn( + "`generate_address_2` is deprecated, use `mk_contract_address_2`", + DeprecationWarning, ) + return mk_contract_address_2(from_, salt, init_code) diff --git a/gnosis/safe/multi_send.py b/gnosis/safe/multi_send.py index 7828e1aa2..7319fe2f0 100644 --- a/gnosis/safe/multi_send.py +++ b/gnosis/safe/multi_send.py @@ -10,6 +10,7 @@ from gnosis.eth.contracts import get_multi_send_contract from gnosis.eth.ethereum_client import EthereumTxSent from gnosis.eth.typing import EthereumData +from gnosis.eth.utils import fast_bytes_to_checksum_address, fast_is_checksum_address logger = getLogger(__name__) @@ -115,7 +116,7 @@ def _decode_multisend_data(cls, encoded_multisend_tx: Union[str, bytes]): """ encoded_multisend_tx = HexBytes(encoded_multisend_tx) operation = MultiSendOperation(encoded_multisend_tx[0]) - to = Web3.toChecksumAddress(encoded_multisend_tx[1 : 1 + 20]) + to = fast_bytes_to_checksum_address(encoded_multisend_tx[1 : 1 + 20]) value = int.from_bytes(encoded_multisend_tx[21 : 21 + 32], byteorder="big") data_length = int.from_bytes( encoded_multisend_tx[21 + 32 : 21 + 32 * 2], byteorder="big" @@ -149,7 +150,7 @@ def _decode_multisend_old_transaction( operation = MultiSendOperation( int.from_bytes(encoded_multisend_tx[:32], byteorder="big") ) - to = Web3.toChecksumAddress(encoded_multisend_tx[32:64][-20:]) + to = fast_bytes_to_checksum_address(encoded_multisend_tx[32:64][-20:]) value = int.from_bytes(encoded_multisend_tx[64:96], byteorder="big") data_length = int.from_bytes(encoded_multisend_tx[128:160], byteorder="big") data = encoded_multisend_tx[160 : 160 + data_length] @@ -165,7 +166,7 @@ class MultiSend: dummy_w3 = Web3() def __init__(self, address: str, ethereum_client: EthereumClient): - assert Web3.isChecksumAddress(address), ( + assert fast_is_checksum_address(address), ( "%s proxy factory address not valid" % address ) diff --git a/gnosis/safe/proxy_factory.py b/gnosis/safe/proxy_factory.py index 8db2a13b6..48edd1702 100644 --- a/gnosis/safe/proxy_factory.py +++ b/gnosis/safe/proxy_factory.py @@ -3,7 +3,6 @@ from eth_account.signers.local import LocalAccount from eth_typing import ChecksumAddress -from web3 import Web3 from web3.contract import Contract from gnosis.eth import EthereumClient @@ -18,7 +17,7 @@ get_proxy_factory_V1_1_1_contract, ) from gnosis.eth.ethereum_client import EthereumTxSent -from gnosis.eth.utils import compare_byte_code +from gnosis.eth.utils import compare_byte_code, fast_is_checksum_address try: from functools import cache @@ -33,7 +32,7 @@ class ProxyFactory: def __init__(self, address: ChecksumAddress, ethereum_client: EthereumClient): - assert Web3.isChecksumAddress(address), ( + assert fast_is_checksum_address(address), ( "%s proxy factory address not valid" % address ) diff --git a/gnosis/safe/safe.py b/gnosis/safe/safe.py index 492462b5e..19dda84d3 100644 --- a/gnosis/safe/safe.py +++ b/gnosis/safe/safe.py @@ -24,7 +24,11 @@ get_safe_V1_3_0_contract, ) from gnosis.eth.ethereum_client import EthereumClient, EthereumTxSent -from gnosis.eth.utils import get_eth_address_with_key +from gnosis.eth.utils import ( + fast_bytes_to_checksum_address, + fast_is_checksum_address, + get_eth_address_with_key, +) from gnosis.safe.proxy_factory import ProxyFactory from ..eth.typing import EthereumData @@ -91,7 +95,7 @@ def __init__(self, address: ChecksumAddress, ethereum_client: EthereumClient): :param address: Safe address :param ethereum_client: Initialized ethereum client """ - assert Web3.isChecksumAddress(address), "%s is not a valid address" % address + assert fast_is_checksum_address(address), "%s is not a valid address" % address self.ethereum_client = ethereum_client self.w3 = self.ethereum_client.w3 @@ -888,9 +892,9 @@ def retrieve_fallback_handler( self.address, self.FALLBACK_HANDLER_STORAGE_SLOT, block_identifier=block_identifier, - )[-20:] + )[-20:].rjust(20, b"\0") if len(address) == 20: - return Web3.toChecksumAddress(address) + return fast_bytes_to_checksum_address(address) else: return NULL_ADDRESS @@ -899,20 +903,19 @@ def retrieve_guard( ) -> ChecksumAddress: address = self.ethereum_client.w3.eth.get_storage_at( self.address, self.GUARD_STORAGE_SLOT, block_identifier=block_identifier - )[-20:] + )[-20:].rjust(20, b"\0") if len(address) == 20: - return Web3.toChecksumAddress(address) + return fast_bytes_to_checksum_address(address) else: return NULL_ADDRESS def retrieve_master_copy_address( self, block_identifier: Optional[BlockIdentifier] = "latest" ) -> ChecksumAddress: - bytes_address = self.w3.eth.get_storage_at( + address = self.w3.eth.get_storage_at( self.address, "0x00", block_identifier=block_identifier - )[-20:] - int_address = int.from_bytes(bytes_address, byteorder="big") - return Web3.toChecksumAddress("{:#042x}".format(int_address)) + )[-20:].rjust(20, b"\0") + return fast_bytes_to_checksum_address(address) def retrieve_modules( self, diff --git a/gnosis/safe/safe_create2_tx.py b/gnosis/safe/safe_create2_tx.py index 092411991..7f17b2325 100644 --- a/gnosis/safe/safe_create2_tx.py +++ b/gnosis/safe/safe_create2_tx.py @@ -15,7 +15,7 @@ get_safe_V1_1_1_contract, get_safe_V1_3_0_contract, ) -from gnosis.eth.utils import generate_address_2 +from gnosis.eth.utils import fast_is_checksum_address, mk_contract_address_2 logger = getLogger(__name__) @@ -54,8 +54,8 @@ def __init__(self, w3: Web3, master_copy_address: str, proxy_factory_address: st :param master_copy_address: `Gnosis Safe` master copy address :param proxy_factory_address: `Gnosis Proxy Factory` address """ - assert Web3.isChecksumAddress(master_copy_address) - assert Web3.isChecksumAddress(proxy_factory_address) + assert fast_is_checksum_address(master_copy_address) + assert fast_is_checksum_address(proxy_factory_address) self.w3 = w3 self.master_copy_address = master_copy_address @@ -110,8 +110,8 @@ def build( fallback_handler = fallback_handler or NULL_ADDRESS payment_receiver = payment_receiver or NULL_ADDRESS payment_token = payment_token or NULL_ADDRESS - assert Web3.isChecksumAddress(payment_receiver) - assert Web3.isChecksumAddress(payment_token) + assert fast_is_checksum_address(payment_receiver) + assert fast_is_checksum_address(payment_token) # Get bytes for `setup(address[] calldata _owners, uint256 _threshold, address to, bytes calldata data, # address paymentToken, uint256 payment, address payable paymentReceiver)` @@ -222,7 +222,7 @@ def calculate_create2_address(self, safe_setup_data: bytes, salt_nonce: int): ["bytes", "uint256"], [proxy_creation_code, int(self.master_copy_address, 16)], ) - return generate_address_2( + return mk_contract_address_2( self.proxy_factory_contract.address, salt, deployment_data ) diff --git a/gnosis/safe/safe_creation_tx.py b/gnosis/safe/safe_creation_tx.py index bc8ed413b..0fe09dee5 100644 --- a/gnosis/safe/safe_creation_tx.py +++ b/gnosis/safe/safe_creation_tx.py @@ -7,7 +7,6 @@ from eth.constants import SECPK1_N from eth.vm.forks.frontier.transactions import FrontierTransaction from eth_keys.exceptions import BadSignature -from eth_utils import to_canonical_address, to_checksum_address from hexbytes import HexBytes from web3 import Web3 from web3.contract import ContractConstructor @@ -18,7 +17,11 @@ get_paying_proxy_contract, get_safe_V0_0_1_contract, ) -from gnosis.eth.utils import mk_contract_address +from gnosis.eth.utils import ( + fast_is_checksum_address, + fast_to_checksum_address, + mk_contract_address, +) logger = getLogger(__name__) @@ -58,9 +61,9 @@ def __init__( assert 0 < threshold <= len(owners) funder = funder or NULL_ADDRESS payment_token = payment_token or NULL_ADDRESS - assert Web3.isChecksumAddress(master_copy) - assert Web3.isChecksumAddress(funder) - assert Web3.isChecksumAddress(payment_token) + assert fast_is_checksum_address(master_copy) + assert fast_is_checksum_address(funder) + assert fast_is_checksum_address(payment_token) self.w3 = w3 self.owners = owners @@ -107,7 +110,7 @@ def __init__( ) self.tx_raw = rlp.encode(self.tx_pyethereum) self.tx_hash = self.tx_pyethereum.hash - self.deployer_address = to_checksum_address(self.tx_pyethereum.sender) + self.deployer_address = fast_to_checksum_address(self.tx_pyethereum.sender) self.safe_address = mk_contract_address(self.tx_pyethereum.sender, 0) self.v = self.tx_pyethereum.v @@ -264,7 +267,7 @@ def _build_contract_creation_tx_with_valid_signature( nonce, gas_price, gas, to, value, HexBytes(data), v=v, r=r, s=s ) sender_address = contract_creation_tx.sender - contract_address: bytes = to_canonical_address( + contract_address: bytes = HexBytes( mk_contract_address(sender_address, nonce) ) if sender_address in (zero_address, f_address) or contract_address in ( diff --git a/gnosis/safe/safe_signature.py b/gnosis/safe/safe_signature.py index 9a123bd73..966ecce2b 100644 --- a/gnosis/safe/safe_signature.py +++ b/gnosis/safe/safe_signature.py @@ -7,12 +7,12 @@ from eth_abi.exceptions import DecodingError from eth_account.messages import defunct_hash_message from eth_typing import ChecksumAddress -from eth_utils import to_checksum_address from hexbytes import HexBytes from web3.exceptions import BadFunctionCallOutput from gnosis.eth import EthereumClient from gnosis.eth.contracts import get_safe_contract, get_safe_V1_1_1_contract +from gnosis.eth.utils import fast_to_checksum_address from gnosis.safe.signatures import ( get_signing_address, signature_split, @@ -61,7 +61,9 @@ def uint_to_address(value: int) -> ChecksumAddress: encoded = encode_single("uint", value) # Remove padding bytes, as Solidity will ignore it but `eth_abi` will not encoded_without_padding_bytes = b"\x00" * 12 + encoded[-20:] - return to_checksum_address(decode_single("address", encoded_without_padding_bytes)) + return fast_to_checksum_address( + decode_single("address", encoded_without_padding_bytes) + ) class SafeSignature(ABC): diff --git a/gnosis/safe/safe_tx.py b/gnosis/safe/safe_tx.py index 38cc4d658..dfcffe8c6 100644 --- a/gnosis/safe/safe_tx.py +++ b/gnosis/safe/safe_tx.py @@ -5,7 +5,6 @@ from eth_account import Account from hexbytes import HexBytes from packaging.version import Version -from web3 import Web3 from web3.exceptions import BadFunctionCallOutput, ContractLogicError from web3.types import BlockIdentifier, TxParams, Wei @@ -14,6 +13,7 @@ from gnosis.eth.contracts import get_safe_contract from ..eth.ethereum_client import TxSpeed +from ..eth.utils import fast_keccak from .exceptions import ( CouldNotFinishInitialization, CouldNotPayGasWithEther, @@ -201,7 +201,7 @@ def eip712_structured_data(self) -> Dict: def safe_tx_hash(self) -> HexBytes: message, domain = self._eip712_payload signable_bytes = message.signable_bytes(domain) - return HexBytes(Web3.keccak(signable_bytes)) + return HexBytes(fast_keccak(signable_bytes)) @property def signers(self) -> List[str]: diff --git a/gp_cli.py b/gp_cli.py index b013d1efa..484460ddf 100644 --- a/gp_cli.py +++ b/gp_cli.py @@ -1,3 +1,6 @@ +from gnosis.eth.utils import fast_keccak + + def confirm_prompt(question: str) -> bool: reply = None while reply not in ("y", "n"): @@ -11,8 +14,6 @@ def confirm_prompt(question: str) -> bool: import sys import time - from web3 import Web3 - from gnosis.eth import EthereumNetwork from gnosis.eth.constants import NULL_ADDRESS from gnosis.protocol import GnosisProtocolAPI, Order, OrderKind @@ -54,7 +55,7 @@ def confirm_prompt(question: str) -> bool: sellAmount=amount_wei, buyAmount=buy_amount, validTo=int(time.time()) + (60 * 60), # Valid for 1 hour - appData=Web3.keccak(text="gp-cli"), + appData=fast_keccak(text="gp-cli"), feeAmount=0, kind="sell", # `sell` or `buy` partiallyFillable=not args.require_full_fill, diff --git a/requirements.txt b/requirements.txt index d811b43d1..42722347e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,6 @@ packaging psycopg2-binary==2.9.3 py-evm==0.5.0a3 requests==2.28.0 +pysha3>=1.0.0 typing-extensions==3.10.0.2 web3==5.29.2 diff --git a/setup.py b/setup.py index 87ea62583..b35f6e026 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,9 @@ "eip712_structs", "packaging", "py-evm>=0.5.0a3", - "typing-extensions>=3.10; python_version < '3.8'", + "pysha3>=1.0.0", "requests>=2", + "typing-extensions>=3.10; python_version < '3.8'", "web3>=5.23.0", ] From 2ac405957d92af8075ecf075c9ed076829aa806c Mon Sep 17 00:00:00 2001 From: Uxio Fuentefria Date: Thu, 16 Jun 2022 13:13:07 +0200 Subject: [PATCH 2/2] Set version 4.1.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b35f6e026..e42dd91f8 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( name="safe-eth-py", - version="4.0.1", + version="4.1.0", packages=find_packages(), package_data={"gnosis": ["py.typed"]}, install_requires=requirements,