Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for Web3 v6 #485

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ethereumetl/jobs/export_origin_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,11 @@ def _export_batch(self, block_number_batch):
except ValueError as e:
if str(e) == "{'code': -32000, 'message': 'the method is currently not implemented: eth_newFilter'}":
self._supports_eth_newFilter = False
events = self.web3.eth.getLogs(filter_params)
events = self.web3.eth.get_logs(filter_params)
else:
raise(e)
else:
events = self.web3.eth.getLogs(filter_params)
events = self.web3.eth.get_logs(filter_params)
for event in events:
log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
listing, shop_products = self.event_extractor.extract_event_from_log(log, batch['contract_version'])
Expand All @@ -124,7 +124,7 @@ def _export_batch(self, block_number_batch):
self.shop_product_exporter.export_item(item)

if self._supports_eth_newFilter:
self.web3.eth.uninstallFilter(event_filter.filter_id)
self.web3.eth.uninstall_filter(event_filter.filter_id)

def _end(self):
self.batch_work_executor.shutdown()
Expand Down
4 changes: 2 additions & 2 deletions ethereumetl/jobs/export_token_transfers_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def _export_batch(self, block_number_batch):
except ValueError as e:
if str(e) == "{'code': -32000, 'message': 'the method is currently not implemented: eth_newFilter'}":
self._supports_eth_newFilter = False
events = self.web3.eth.getLogs(filter_params)
events = self.web3.eth.get_logs(filter_params)
else:
raise(e)
for event in events:
Expand All @@ -91,7 +91,7 @@ def _export_batch(self, block_number_batch):
self.item_exporter.export_item(self.token_transfer_mapper.token_transfer_to_dict(token_transfer))

if self._supports_eth_newFilter:
self.web3.eth.uninstallFilter(event_filter.filter_id)
self.web3.eth.uninstall_filter(event_filter.filter_id)

def _end(self):
self.batch_work_executor.shutdown()
Expand Down
3 changes: 1 addition & 2 deletions ethereumetl/jobs/export_traces_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,10 @@ def _export_batch(self, block_number_batch):

# TODO: Change to traceFilter when this issue is fixed
# https://github.com/paritytech/parity-ethereum/issues/9822
json_traces = self.web3.parity.traceBlock(block_number)
json_traces = self.web3.tracing.trace_block(block_number)

if json_traces is None:
raise ValueError('Response from the node is None. Is the node fully synced? Is the node started with tracing enabled? Is trace_block API enabled?')

traces = [self.trace_mapper.json_dict_to_trace(json_trace) for json_trace in json_traces]
all_traces.extend(traces)

Expand Down
18 changes: 8 additions & 10 deletions ethereumetl/mappers/trace_mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@

from ethereumetl.domain.trace import EthTrace
from ethereumetl.mainnet_daofork_state_changes import DAOFORK_BLOCK_NUMBER
from ethereumetl.utils import hex_to_dec, to_normalized_address
from ethereumetl.utils import hex_to_dec, to_normalized_address, hex_to_str


class EthTraceMapper(object):
def json_dict_to_trace(self, json_dict):
trace = EthTrace()

trace.block_number = json_dict.get('blockNumber')
trace.transaction_hash = json_dict.get('transactionHash')
trace.transaction_hash = hex_to_str(json_dict.get('transactionHash'))
trace.transaction_index = json_dict.get('transactionPosition')
trace.subtraces = json_dict.get('subtraces')
trace.trace_address = json_dict.get('traceAddress', [])
Expand Down Expand Up @@ -62,12 +62,12 @@ def json_dict_to_trace(self, json_dict):
if trace_type == 'call':
trace.call_type = action.get('callType')
trace.to_address = to_normalized_address(action.get('to'))
trace.input = action.get('input')
trace.output = result.get('output')
trace.input = hex_to_str(action.get('input'))
trace.output = hex_to_str(result.get('output'))
elif trace_type == 'create':
trace.to_address = result.get('address')
trace.input = action.get('init')
trace.output = result.get('code')
trace.input = hex_to_str(action.get('init'))
trace.output = hex_to_str(result.get('code'))
elif trace_type == 'suicide':
trace.from_address = to_normalized_address(action.get('address'))
trace.to_address = to_normalized_address(action.get('refundAddress'))
Expand Down Expand Up @@ -132,10 +132,8 @@ def _iterate_transaction_trace(self, block_number, tx_index, tx_trace, trace_add

trace.from_address = to_normalized_address(tx_trace.get('from'))
trace.to_address = to_normalized_address(tx_trace.get('to'))

trace.input = tx_trace.get('input')
trace.output = tx_trace.get('output')

trace.input = hex_to_str(tx_trace.get('input'))
trace.output = hex_to_str(tx_trace.get('output'))
trace.value = hex_to_dec(tx_trace.get('value'))
trace.gas = hex_to_dec(tx_trace.get('gas'))
trace.gas_used = hex_to_dec(tx_trace.get('gasUsed'))
Expand Down
6 changes: 3 additions & 3 deletions ethereumetl/service/eth_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,13 @@ def __init__(self, web3):

def get_first_point(self):
# Ignore the genesis block as its timestamp is 0
return block_to_point(self._web3.eth.getBlock(1))
return block_to_point(self._web3.eth.get_block(1))

def get_last_point(self):
return block_to_point(self._web3.eth.getBlock('latest'))
return block_to_point(self._web3.eth.get_block('latest'))

def get_point(self, x):
return block_to_point(self._web3.eth.getBlock(x))
return block_to_point(self._web3.eth.get_block(x))


def block_to_point(block):
Expand Down
2 changes: 1 addition & 1 deletion ethereumetl/service/eth_token_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def __init__(self, web3, function_call_result_transformer=None):
self._function_call_result_transformer = function_call_result_transformer

def get_token(self, token_address):
checksum_address = self._web3.toChecksumAddress(token_address)
checksum_address = self._web3.to_checksum_address(token_address)
contract = self._web3.eth.contract(address=checksum_address, abi=ERC20_ABI)
contract_alternative_1 = self._web3.eth.contract(address=checksum_address, abi=ERC20_ABI_ALTERNATIVE_1)

Expand Down
4 changes: 2 additions & 2 deletions ethereumetl/service/origin_extractor.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import base58
import logging

from ethereumetl.utils import hex_to_dec, to_normalized_address
from ethereumetl.utils import hex_to_dec, hex_to_str
from ethereumetl.ipfs.origin import get_origin_marketplace_data

#
Expand Down Expand Up @@ -55,7 +55,7 @@ def extract_event_from_log(self, receipt_log, contract_version):
return None, []

listing_id = hex_to_dec(topics[2])
ipfs_hash = hex_to_ipfs_hash(receipt_log.data)
ipfs_hash = hex_to_ipfs_hash(hex_to_str(receipt_log.data))

full_listing_id = compose_listing_id(1, contract_version, listing_id)
marketplace_listing, shop_products = get_origin_marketplace_data(receipt_log, full_listing_id, self.ipfs_client, ipfs_hash)
Expand Down
4 changes: 2 additions & 2 deletions ethereumetl/service/token_transfer_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from builtins import map

from ethereumetl.domain.token_transfer import EthTokenTransfer
from ethereumetl.utils import chunk_string, hex_to_dec, to_normalized_address
from ethereumetl.utils import chunk_string, hex_to_dec, hex_to_str, to_normalized_address

# https://ethereum.stackexchange.com/questions/12553/understanding-logs-and-log-blooms
TRANSFER_EVENT_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
Expand All @@ -42,7 +42,7 @@ def extract_transfer_from_log(self, receipt_log):

if (topics[0]).casefold() == TRANSFER_EVENT_TOPIC:
# Handle unindexed event fields
topics_with_data = topics + split_to_words(receipt_log.data)
topics_with_data = topics + split_to_words(hex_to_str(receipt_log.data))
# if the number of topics and fields in data part != 4, then it's a weird event
if len(topics_with_data) != 4:
logger.warning("The number of topics and data parts is not equal to 4 in log {} of transaction {}"
Expand Down
2 changes: 1 addition & 1 deletion ethereumetl/streaming/eth_streamer_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def open(self):

def get_current_block_number(self):
w3 = build_web3(self.batch_web3_provider)
return int(w3.eth.getBlock("latest").number)
return int(w3.eth.get_block("latest").number)

def export_all(self, start_block, end_block):
# Export blocks and transactions
Expand Down
14 changes: 14 additions & 0 deletions ethereumetl/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,25 @@

import itertools
import warnings
from eth_utils import to_int, to_text, text_if_str
from hexbytes import HexBytes

from ethereumetl.misc.retriable_value_error import RetriableValueError


def hex_to_str(hex_string):
if isinstance(hex_string, HexBytes):
return hex_string.hex()

if hex_string is not None:
return text_if_str(to_text, hex_string)
return hex_string


def hex_to_dec(hex_string):
if type(hex_string) is int:
return to_int(hex_string)

if hex_string is None:
return None
try:
Expand Down
12 changes: 6 additions & 6 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,22 @@ def read(fname):
keywords='ethereum',
python_requires='>=3.7.2,<4',
install_requires=[
'web3>=5.29,<6',
'eth-utils==1.10',
'eth-abi>=2.2.0,<3.0.0',
'web3>=6,<7',
'eth-utils>=4.0.0',
'eth-abi>=5',
# TODO: This has to be removed when "ModuleNotFoundError: No module named 'eth_utils.toolz'" is fixed at eth-abi
'python-dateutil>=2.8.0,<3',
'click>=8.0.4,<9',
'ethereum-dasm==0.1.4',
'ethereum-dasm==0.1.5',
'urllib3<2',
'base58',
'requests'
],
extras_require={
'streaming': [
'timeout-decorator==0.4.1',
'google-cloud-pubsub==2.13.0',
'google-cloud-storage==1.33.0',
'google-cloud-pubsub==2.13.12',
'google-cloud-storage==1.44.0',
'kafka-python==2.0.2',
'sqlalchemy==1.4',
'pg8000==1.16.6',
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
block_number,transaction_hash,transaction_index,from_address,to_address,value,input,output,trace_type,call_type,reward_type,gas,gas_used,subtraces,trace_address,error,status,trace_id
1000690,0x2c3cfb8d7f3039fcad64bd231beb0229af55f9114d74aef12a6f228ad1dac100,0,0xaf21e07e5a929d16026a7b4d88f3906a8d2e4942,0x5b3c526b152b1f3d8eabe2ec27f49b904ad51cad,64655529900000002048,0x,0x,call,call,,0,0,0,,,1,call_0x2c3cfb8d7f3039fcad64bd231beb0229af55f9114d74aef12a6f228ad1dac100_
1000690,0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871,1,0xacdee28d8ca76187883831a37f551a5904cdf191,0xa7e3cf952ea8d9438a26ee346c295f1ada328ae1,0,0x606060405260026101086000505560405161015638038061015683398101604052805160805160a051919092019190808383815160019081018155600090600160a060020a0332169060029060038390559183525061010260205260408220555b82518110156100eb57828181518110156100025790602001906020020151600160a060020a03166002600050826002016101008110156100025790900160005081905550806002016101026000506000858481518110156100025790602001906020020151600160a060020a0316815260200190815260200160002060005081905550600101610060565b81600060005081905550505050806101056000508190555061010f62015180420490565b61010755505050506031806101256000396000f3003660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f30000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052b7d2dcc80cd2e40000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000acdee28d8ca76187883831a37f551a5904cdf191,0x3660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f3,create,,,954720,160631,0,,,1,create_0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871_
1000690,0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871,1,0xacdee28d8ca76187883831a37f551a5904cdf191,0xa7e3CF952EA8D9438a26EE346c295F1ada328AE1,0,0x606060405260026101086000505560405161015638038061015683398101604052805160805160a051919092019190808383815160019081018155600090600160a060020a0332169060029060038390559183525061010260205260408220555b82518110156100eb57828181518110156100025790602001906020020151600160a060020a03166002600050826002016101008110156100025790900160005081905550806002016101026000506000858481518110156100025790602001906020020151600160a060020a0316815260200190815260200160002060005081905550600101610060565b81600060005081905550505050806101056000508190555061010f62015180420490565b61010755505050506031806101256000396000f3003660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f30000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052b7d2dcc80cd2e40000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000acdee28d8ca76187883831a37f551a5904cdf191,0x3660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f3,create,,,954720,160631,0,,,1,create_0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871_
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that the to_address is checksummed and the from_address is not. Did Web3 v6 add address checksumming? Is there a way to disable it for backward compatibility?

1000690,,,,0xf8b483dba2c3b7176a3da549ad41a48bb3121069,5000000000000000000,,,reward,,block,,,0,,,1,reward_1000690_0
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"type": "contract", "address": "0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "bytecode": "0x6060604052361561008d5760e060020a600035046306fdde03811461008f578063095ea7b3146100a557806318160ddd1461012457806323b872dd1461012f578063313ce567146101dc578063475a9fa9146101f057806370a0823114610215578063721a37d21461024357806395d89b411461008f578063a9059cbb14610268578063dd62ed3e146102e7575b005b61031d6040805160208101909152600081525b90565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db63c6605267600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b6102316003546100a2565b61038b60043560243560443560008054604080517fa00bfa1100000000000000000000000000000000000000000000000000000000815260016004820152600160a060020a038781166024830152868116604483015260648201869052929092166084830152517319ee743d2e356d5f0e4d97cc09b96d06e933d0db9163a00bfa119160a482810192602092919082900301818660325a03f4156100025750506040515195945050505050565b604080516000815290519081900360200190f35b61038b6004356024356000805433600160a060020a0390811691161461039f57610002565b600160a060020a03600435166000908152600160205260409020545b60408051918252519081900360200190f35b61038b6004356024356000805433600160a060020a039081169116146103ce57610002565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db6388d5fecb600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b610231600435602435600160a060020a038281166000908152600260209081526040808320938516835292905220545b92915050565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561037d5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b604080519115158252519081900360200190f35b50600160a060020a03821660009081526001602081905260409091208054830190556003805483019055610317565b600160a060020a038316600090815260016020526040902054821161040a57506040600020805482900390556003805482900390556001610317565b50600061031756", "function_sighashes": ["0x06fdde03", "0x095ea7b3", "0x18160ddd", "0x23b872dd", "0x313ce567", "0x475a9fa9", "0x70a08231", "0x721a37d2", "0x95d89b41", "0xa9059cbb", "0xdd62ed3e"], "is_erc20": true, "is_erc721": false, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "contract_2112234_0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "item_timestamp": "2016-08-21T10:13:48Z"}
{"type": "contract", "address": "0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "bytecode": "0x6060604052361561008d5760e060020a600035046306fdde03811461008f578063095ea7b3146100a557806318160ddd1461012457806323b872dd1461012f578063313ce567146101dc578063475a9fa9146101f057806370a0823114610215578063721a37d21461024357806395d89b411461008f578063a9059cbb14610268578063dd62ed3e146102e7575b005b61031d6040805160208101909152600081525b90565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db63c6605267600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b6102316003546100a2565b61038b60043560243560443560008054604080517fa00bfa1100000000000000000000000000000000000000000000000000000000815260016004820152600160a060020a038781166024830152868116604483015260648201869052929092166084830152517319ee743d2e356d5f0e4d97cc09b96d06e933d0db9163a00bfa119160a482810192602092919082900301818660325a03f4156100025750506040515195945050505050565b604080516000815290519081900360200190f35b61038b6004356024356000805433600160a060020a0390811691161461039f57610002565b600160a060020a03600435166000908152600160205260409020545b60408051918252519081900360200190f35b61038b6004356024356000805433600160a060020a039081169116146103ce57610002565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db6388d5fecb600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b610231600435602435600160a060020a038281166000908152600260209081526040808320938516835292905220545b92915050565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561037d5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b604080519115158252519081900360200190f35b50600160a060020a03821660009081526001602081905260409091208054830190556003805483019055610317565b600160a060020a038316600090815260016020526040902054821161040a57506040600020805482900390556003805482900390556001610317565b50600061031756", "function_sighashes": ["0x06fdde03", "0x095ea7b3", "0x18160ddd", "0x23b872dd", "0x313ce567", "0x475a9fa9", "0x70a08231", "0x721a37d2", "0x95d89b41", "0xa9059cbb", "0xdd62ed3e"], "is_erc20": true, "is_erc721": false, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "contract_2112234_0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "item_timestamp": "2016-08-21T10:13:48Z"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"type": "token", "address": "0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "symbol": "", "name": "", "decimals": 0, "total_supply": 0, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "token_2112234_0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "item_timestamp": "2016-08-21T10:13:48Z"}
{"type": "token", "address": "0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "symbol": "", "name": "", "decimals": 0, "total_supply": 0, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "token_2112234_0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "item_timestamp": "2016-08-21T10:13:48Z"}
Loading