Add support for Web3 v6

ethereumetl/jobs/export_origin_job.py

@@ -108,11 +108,11 @@ def _export_batch(self, block_number_batch):
                 except ValueError as e:
                     if str(e) == "{'code': -32000, 'message': 'the method is currently not implemented: eth_newFilter'}":
                         self._supports_eth_newFilter = False
-                        events = self.web3.eth.getLogs(filter_params)
+                        events = self.web3.eth.get_logs(filter_params)
                     else:
                         raise(e)
             else:
-                events = self.web3.eth.getLogs(filter_params)
+                events = self.web3.eth.get_logs(filter_params)
             for event in events:
                 log = self.receipt_log_mapper.web3_dict_to_receipt_log(event)
                 listing, shop_products = self.event_extractor.extract_event_from_log(log, batch['contract_version'])
@@ -124,7 +124,7 @@ def _export_batch(self, block_number_batch):
                     self.shop_product_exporter.export_item(item)
 
             if self._supports_eth_newFilter:
-                self.web3.eth.uninstallFilter(event_filter.filter_id)
+                self.web3.eth.uninstall_filter(event_filter.filter_id)
 
     def _end(self):
         self.batch_work_executor.shutdown()

ethereumetl/jobs/export_token_transfers_job.py

@@ -81,7 +81,7 @@ def _export_batch(self, block_number_batch):
         except ValueError as e:
             if str(e) == "{'code': -32000, 'message': 'the method is currently not implemented: eth_newFilter'}":
                 self._supports_eth_newFilter = False
-                events = self.web3.eth.getLogs(filter_params)
+                events = self.web3.eth.get_logs(filter_params)
             else:
                 raise(e)
         for event in events:
@@ -91,7 +91,7 @@ def _export_batch(self, block_number_batch):
                 self.item_exporter.export_item(self.token_transfer_mapper.token_transfer_to_dict(token_transfer))
 
         if self._supports_eth_newFilter:
-            self.web3.eth.uninstallFilter(event_filter.filter_id)
+            self.web3.eth.uninstall_filter(event_filter.filter_id)
 
     def _end(self):
         self.batch_work_executor.shutdown()

ethereumetl/jobs/export_traces_job.py

@@ -87,11 +87,10 @@ def _export_batch(self, block_number_batch):
 
         # TODO: Change to traceFilter when this issue is fixed
         # https://github.com/paritytech/parity-ethereum/issues/9822
-        json_traces = self.web3.parity.traceBlock(block_number)
+        json_traces = self.web3.tracing.trace_block(block_number)
 
         if json_traces is None:
             raise ValueError('Response from the node is None. Is the node fully synced? Is the node started with tracing enabled? Is trace_block API enabled?')
-
         traces = [self.trace_mapper.json_dict_to_trace(json_trace) for json_trace in json_traces]
         all_traces.extend(traces)
 

ethereumetl/mappers/trace_mapper.py

@@ -23,15 +23,15 @@
 
 from ethereumetl.domain.trace import EthTrace
 from ethereumetl.mainnet_daofork_state_changes import DAOFORK_BLOCK_NUMBER
-from ethereumetl.utils import hex_to_dec, to_normalized_address
+from ethereumetl.utils import hex_to_dec, to_normalized_address, hex_to_str
 
 
 class EthTraceMapper(object):
     def json_dict_to_trace(self, json_dict):
         trace = EthTrace()
 
         trace.block_number = json_dict.get('blockNumber')
-        trace.transaction_hash = json_dict.get('transactionHash')
+        trace.transaction_hash = hex_to_str(json_dict.get('transactionHash'))
         trace.transaction_index = json_dict.get('transactionPosition')
         trace.subtraces = json_dict.get('subtraces')
         trace.trace_address = json_dict.get('traceAddress', [])
@@ -62,12 +62,12 @@ def json_dict_to_trace(self, json_dict):
         if trace_type == 'call':
             trace.call_type = action.get('callType')
             trace.to_address = to_normalized_address(action.get('to'))
-            trace.input = action.get('input')
-            trace.output = result.get('output')
+            trace.input = hex_to_str(action.get('input'))
+            trace.output = hex_to_str(result.get('output'))
         elif trace_type == 'create':
             trace.to_address = result.get('address')
-            trace.input = action.get('init')
-            trace.output = result.get('code')
+            trace.input = hex_to_str(action.get('init'))
+            trace.output = hex_to_str(result.get('code'))
         elif trace_type == 'suicide':
             trace.from_address = to_normalized_address(action.get('address'))
             trace.to_address = to_normalized_address(action.get('refundAddress'))
@@ -132,10 +132,8 @@ def _iterate_transaction_trace(self, block_number, tx_index, tx_trace, trace_add
 
         trace.from_address = to_normalized_address(tx_trace.get('from'))
         trace.to_address = to_normalized_address(tx_trace.get('to'))
-
-        trace.input = tx_trace.get('input')
-        trace.output = tx_trace.get('output')
-
+        trace.input = hex_to_str(tx_trace.get('input'))
+        trace.output = hex_to_str(tx_trace.get('output'))
         trace.value = hex_to_dec(tx_trace.get('value'))
         trace.gas = hex_to_dec(tx_trace.get('gas'))
         trace.gas_used = hex_to_dec(tx_trace.get('gasUsed'))

ethereumetl/service/eth_service.py

@@ -71,13 +71,13 @@ def __init__(self, web3):
 
     def get_first_point(self):
         # Ignore the genesis block as its timestamp is 0
-        return block_to_point(self._web3.eth.getBlock(1))
+        return block_to_point(self._web3.eth.get_block(1))
 
     def get_last_point(self):
-        return block_to_point(self._web3.eth.getBlock('latest'))
+        return block_to_point(self._web3.eth.get_block('latest'))
 
     def get_point(self, x):
-        return block_to_point(self._web3.eth.getBlock(x))
+        return block_to_point(self._web3.eth.get_block(x))
 
 
 def block_to_point(block):

ethereumetl/service/eth_token_service.py

@@ -35,7 +35,7 @@ def __init__(self, web3, function_call_result_transformer=None):
         self._function_call_result_transformer = function_call_result_transformer
 
     def get_token(self, token_address):
-        checksum_address = self._web3.toChecksumAddress(token_address)
+        checksum_address = self._web3.to_checksum_address(token_address)
         contract = self._web3.eth.contract(address=checksum_address, abi=ERC20_ABI)
         contract_alternative_1 = self._web3.eth.contract(address=checksum_address, abi=ERC20_ABI_ALTERNATIVE_1)
 

ethereumetl/service/origin_extractor.py

@@ -1,7 +1,7 @@
 import base58
 import logging
 
-from ethereumetl.utils import hex_to_dec, to_normalized_address
+from ethereumetl.utils import hex_to_dec, hex_to_str
 from ethereumetl.ipfs.origin import get_origin_marketplace_data
 
 #
@@ -55,7 +55,7 @@ def extract_event_from_log(self, receipt_log, contract_version):
             return None, []
 
         listing_id = hex_to_dec(topics[2])
-        ipfs_hash = hex_to_ipfs_hash(receipt_log.data)
+        ipfs_hash = hex_to_ipfs_hash(hex_to_str(receipt_log.data))
 
         full_listing_id = compose_listing_id(1, contract_version, listing_id)
         marketplace_listing, shop_products = get_origin_marketplace_data(receipt_log, full_listing_id, self.ipfs_client, ipfs_hash)

ethereumetl/service/token_transfer_extractor.py

@@ -25,7 +25,7 @@
 from builtins import map
 
 from ethereumetl.domain.token_transfer import EthTokenTransfer
-from ethereumetl.utils import chunk_string, hex_to_dec, to_normalized_address
+from ethereumetl.utils import chunk_string, hex_to_dec, hex_to_str, to_normalized_address
 
 # https://ethereum.stackexchange.com/questions/12553/understanding-logs-and-log-blooms
 TRANSFER_EVENT_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
@@ -42,7 +42,7 @@ def extract_transfer_from_log(self, receipt_log):
 
         if (topics[0]).casefold() == TRANSFER_EVENT_TOPIC:
             # Handle unindexed event fields
-            topics_with_data = topics + split_to_words(receipt_log.data)
+            topics_with_data = topics + split_to_words(hex_to_str(receipt_log.data))
             # if the number of topics and fields in data part != 4, then it's a weird event
             if len(topics_with_data) != 4:
                 logger.warning("The number of topics and data parts is not equal to 4 in log {} of transaction {}"

ethereumetl/streaming/eth_streamer_adapter.py

@@ -38,7 +38,7 @@ def open(self):
 
     def get_current_block_number(self):
         w3 = build_web3(self.batch_web3_provider)
-        return int(w3.eth.getBlock("latest").number)
+        return int(w3.eth.get_block("latest").number)
 
     def export_all(self, start_block, end_block):
         # Export blocks and transactions

ethereumetl/utils.py

@@ -23,11 +23,25 @@
 
 import itertools
 import warnings
+from eth_utils import to_int, to_text, text_if_str
+from hexbytes import HexBytes
 
 from ethereumetl.misc.retriable_value_error import RetriableValueError
 
 
+def hex_to_str(hex_string):
+    if isinstance(hex_string, HexBytes):
+        return hex_string.hex()
+
+    if hex_string is not None:
+        return text_if_str(to_text, hex_string)
+    return hex_string
+
+
 def hex_to_dec(hex_string):
+    if type(hex_string) is int:
+        return to_int(hex_string)
+
     if hex_string is None:
         return None
     try:

setup.py

@@ -31,22 +31,22 @@ def read(fname):
     keywords='ethereum',
     python_requires='>=3.7.2,<4',
     install_requires=[
-        'web3>=5.29,<6',
-        'eth-utils==1.10',
-        'eth-abi>=2.2.0,<3.0.0',
+        'web3>=6,<7',
+        'eth-utils>=4.0.0',
+        'eth-abi>=5',
         # TODO: This has to be removed when "ModuleNotFoundError: No module named 'eth_utils.toolz'" is fixed at eth-abi
         'python-dateutil>=2.8.0,<3',
         'click>=8.0.4,<9',
-        'ethereum-dasm==0.1.4',
+        'ethereum-dasm==0.1.5',
         'urllib3<2',
         'base58',
         'requests'
     ],
     extras_require={
         'streaming': [
             'timeout-decorator==0.4.1',
-            'google-cloud-pubsub==2.13.0',
-            'google-cloud-storage==1.33.0',
+            'google-cloud-pubsub==2.13.12',
+            'google-cloud-storage==1.44.0',
             'kafka-python==2.0.2',
             'sqlalchemy==1.4',
             'pg8000==1.16.6',

tests/resources/test_export_traces_job/block_with_create/expected_traces.csv

@@ -1,4 +1,4 @@
 block_number,transaction_hash,transaction_index,from_address,to_address,value,input,output,trace_type,call_type,reward_type,gas,gas_used,subtraces,trace_address,error,status,trace_id
 1000690,0x2c3cfb8d7f3039fcad64bd231beb0229af55f9114d74aef12a6f228ad1dac100,0,0xaf21e07e5a929d16026a7b4d88f3906a8d2e4942,0x5b3c526b152b1f3d8eabe2ec27f49b904ad51cad,64655529900000002048,0x,0x,call,call,,0,0,0,,,1,call_0x2c3cfb8d7f3039fcad64bd231beb0229af55f9114d74aef12a6f228ad1dac100_
-1000690,0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871,1,0xacdee28d8ca76187883831a37f551a5904cdf191,0xa7e3cf952ea8d9438a26ee346c295f1ada328ae1,0,0x606060405260026101086000505560405161015638038061015683398101604052805160805160a051919092019190808383815160019081018155600090600160a060020a0332169060029060038390559183525061010260205260408220555b82518110156100eb57828181518110156100025790602001906020020151600160a060020a03166002600050826002016101008110156100025790900160005081905550806002016101026000506000858481518110156100025790602001906020020151600160a060020a0316815260200190815260200160002060005081905550600101610060565b81600060005081905550505050806101056000508190555061010f62015180420490565b61010755505050506031806101256000396000f3003660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f30000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052b7d2dcc80cd2e40000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000acdee28d8ca76187883831a37f551a5904cdf191,0x3660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f3,create,,,954720,160631,0,,,1,create_0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871_
+1000690,0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871,1,0xacdee28d8ca76187883831a37f551a5904cdf191,0xa7e3CF952EA8D9438a26EE346c295F1ada328AE1,0,0x606060405260026101086000505560405161015638038061015683398101604052805160805160a051919092019190808383815160019081018155600090600160a060020a0332169060029060038390559183525061010260205260408220555b82518110156100eb57828181518110156100025790602001906020020151600160a060020a03166002600050826002016101008110156100025790900160005081905550806002016101026000506000858481518110156100025790602001906020020151600160a060020a0316815260200190815260200160002060005081905550600101610060565b81600060005081905550505050806101056000508190555061010f62015180420490565b61010755505050506031806101256000396000f3003660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f30000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052b7d2dcc80cd2e40000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000acdee28d8ca76187883831a37f551a5904cdf191,0x3660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f3,create,,,954720,160631,0,,,1,create_0xe4de004685c7b5fe48854cfb54cd59686f3f7cbf95e68c3984dfd672fe677871_
 1000690,,,,0xf8b483dba2c3b7176a3da549ad41a48bb3121069,5000000000000000000,,,reward,,block,,,0,,,1,reward_1000690_0

tests/resources/test_stream/blocks_2112234_2112234/expected_contracts.json

@@ -1 +1 @@
-{"type": "contract", "address": "0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "bytecode": "0x6060604052361561008d5760e060020a600035046306fdde03811461008f578063095ea7b3146100a557806318160ddd1461012457806323b872dd1461012f578063313ce567146101dc578063475a9fa9146101f057806370a0823114610215578063721a37d21461024357806395d89b411461008f578063a9059cbb14610268578063dd62ed3e146102e7575b005b61031d6040805160208101909152600081525b90565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db63c6605267600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b6102316003546100a2565b61038b60043560243560443560008054604080517fa00bfa1100000000000000000000000000000000000000000000000000000000815260016004820152600160a060020a038781166024830152868116604483015260648201869052929092166084830152517319ee743d2e356d5f0e4d97cc09b96d06e933d0db9163a00bfa119160a482810192602092919082900301818660325a03f4156100025750506040515195945050505050565b604080516000815290519081900360200190f35b61038b6004356024356000805433600160a060020a0390811691161461039f57610002565b600160a060020a03600435166000908152600160205260409020545b60408051918252519081900360200190f35b61038b6004356024356000805433600160a060020a039081169116146103ce57610002565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db6388d5fecb600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b610231600435602435600160a060020a038281166000908152600260209081526040808320938516835292905220545b92915050565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561037d5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b604080519115158252519081900360200190f35b50600160a060020a03821660009081526001602081905260409091208054830190556003805483019055610317565b600160a060020a038316600090815260016020526040902054821161040a57506040600020805482900390556003805482900390556001610317565b50600061031756", "function_sighashes": ["0x06fdde03", "0x095ea7b3", "0x18160ddd", "0x23b872dd", "0x313ce567", "0x475a9fa9", "0x70a08231", "0x721a37d2", "0x95d89b41", "0xa9059cbb", "0xdd62ed3e"], "is_erc20": true, "is_erc721": false, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "contract_2112234_0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "item_timestamp": "2016-08-21T10:13:48Z"}
+{"type": "contract", "address": "0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "bytecode": "0x6060604052361561008d5760e060020a600035046306fdde03811461008f578063095ea7b3146100a557806318160ddd1461012457806323b872dd1461012f578063313ce567146101dc578063475a9fa9146101f057806370a0823114610215578063721a37d21461024357806395d89b411461008f578063a9059cbb14610268578063dd62ed3e146102e7575b005b61031d6040805160208101909152600081525b90565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db63c6605267600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b6102316003546100a2565b61038b60043560243560443560008054604080517fa00bfa1100000000000000000000000000000000000000000000000000000000815260016004820152600160a060020a038781166024830152868116604483015260648201869052929092166084830152517319ee743d2e356d5f0e4d97cc09b96d06e933d0db9163a00bfa119160a482810192602092919082900301818660325a03f4156100025750506040515195945050505050565b604080516000815290519081900360200190f35b61038b6004356024356000805433600160a060020a0390811691161461039f57610002565b600160a060020a03600435166000908152600160205260409020545b60408051918252519081900360200190f35b61038b6004356024356000805433600160a060020a039081169116146103ce57610002565b61038b60043560243560007319ee743d2e356d5f0e4d97cc09b96d06e933d0db6388d5fecb600160005085856040518460e060020a0281526004018084815260200183600160a060020a0316815260200182815260200193505050506020604051808303818660325a03f4156100025750506040515191506103179050565b610231600435602435600160a060020a038281166000908152600260209081526040808320938516835292905220545b92915050565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561037d5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b604080519115158252519081900360200190f35b50600160a060020a03821660009081526001602081905260409091208054830190556003805483019055610317565b600160a060020a038316600090815260016020526040902054821161040a57506040600020805482900390556003805482900390556001610317565b50600061031756", "function_sighashes": ["0x06fdde03", "0x095ea7b3", "0x18160ddd", "0x23b872dd", "0x313ce567", "0x475a9fa9", "0x70a08231", "0x721a37d2", "0x95d89b41", "0xa9059cbb", "0xdd62ed3e"], "is_erc20": true, "is_erc721": false, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "contract_2112234_0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "item_timestamp": "2016-08-21T10:13:48Z"}

tests/resources/test_stream/blocks_2112234_2112234/expected_tokens.json

@@ -1 +1 @@
-{"type": "token", "address": "0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "symbol": "", "name": "", "decimals": 0, "total_supply": 0, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "token_2112234_0xdbdacfc9eb9d42559ac1efbdb40460c728139e6a", "item_timestamp": "2016-08-21T10:13:48Z"}
+{"type": "token", "address": "0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "symbol": "", "name": "", "decimals": 0, "total_supply": 0, "block_number": 2112234, "block_timestamp": 1471774428, "block_hash": "0xd279067d9852394d6b6c00b13c49696503c9618d3ed3b23c6b1b1321857ddd92", "item_id": "token_2112234_0xdbDacFc9EB9d42559ac1EFBDb40460c728139E6A", "item_timestamp": "2016-08-21T10:13:48Z"}