Expose max_retries option to the cli

ethereumetl/cli/export_all.py

@@ -114,11 +114,12 @@ def get_partitions(start, end, partition_batch_size, provider_uri):
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output-dir', default='output', show_default=True, type=str, help='Output directory, partitioned in Hive style.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-B', '--export-batch-size', default=100, show_default=True, type=int, help='The number of requests in JSON RPC batches.')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
-def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size,
+def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, max_retries, export_batch_size,
                chain='ethereum'):
     """Exports all data for a range of blocks."""
     provider_uri = check_classic_provider_uri(chain, provider_uri)
     export_all_common(get_partitions(start, end, partition_batch_size, provider_uri),
-                      output_dir, provider_uri, max_workers, export_batch_size)
+                      output_dir, provider_uri, max_workers, export_batch_size, max_retries)

ethereumetl/cli/export_blocks_and_transactions.py

@@ -41,13 +41,14 @@
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('--blocks-output', default=None, show_default=True, type=str,
               help='The output file for blocks. If not provided blocks will not be exported. Use "-" for stdout')
 @click.option('--transactions-output', default=None, show_default=True, type=str,
               help='The output file for transactions. '
                    'If not provided transactions will not be exported. Use "-" for stdout')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
-def export_blocks_and_transactions(start_block, end_block, batch_size, provider_uri, max_workers, blocks_output,
+def export_blocks_and_transactions(start_block, end_block, batch_size, provider_uri, max_workers, max_retries, blocks_output,
                                    transactions_output, chain='ethereum'):
     """Exports blocks and transactions."""
     provider_uri = check_classic_provider_uri(chain, provider_uri)
@@ -60,6 +61,7 @@ def export_blocks_and_transactions(start_block, end_block, batch_size, provider_
         batch_size=batch_size,
         batch_web3_provider=ThreadLocalProxy(lambda: get_provider_from_uri(provider_uri, batch=True)),
         max_workers=max_workers,
+        max_retries=max_retries,
         item_exporter=blocks_and_transactions_item_exporter(blocks_output, transactions_output),
         export_blocks=blocks_output is not None,
         export_transactions=transactions_output is not None)

ethereumetl/cli/export_contracts.py

@@ -40,11 +40,12 @@
               help='The file containing contract addresses, one per line.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
-def export_contracts(batch_size, contract_addresses, output, max_workers, provider_uri, chain='ethereum'):
+def export_contracts(batch_size, contract_addresses, output, max_workers, max_retries, provider_uri, chain='ethereum'):
     """Exports contracts bytecode and sighashes."""
     check_classic_provider_uri(chain, provider_uri)
     with smart_open(contract_addresses, 'r') as contract_addresses_file:
@@ -55,6 +56,7 @@ def export_contracts(batch_size, contract_addresses, output, max_workers, provid
             batch_size=batch_size,
             batch_web3_provider=ThreadLocalProxy(lambda: get_provider_from_uri(provider_uri, batch=True)),
             item_exporter=contracts_item_exporter(output),
-            max_workers=max_workers)
+            max_workers=max_workers,
+            max_retries=max_retries)
 
         job.run()

ethereumetl/cli/export_geth_traces.py

@@ -39,17 +39,19 @@
 @click.option('-o', '--output', default='-', show_default=True, type=str,
               help='The output file for geth traces. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-p', '--provider-uri', required=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or http://localhost:8545/')
-def export_geth_traces(start_block, end_block, batch_size, output, max_workers, provider_uri):
+def export_geth_traces(start_block, end_block, batch_size, output, max_workers, max_retries, provider_uri):
     """Exports traces from geth node."""
     job = ExportGethTracesJob(
         start_block=start_block,
         end_block=end_block,
         batch_size=batch_size,
         batch_web3_provider=ThreadLocalProxy(lambda: get_provider_from_uri(provider_uri, batch=True)),
         max_workers=max_workers,
+        max_retries=max_retries,
         item_exporter=geth_traces_item_exporter(output))
 
     job.run()

ethereumetl/cli/export_origin.py

@@ -40,9 +40,10 @@
 @click.option('--marketplace-output', default='-', show_default=True, type=str, help='The output file for marketplace data. If not specified stdout is used.')
 @click.option('--shop-output', default='-', show_default=True, type=str, help='The output file for shop data. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-p', '--provider-uri', required=True, type=str,
               help='The URI of the web3 provider e.g. file://$HOME/Library/Ethereum/geth.ipc or http://localhost:8545/')
-def export_origin(start_block, end_block, batch_size, marketplace_output, shop_output, max_workers, provider_uri):
+def export_origin(start_block, end_block, batch_size, marketplace_output, shop_output, max_workers, max_retries, provider_uri):
     """Exports Origin Protocol data."""
     job = ExportOriginJob(
         start_block=start_block,
@@ -52,5 +53,6 @@ def export_origin(start_block, end_block, batch_size, marketplace_output, shop_o
         ipfs_client=get_origin_ipfs_client(),
         marketplace_listing_exporter=origin_marketplace_listing_item_exporter(marketplace_output),
         shop_product_exporter=origin_shop_product_item_exporter(shop_output),
-        max_workers=max_workers)
+        max_workers=max_workers,
+        max_retries=max_retries)
     job.run()

ethereumetl/cli/export_receipts_and_logs.py

@@ -42,13 +42,14 @@
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('--receipts-output', default=None, show_default=True, type=str,
               help='The output file for receipts. If not provided receipts will not be exported. Use "-" for stdout')
 @click.option('--logs-output', default=None, show_default=True, type=str,
               help='The output file for receipt logs. '
                    'If not provided receipt logs will not be exported. Use "-" for stdout')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
-def export_receipts_and_logs(batch_size, transaction_hashes, provider_uri, max_workers, receipts_output, logs_output,
+def export_receipts_and_logs(batch_size, transaction_hashes, provider_uri, max_workers, max_retries, receipts_output, logs_output,
                              chain='ethereum'):
     """Exports receipts and logs."""
     provider_uri = check_classic_provider_uri(chain, provider_uri)
@@ -58,6 +59,7 @@ def export_receipts_and_logs(batch_size, transaction_hashes, provider_uri, max_w
             batch_size=batch_size,
             batch_web3_provider=ThreadLocalProxy(lambda: get_provider_from_uri(provider_uri, batch=True)),
             max_workers=max_workers,
+            max_retries=max_retries,
             item_exporter=receipts_and_logs_item_exporter(receipts_output, logs_output),
             export_receipts=receipts_output is not None,
             export_logs=logs_output is not None)

ethereumetl/cli/export_token_transfers.py

@@ -40,10 +40,11 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-p', '--provider-uri', required=True, type=str,
               help='The URI of the web3 provider e.g. file://$HOME/Library/Ethereum/geth.ipc or http://localhost:8545/')
 @click.option('-t', '--tokens', default=None, show_default=True, type=str, multiple=True, help='The list of token addresses to filter by.')
-def export_token_transfers(start_block, end_block, batch_size, output, max_workers, provider_uri, tokens):
+def export_token_transfers(start_block, end_block, batch_size, output, max_workers, max_retries, provider_uri, tokens):
     """Exports ERC20/ERC721 transfers."""
     job = ExportTokenTransfersJob(
         start_block=start_block,
@@ -52,5 +53,6 @@ def export_token_transfers(start_block, end_block, batch_size, output, max_worke
         web3=ThreadLocalProxy(lambda: build_web3(get_provider_from_uri(provider_uri))),
         item_exporter=token_transfers_item_exporter(output),
         max_workers=max_workers,
+        max_retries=max_retries,
         tokens=tokens)
     job.run()

ethereumetl/cli/export_tokens.py

@@ -41,18 +41,20 @@
               help='The file containing token addresses, one per line.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
-def export_tokens(token_addresses, output, max_workers, provider_uri, chain='ethereum'):
+def export_tokens(token_addresses, output, max_workers, max_retries, provider_uri, chain='ethereum'):
     """Exports ERC20/ERC721 tokens."""
     provider_uri = check_classic_provider_uri(chain, provider_uri)
     with smart_open(token_addresses, 'r') as token_addresses_file:
         job = ExportTokensJob(
             token_addresses_iterable=(token_address.strip() for token_address in token_addresses_file),
             web3=ThreadLocalProxy(lambda: build_web3(get_provider_from_uri(provider_uri))),
             item_exporter=tokens_item_exporter(output),
-            max_workers=max_workers)
+            max_workers=max_workers,
+            max_retries=max_retries)
 
         job.run()

ethereumetl/cli/export_traces.py

@@ -40,14 +40,15 @@
 @click.option('-b', '--batch-size', default=5, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('-p', '--provider-uri', required=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/.local/share/io.parity.ethereum/jsonrpc.ipc or http://localhost:8545/')
 @click.option('--genesis-traces/--no-genesis-traces', default=False, show_default=True, help='Whether to include genesis traces')
 @click.option('--daofork-traces/--no-daofork-traces', default=False, show_default=True, help='Whether to include daofork traces')
 @click.option('-t', '--timeout', default=60, show_default=True, type=int, help='IPC or HTTP request timeout.')
 @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.')
-def export_traces(start_block, end_block, batch_size, output, max_workers, provider_uri,
+def export_traces(start_block, end_block, batch_size, output, max_workers, max_retries, provider_uri,
                   genesis_traces, daofork_traces, timeout=60, chain='ethereum'):
     """Exports traces from parity node."""
     if chain == 'classic' and daofork_traces == True:
@@ -60,6 +61,7 @@ def export_traces(start_block, end_block, batch_size, output, max_workers, provi
         web3=ThreadLocalProxy(lambda: build_web3(get_provider_from_uri(provider_uri, timeout=timeout))),
         item_exporter=traces_item_exporter(output),
         max_workers=max_workers,
+        max_retries=max_retries,
         include_genesis_traces=genesis_traces,
         include_daofork_traces=daofork_traces)
 

ethereumetl/cli/extract_contracts.py

@@ -39,7 +39,8 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-def extract_contracts(traces, batch_size, output, max_workers):
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
+def extract_contracts(traces, batch_size, output, max_workers, max_retries):
     """Extracts contracts from traces file."""
 
     set_max_field_size_limit()
@@ -53,6 +54,7 @@ def extract_contracts(traces, batch_size, output, max_workers):
             traces_iterable=traces_iterable,
             batch_size=batch_size,
             max_workers=max_workers,
+            max_retries=max_retries,
             item_exporter=contracts_item_exporter(output))
 
         job.run()

ethereumetl/cli/extract_geth_traces.py

@@ -37,7 +37,9 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-def extract_geth_traces(input, batch_size, output, max_workers):
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
+
+def extract_geth_traces(input, batch_size, output, max_workers, max_retries):
     """Extracts geth traces from JSON lines file."""
     with smart_open(input, 'r') as geth_traces_file:
         if input.endswith('.json'):
@@ -48,6 +50,7 @@ def extract_geth_traces(input, batch_size, output, max_workers):
             traces_iterable=traces_iterable,
             batch_size=batch_size,
             max_workers=max_workers,
+            max_retries=max_retries,
             item_exporter=traces_item_exporter(output))
 
         job.run()

ethereumetl/cli/extract_token_transfers.py

@@ -39,8 +39,9 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('--values-as-strings', default=False, show_default=True, is_flag=True, help='Whether to convert values to strings.')
-def extract_token_transfers(logs, batch_size, output, max_workers, values_as_strings=False):
+def extract_token_transfers(logs, batch_size, output, max_workers, max_retries, values_as_strings=False):
     """Extracts ERC20/ERC721 transfers from logs file."""
     with smart_open(logs, 'r') as logs_file:
         if logs.endswith('.json'):
@@ -52,6 +53,7 @@ def extract_token_transfers(logs, batch_size, output, max_workers, values_as_str
             logs_iterable=logs_reader,
             batch_size=batch_size,
             max_workers=max_workers,
+            max_retries=max_retries,
             item_exporter=token_transfers_item_exporter(output, converters=converters))
 
         job.run()

ethereumetl/cli/stream.py

@@ -50,10 +50,11 @@
 @click.option('-b', '--batch-size', default=10, show_default=True, type=int, help='How many blocks to batch in single request')
 @click.option('-B', '--block-batch-size', default=1, show_default=True, type=int, help='How many blocks to batch in single sync round')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The number of workers')
+@click.option('-r', '--max-retries', default=5, show_default=True, type=int, help='The maximum number of retries')
 @click.option('--log-file', default=None, show_default=True, type=str, help='Log file')
 @click.option('--pid-file', default=None, show_default=True, type=str, help='pid file')
 def stream(last_synced_block_file, lag, provider_uri, output, start_block, entity_types,
-           period_seconds=10, batch_size=2, block_batch_size=10, max_workers=5, log_file=None, pid_file=None):
+           period_seconds=10, batch_size=2, block_batch_size=10, max_workers=5, max_retries=5, log_file=None, pid_file=None):
     """Streams all data types to console or Google Pub/Sub."""
     configure_logging(log_file)
     configure_signals()
@@ -71,6 +72,7 @@ def stream(last_synced_block_file, lag, provider_uri, output, start_block, entit
         item_exporter=create_item_exporters(output),
         batch_size=batch_size,
         max_workers=max_workers,
+        max_retries=max_retries,
         entity_types=entity_types
     )
     streamer = Streamer(