Merge pull request #2124 from carver/upgrade-cruft

Upgrade cruft & fix a bunch of flaky tests

.circleci/build_geth.sh

@@ -5,8 +5,9 @@ pip install --user py-geth>=2.1.0
 export GOROOT=/usr/local/go
 export GETH_BINARY="$HOME/.py-geth/geth-$GETH_VERSION/bin/geth"
 if [ ! -e "$GETH_BINARY" ]; then
-  curl -O https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz
-  tar xvf go1.10.linux-amd64.tar.gz
+  export GO_VERSION="1.13"
+  wget "https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz"
+  tar zxvf go$GO_VERSION.linux-amd64.tar.gz
   sudo chown -R root:root ./go
   sudo mv go /usr/local
   sudo ln -s /usr/local/go/bin/go /usr/local/bin/go

.circleci/config.yml

@@ -208,7 +208,7 @@ jobs:
       - image: circleci/python:3.7
         environment:
           TOXENV: py37-sync_integration
-          GETH_VERSION: v1.9.6
+          GETH_VERSION: v1.10.1
   py37-long_run_integration:
     <<: *common
     docker:

newsfragments/2124.internal.rst

@@ -0,0 +1 @@
+Upgrade some dependencies: geth to v1.10.0, eth-tester to v0.5.0-beta.3, pytest-xdist to v1.34.0, and coincurve to v15. Also fixed a couple flaky tests and removed the geth bootnodes (because of unresolved issues on Trinity's end, probably partially due to not having a eth/66 implementation).

p2p/constants.py

@@ -53,15 +53,16 @@
     'enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303',   # noqa: E501
 
     # Geth Bootnodes
+    # Disabled until eth/66 is implemented
     # from https://github.com/ethereum/go-ethereum/blob/1bed5afd92c22a5001aff01620671caccd94a6f8/params/bootnodes.go#L22  # noqa: E501
-    "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303",  # noqa: E501
-    "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303",  # noqa: E501
-    "enode://ca6de62fce278f96aea6ec5a2daadb877e51651247cb96ee310a318def462913b653963c155a0ef6c7d50048bba6e6cea881130857413d9f50a621546b590758@34.255.23.113:30303",  # noqa: E501
-    "enode://279944d8dcd428dffaa7436f25ca0ca43ae19e7bcf94a8fb7d1641651f92d121e972ac2e8f381414b80cc8e5555811c2ec6e1a99bb009b3f53c4c69923e11bd8@35.158.244.151:30303",  # noqa: E501
-    "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303",  # noqa: E501
-    "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303",  # noqa: E501
-    "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303",  # noqa: E501
-    "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303",  # noqa: E501
+    # "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303",  # noqa: E501
+    # "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303",  # noqa: E501
+    # "enode://ca6de62fce278f96aea6ec5a2daadb877e51651247cb96ee310a318def462913b653963c155a0ef6c7d50048bba6e6cea881130857413d9f50a621546b590758@34.255.23.113:30303",  # noqa: E501
+    # "enode://279944d8dcd428dffaa7436f25ca0ca43ae19e7bcf94a8fb7d1641651f92d121e972ac2e8f381414b80cc8e5555811c2ec6e1a99bb009b3f53c4c69923e11bd8@35.158.244.151:30303",  # noqa: E501
+    # "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303",  # noqa: E501
+    # "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303",  # noqa: E501
+    # "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303",  # noqa: E501
+    # "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303",  # noqa: E501
 
     # Parity Bootnodes
     # from https://raw.githubusercontent.com/paritytech/parity-ethereum/master/ethcore/res/ethereum/foundation.json  # noqa: E501

pytest.ini

@@ -7,3 +7,4 @@ log_date_format = %m-%d %H:%M:%S
 markers =
   slow: test is known to be excessively slow...
 timeout = 300
+timeout_method = thread

setup.py

@@ -13,7 +13,7 @@
         "asyncio-cancel-token>=0.2,<0.3",
         "async_lru>=0.1.0,<1.0.0",
         "cached-property>=1.5.1,<2",
-        "coincurve>=10.0.0,<11.0.0",
+        "coincurve>=15.0.0,<16.0.0",
         # cryptography does not use semver and allows breaking changes within `0.3` version bumps.
         "cryptography>=3.0,<3.2",
         "eth-enr>=0.3.0,<0.4",
@@ -36,7 +36,6 @@
         "asyncio-run-in-process==0.1.0a10",
         "bloom-filter==1.3",
         "cachetools>=3.1.0,<4.0.0",
-        "coincurve>=10.0.0,<11.0.0",
         "eth-utils>=1.9.3,<2",
         "ipython>=7.8.0,<7.10.0",  # attach fails with v7.10.{0,1}
         "jsonschema>=3.2,<4",
@@ -61,14 +60,13 @@
         "pexpect>=4.6, <5",
         "factory-boy==2.12.0",
         "pytest>=5.3.0,<5.4",
-        "pytest-cov>=2.8.1,<2.9",
+        "pytest-cov>=2.11.1,<3",
         "pytest-mock>=1.12.1,<1.13",
-        "pytest-randomly>=3.1.0,<3.2",
+        "pytest-randomly>=3.3.0,<4",
         "pytest-timeout>=1.4.2,<2",
         "pytest-watch>=4.2.0,<4.3",
-        # xdist pinned at <1.29 due to: https://github.com/pytest-dev/pytest-xdist/issues/472
-        "pytest-xdist>=1.29.0,<1.30",
-        "eth-tester==0.5.0b2",
+        "pytest-xdist>=1.34.0,<2",
+        "eth-tester==0.5.0b3",
     ],
     # We have to keep some separation between trio and asyncio based tests
     # because `pytest-asyncio` is greedy and tries to run all asyncio fixtures.

tests/core/test_ipc_log_listener.py

@@ -52,7 +52,8 @@ def do_other_process_logging(ipc_path):
         assert len(handler.logs) == 0
         proc.start()
         proc.join()
-        assert len(handler.logs) == 3
+
+    assert len(handler.logs) == 3
 
     error_log, info_log, debug_log = handler.logs
 

tests/integration/test_etherscan_checkpoint_resolver.py

@@ -1,3 +1,5 @@
+import warnings
+
 import pytest
 
 from eth_utils import (
@@ -14,6 +16,9 @@
     GOERLI_NETWORK_ID,
     ROPSTEN_NETWORK_ID,
 )
+from trinity.components.builtin.syncer.etherscan_api import (
+    EtherscanAPIError,
+)
 
 
 # These are just arbitrarily choosen scores that we know can serve as a rough validity check.
@@ -31,9 +36,13 @@
     )
 )
 def test_parse_checkpoint(uri, network_id, min_expected_score):
-    checkpoint = parse_checkpoint_uri(uri, network_id)
-    assert checkpoint.score >= min_expected_score
-    assert is_block_hash(encode_hex(checkpoint.block_hash))
+    try:
+        checkpoint = parse_checkpoint_uri(uri, network_id)
+    except EtherscanAPIError as e:
+        warnings.warn(UserWarning(f'Etherscan API issue: "{e}"'))
+    else:
+        assert checkpoint.score >= min_expected_score
+        assert is_block_hash(encode_hex(checkpoint.block_hash))
 
 
 @pytest.mark.parametrize(
@@ -43,5 +52,9 @@ def test_parse_checkpoint(uri, network_id, min_expected_score):
     )
 )
 def test_get_clique_checkpoint_block_number(network_id, epoch_length):
-    block = get_checkpoint_block_byetherscan(network_id)
-    assert to_int(hexstr=block.get('number')) % epoch_length == 0
+    try:
+        block = get_checkpoint_block_byetherscan(network_id)
+    except EtherscanAPIError as e:
+        warnings.warn(UserWarning(f'Etherscan API issue: "{e}"'))
+    else:
+        assert to_int(hexstr=block.get('number')) % epoch_length == 0

tests/integration/test_sync.py

@@ -82,7 +82,7 @@ def geth_binary():
 def geth_command_arguments(geth_binary, geth_ipc_path, geth_datadir, geth_port):
     return (
         geth_binary,
-        '--testnet',
+        '--ropsten',
         '--datadir', str(geth_datadir),
         '--ipcpath', geth_ipc_path,
         '--nodiscover',
@@ -140,7 +140,7 @@ async def test_sync_integration(request, caplog, geth_ipc_path, enode, geth_proc
 
     The fixture for this test was generated with:
 
-        geth --testnet --syncmode full
+        geth --ropsten --syncmode full
 
     It only needs the first 11 blocks for this test to succeed.
     """

tests/p2p/test_token_bucket.py

@@ -42,23 +42,25 @@ async def test_token_bucket_initial_tokens():
 
 @pytest.mark.asyncio
 async def test_token_bucket_hits_limit():
-    bucket = TokenBucket(1000, 10)
+    CAPACITY = 50
+    TOKENS_PER_SECOND = 1000
+    bucket = TokenBucket(TOKENS_PER_SECOND, CAPACITY)
 
-    bucket.take_nowait(10)
+    bucket.take_nowait(CAPACITY)
     start_at = time.perf_counter()
-    # first 10 tokens should be roughly instant
-    # next 10 tokens should each take 1/1000th second each to generate.
+    # first CAPACITY tokens should be roughly instant
+    # next CAPACITY tokens should each take 1/TOKENS_PER_SECOND second each to generate.
     while True:
-        if bucket.can_take(10):
+        if bucket.can_take(CAPACITY):
             break
         else:
             await asyncio.sleep(0)
 
     end_at = time.perf_counter()
 
-    # we use a zero-measure of 20 to account for the loop overhead.
-    zero = await measure_zero(10)
-    expected_delta = 10 / 1000 + zero
+    # we use a zero-measure of CAPACITY loops to account for the loop overhead.
+    zero = await measure_zero(CAPACITY)
+    expected_delta = CAPACITY / TOKENS_PER_SECOND + zero
     delta = end_at - start_at
 
     # allow up to 10% difference in expected time

tox.ini

@@ -29,27 +29,27 @@ setenv =
 passenv =
     TRAVIS_EVENT_TYPE
 commands=
-    eth1-core: pytest -n 4 {posargs:tests/core/}
-    p2p: pytest -n 4 {posargs:tests/p2p}
-    p2p-trio: pytest -n 4 {posargs:tests-trio/p2p-trio}
-    eth1-components: pytest -n 4 {posargs:tests/components/tx_pool/}
+    eth1-core: pytest --forked -n 4 {posargs:tests/core/}
+    p2p: pytest --forked -n 4 {posargs:tests/p2p}
+    p2p-trio: pytest --forked -n 4 {posargs:tests-trio/p2p-trio}
+    eth1-components: pytest --forked -n 4 {posargs:tests/components/tx_pool/}
     # Fixtures that test transitioning from one VM to another. These are otherwise filtered out by
     # the --fork <ForkName> mechanism because they have a fork such as `FrontierToHomesteadAt5`
-    rpc-state-fork-transition: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'BlockchainTests/TransitionTests'}
+    rpc-state-fork-transition: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'BlockchainTests/TransitionTests'}
     # Fork/VM-specific state transition tests; long-running categories run separately!
-    rpc-state-frontier: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Frontier -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
-    rpc-state-homestead: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Homestead -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
-    rpc-state-tangerine_whistle: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork EIP150 -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
-    rpc-state-spurious_dragon: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork EIP158 -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-frontier: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Frontier -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-homestead: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Homestead -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-tangerine_whistle: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork EIP150 -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-spurious_dragon: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork EIP158 -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
     # The following test seems to consume a lot of memory. Restricting to 3 processes reduces crashes
-    rpc-state-byzantium: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Byzantium -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
-    rpc-state-constantinople: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Constantinople -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
-    rpc-state-petersburg: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork ConstantinopleFix -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
-    rpc-state-istanbul: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Istanbul -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-byzantium: pytest --forked -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Byzantium -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-constantinople: pytest --forked -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Constantinople -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-petersburg: pytest --forked -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork ConstantinopleFix -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
+    rpc-state-istanbul: pytest --forked -n 2 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Istanbul -k 'not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
     # Long-running categories.
-    rpc-state-quadratic: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stQuadraticComplexityTest'}
-    rpc-state-sstore: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stSStoreTest'}
-    rpc-state-zero_knowledge: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stZeroKnowledge'}
+    rpc-state-quadratic: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stQuadraticComplexityTest'}
+    rpc-state-sstore: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stSStoreTest'}
+    rpc-state-zero_knowledge: pytest --forked -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stZeroKnowledge'}
     sync_integration: pytest -s --integration {posargs:tests/integration/test_sync.py --log-cli-level=debug}
 
 deps = .[p2p,trinity,test,test-asyncio]
@@ -60,11 +60,11 @@ basepython =
 
 [testenv:py38-p2p-trio]
 deps = .[p2p,test,test-trio]
-commands = pytest -n 4 {posargs:tests-trio/p2p-trio}
+commands = pytest --forked -n 4 {posargs:tests-trio/p2p-trio}
 
 [testenv:py37-p2p-trio]
 deps = .[p2p,test,test-trio]
-commands = pytest -n 4 {posargs:tests-trio/p2p-trio}
+commands = pytest --forked -n 4 {posargs:tests-trio/p2p-trio}
 
 [testenv:py37-docs]
 whitelist_externals=

trinity/sync/beam/chain.py

@@ -810,10 +810,11 @@ async def import_block(
             self.metrics_registry.counter('trinity.sync/block_witness_hashes_missing').inc()
             self.logger.info("Missing witness for %s. Attempting to fetch during import", block)
             preferred_peer = None
-            self.manager.run_task(
-                self._event_bus.request,
-                FetchBlockWitness(preferred_peer, block.hash, block.number),
-            )
+            if self.manager.is_running:
+                self.manager.run_task(
+                    self._event_bus.request,
+                    FetchBlockWitness(preferred_peer, block.hash, block.number),
+                )
         else:
             block_witness_uncollected = self._state_downloader._get_unique_missing_hashes(
                 wit_hashes)