From ba969f9fb8e52b3ad1fc0831abed71ed5305dcae Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 23 Jan 2024 14:16:10 +0000 Subject: [PATCH] chore: make CI green (#3114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building off @vgonkivs's [work to skip tests](https://github.com/celestiaorg/celestia-node/pull/2802), this attempts to get a baseline "green" ci for us. There are still a couple of tests that VERY intermittently flake in CI, but this way they will stand out WHEN they happen and we can track down vs entire thing being red always. This does quite a few things - introduces build tags on swamp tests in each named file to allow us to run parts of the swamp/integration tests independently (ie: `go test ./... -tags=blob` - adds an `integration` tag to allow running all still - splits the integration tests into it's own workflow file (`integration-tests.yml`) which is now triggered from `go-ci.yml` - splits each swamp/integration tests to run as its own job so we can see which are failing/flakey more explicitly - utilizes go's -short test flag to Skip a few swamp/integration tests that are consistently failing. Current we are skipping `TestFullReconstructFromFulls`, `TestFullReconstructFromLights`, `TestSyncStartStopLightWithBridge` which is less than we were originally skipping in https://github.com/celestiaorg/celestia-node/pull/2802/ - plugs in our verbose/debug stuff to integration tests in addition to unit which we had before Unit tests - splits some of the unit tests that were "race flakey" into `*_no_race_test.go` and adds `!race` tag to some others to get to pass consistently when running unit tests with -race flag - macos-latest unit tests still fail on race test ONLY in GitHub actions CI 🤷‍♂️ Next Steps - create issues for each short run / skipped - create issues for any tests that fail race that are NOT the race issue on upstream cosmos-sdk - create issue for macos-latest in GitHub race fail - create issue for macos-latest intermittent fail I think we let this run for a while, then once we see it be consistently green for a bit and isolate any more flakes that pop up, we can toggle on more branch requirements, then in fixing the above issues, we can be green. [Being green is not easy](https://www.youtube.com/watch?v=51BQfPeSK8k). --- .github/workflows/go-ci.yml | 29 ++---- .github/workflows/integration-tests.yml | 115 ++++++++++++++++++++++++ Makefile | 27 +++--- core/fetcher_no_race_test.go | 55 ++++++++++++ core/fetcher_test.go | 43 --------- core/listener_no_race_test.go | 71 +++++++++++++++ core/listener_test.go | 59 ------------ nodebuilder/node_test.go | 2 + nodebuilder/store_test.go | 2 + nodebuilder/tests/api_test.go | 20 ++--- nodebuilder/tests/blob_test.go | 2 + nodebuilder/tests/fraud_test.go | 2 + nodebuilder/tests/helpers_test.go | 35 ++++++++ nodebuilder/tests/nd_test.go | 2 + nodebuilder/tests/p2p_test.go | 7 +- nodebuilder/tests/reconstruct_test.go | 13 ++- nodebuilder/tests/sync_test.go | 12 ++- share/eds/retriever_no_race_test.go | 55 ++++++++++++ share/eds/retriever_test.go | 36 -------- share/p2p/discovery/discovery_test.go | 2 + state/core_access_test.go | 2 + 21 files changed, 390 insertions(+), 201 deletions(-) create mode 100644 .github/workflows/integration-tests.yml create mode 100644 core/fetcher_no_race_test.go create mode 100644 core/listener_no_race_test.go create mode 100644 nodebuilder/tests/helpers_test.go create mode 100644 share/eds/retriever_no_race_test.go diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 3e73290500..5da90de33e 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -14,6 +14,7 @@ concurrency: jobs: setup: + name: Setup runs-on: ubuntu-latest outputs: debug: ${{ steps.debug.outputs.debug }} @@ -104,11 +105,10 @@ jobs: file: ./coverage.txt name: coverage-${{ matrix.os }} - unit_race_test: + unit_test_race: needs: [lint, go_mod_tidy_check] - name: Run Unit Tests with Race Detector + name: Unit Tests with Race Detector (ubuntu-latest) runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v4 @@ -119,24 +119,11 @@ jobs: go-version: ${{ inputs.go-version }} - name: execute test run - run: make test-unit-race + run: make test-unit-race ENABLE_VERBOSE=${{ needs.setup.outputs.debug }} integration_test: + name: Integration Tests needs: [lint, go_mod_tidy_check] - name: Run Integration Tests - runs-on: ubuntu-latest - continue-on-error: true - - steps: - - uses: actions/checkout@v4 - - - name: set up go - uses: actions/setup-go@v5 - with: - go-version: ${{ inputs.go-version }} - - - name: Integration Tests - run: make test-integration - - - name: Integration Tests with Race Detector - run: make test-integration-race + uses: ./.github/workflows/integration-tests.yml + with: + go-version: ${{ inputs.go-version }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000000..cd0fc62385 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,115 @@ +name: Integration Tests + +on: + workflow_call: + inputs: + go-version: + description: 'Go version' + required: true + type: string + +jobs: + api_tests: + name: Integration Tests API + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run API tests + run: make test-integration TAGS=api + + blob_tests: + name: Integration Tests Blob + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run blob tests + run: make test-integration TAGS=blob + + fraud_tests: + name: Integration Tests Fraud + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run fraud tests + run: make test-integration TAGS=fraud + + nd_tests: + name: Integration Tests ND + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run nd tests + run: make test-integration TAGS=nd + + p2p_tests: + name: Integration Tests p2p + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run p2p tests + run: make test-integration TAGS=p2p + + reconstruction_tests: + name: Integration Tests Reconstruction + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run reconstruction tests + run: make test-integration SHORT=true TAGS=reconstruction + + sync_tests: + name: Integration Tests Sync + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run sync tests + run: make test-integration SHORT=true TAGS=sync diff --git a/Makefile b/Makefile index b3b3739cb7..87bcbbf61c 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ PROJECTNAME=$(shell basename "$(PWD)") DIR_FULLPATH=$(shell pwd) versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node" LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" +TAGS=integration +SHORT= ifeq (${PREFIX},) PREFIX := /usr/local endif @@ -13,6 +15,11 @@ else VERBOSE = LOG_AND_FILTER = endif +ifeq ($(SHORT),true) + INTEGRATION_RUN_LENGTH = -short +else + INTEGRATION_RUN_LENGTH = +endif ## help: Get more info on make commands. help: Makefile @echo " Choose a command run in "$(PROJECTNAME)":" @@ -118,29 +125,21 @@ test-unit: ## test-unit-race: Running unit tests with data race detector test-unit-race: @echo "--> Running unit tests with data race detector" - @go test -race `go list ./... | grep -v nodebuilder/tests` + @go test $(VERBOSE) -race -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER) .PHONY: test-unit-race ## test-integration: Running /integration tests located in nodebuilder/tests test-integration: - @echo "--> Running integrations tests" - @go test ./nodebuilder/tests + @echo "--> Running integrations tests $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH)" + @go test $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH) ./nodebuilder/tests .PHONY: test-integration ## test-integration-race: Running integration tests with data race detector located in node/tests test-integration-race: - @echo "--> Running integration tests with data race detector" - @go test -race ./nodebuilder/tests + @echo "--> Running integration tests with data race detector -tags=$(TAGS)" + @go test -race -tags=$(TAGS) ./nodebuilder/tests .PHONY: test-integration-race -## test: Running both unit and integrations tests -test: - @echo "--> Running all tests without data race detector" - @go test ./... - @echo "--> Running all tests with data race detector" - @go test -race ./... -.PHONY: test - ## benchmark: Running all benchmarks benchmark: @echo "--> Running benchmarks" @@ -164,7 +163,6 @@ pb-gen: done; .PHONY: pb-gen - ## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api openrpc-gen: @echo "--> Generating OpenRPC spec" @@ -221,7 +219,6 @@ goreleaser-release: .PHONY: goreleaser-release # Copied from https://github.com/dgraph-io/badger/blob/main/Makefile - USER_ID = $(shell id -u) HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc") JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" diff --git a/core/fetcher_no_race_test.go b/core/fetcher_no_race_test.go new file mode 100644 index 0000000000..890b7c35c1 --- /dev/null +++ b/core/fetcher_no_race_test.go @@ -0,0 +1,55 @@ +//go:build !race + +package core + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/types" +) + +// TestBlockFetcherHeaderValues tests that both the Commit and ValidatorSet +// endpoints are working as intended. +func TestBlockFetcherHeaderValues(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + t.Cleanup(cancel) + + client := StartTestNode(t).Client + fetcher := NewBlockFetcher(client) + + // generate some blocks + newBlockChan, err := fetcher.SubscribeNewBlockEvent(ctx) + require.NoError(t, err) + // read once from channel to generate next block + var h int64 + select { + case evt := <-newBlockChan: + h = evt.Header.Height + case <-ctx.Done(): + require.NoError(t, ctx.Err()) + } + // get Commit from current height + commit, err := fetcher.Commit(ctx, &h) + require.NoError(t, err) + // get ValidatorSet from current height + valSet, err := fetcher.ValidatorSet(ctx, &h) + require.NoError(t, err) + // get next block + var nextBlock types.EventDataSignedBlock + select { + case nextBlock = <-newBlockChan: + case <-ctx.Done(): + require.NoError(t, ctx.Err()) + } + // compare LastCommit from next block to Commit from first block height + assert.Equal(t, nextBlock.Header.LastCommitHash, commit.Hash()) + assert.Equal(t, nextBlock.Header.Height, commit.Height+1) + // compare ValidatorSet hash to the ValidatorsHash from first block height + hexBytes := valSet.Hash() + assert.Equal(t, nextBlock.ValidatorSet.Hash(), hexBytes) + require.NoError(t, fetcher.UnsubscribeNewBlockEvent(ctx)) +} diff --git a/core/fetcher_test.go b/core/fetcher_test.go index 3380dbb402..261b84d78c 100644 --- a/core/fetcher_test.go +++ b/core/fetcher_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/types" ) func TestBlockFetcher_GetBlock_and_SubscribeNewBlockEvent(t *testing.T) { @@ -38,45 +37,3 @@ func TestBlockFetcher_GetBlock_and_SubscribeNewBlockEvent(t *testing.T) { } require.NoError(t, fetcher.UnsubscribeNewBlockEvent(ctx)) } - -// TestBlockFetcherHeaderValues tests that both the Commit and ValidatorSet -// endpoints are working as intended. -func TestBlockFetcherHeaderValues(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - t.Cleanup(cancel) - - client := StartTestNode(t).Client - fetcher := NewBlockFetcher(client) - - // generate some blocks - newBlockChan, err := fetcher.SubscribeNewBlockEvent(ctx) - require.NoError(t, err) - // read once from channel to generate next block - var h int64 - select { - case evt := <-newBlockChan: - h = evt.Header.Height - case <-ctx.Done(): - require.NoError(t, ctx.Err()) - } - // get Commit from current height - commit, err := fetcher.Commit(ctx, &h) - require.NoError(t, err) - // get ValidatorSet from current height - valSet, err := fetcher.ValidatorSet(ctx, &h) - require.NoError(t, err) - // get next block - var nextBlock types.EventDataSignedBlock - select { - case nextBlock = <-newBlockChan: - case <-ctx.Done(): - require.NoError(t, ctx.Err()) - } - // compare LastCommit from next block to Commit from first block height - assert.Equal(t, nextBlock.Header.LastCommitHash, commit.Hash()) - assert.Equal(t, nextBlock.Header.Height, commit.Height+1) - // compare ValidatorSet hash to the ValidatorsHash from first block height - hexBytes := valSet.Hash() - assert.Equal(t, nextBlock.ValidatorSet.Hash(), hexBytes) - require.NoError(t, fetcher.UnsubscribeNewBlockEvent(ctx)) -} diff --git a/core/listener_no_race_test.go b/core/listener_no_race_test.go new file mode 100644 index 0000000000..eac12785ee --- /dev/null +++ b/core/listener_no_race_test.go @@ -0,0 +1,71 @@ +//go:build !race + +package core + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" +) + +// TestListenerWithNonEmptyBlocks ensures that non-empty blocks are actually +// stored to eds.Store. +func TestListenerWithNonEmptyBlocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + + // create mocknet with two pubsub endpoints + ps0, _ := createMocknetWithTwoPubsubEndpoints(ctx, t) + + // create one block to store as Head in local store and then unsubscribe from block events + cfg := DefaultTestConfig() + cfg.ChainID = networkID + fetcher, cctx := createCoreFetcher(t, cfg) + eds := createEdsPubSub(ctx, t) + + store := createStore(t) + err := store.Start(ctx) + require.NoError(t, err) + t.Cleanup(func() { + err = store.Stop(ctx) + require.NoError(t, err) + }) + + // create Listener and start listening + cl := createListener(ctx, t, fetcher, ps0, eds, store, networkID) + err = cl.Start(ctx) + require.NoError(t, err) + + // listen for eds hashes broadcasted through eds-sub and ensure store has + // already stored them + sub, err := eds.Subscribe() + require.NoError(t, err) + t.Cleanup(sub.Cancel) + + empty := share.EmptyRoot() + // TODO extract 16 + for i := 0; i < 16; i++ { + _, err := cctx.FillBlock(16, cfg.Accounts, flags.BroadcastBlock) + require.NoError(t, err) + msg, err := sub.Next(ctx) + require.NoError(t, err) + + if bytes.Equal(empty.Hash(), msg.DataHash) { + continue + } + + has, err := store.Has(ctx, msg.DataHash) + require.NoError(t, err) + require.True(t, has) + } + + err = cl.Stop(ctx) + require.NoError(t, err) + require.Nil(t, cl.cancel) +} diff --git a/core/listener_test.go b/core/listener_test.go index 5ddcd5541d..b3ed11e571 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -1,12 +1,10 @@ package core import ( - "bytes" "context" "testing" "time" - "github.com/cosmos/cosmos-sdk/client/flags" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/event" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" @@ -17,7 +15,6 @@ import ( "github.com/celestiaorg/celestia-node/header" nodep2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -73,62 +70,6 @@ func TestListener(t *testing.T) { require.Nil(t, cl.cancel) } -// TestListenerWithNonEmptyBlocks ensures that non-empty blocks are actually -// stored to eds.Store. -func TestListenerWithNonEmptyBlocks(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - t.Cleanup(cancel) - - // create mocknet with two pubsub endpoints - ps0, _ := createMocknetWithTwoPubsubEndpoints(ctx, t) - - // create one block to store as Head in local store and then unsubscribe from block events - cfg := DefaultTestConfig() - cfg.ChainID = networkID - fetcher, cctx := createCoreFetcher(t, cfg) - eds := createEdsPubSub(ctx, t) - - store := createStore(t) - err := store.Start(ctx) - require.NoError(t, err) - t.Cleanup(func() { - err = store.Stop(ctx) - require.NoError(t, err) - }) - - // create Listener and start listening - cl := createListener(ctx, t, fetcher, ps0, eds, store, networkID) - err = cl.Start(ctx) - require.NoError(t, err) - - // listen for eds hashes broadcasted through eds-sub and ensure store has - // already stored them - sub, err := eds.Subscribe() - require.NoError(t, err) - t.Cleanup(sub.Cancel) - - empty := share.EmptyRoot() - // TODO extract 16 - for i := 0; i < 16; i++ { - _, err := cctx.FillBlock(16, cfg.Accounts, flags.BroadcastBlock) - require.NoError(t, err) - msg, err := sub.Next(ctx) - require.NoError(t, err) - - if bytes.Equal(empty.Hash(), msg.DataHash) { - continue - } - - has, err := store.Has(ctx, msg.DataHash) - require.NoError(t, err) - require.True(t, has) - } - - err = cl.Stop(ctx) - require.NoError(t, err) - require.Nil(t, cl.cancel) -} - func TestListenerWithWrongChainRPC(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index 1d0f3c4fad..41eff32fab 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -1,3 +1,5 @@ +//go:build !race + package nodebuilder import ( diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index bd179c1258..51bd89c5a7 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -1,3 +1,5 @@ +//go:build !race + package nodebuilder import ( diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 2fd4b2d3da..a793ba7b33 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -1,3 +1,5 @@ +//go:build api || integration + package tests import ( @@ -13,26 +15,14 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" - "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) -func getAdminClient(ctx context.Context, nd *nodebuilder.Node, t *testing.T) *client.Client { - t.Helper() - - signer := nd.AdminSigner - listenAddr := "ws://" + nd.RPCServer.ListenAddr() - - jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) - require.NoError(t, err) - - client, err := client.NewClient(ctx, listenAddr, jwt) - require.NoError(t, err) - - return client -} +const ( + btime = time.Millisecond * 300 +) func TestNodeModule(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 627dd850d0..a30e91b2e6 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -1,3 +1,5 @@ +//go:build blob || integration + package tests import ( diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index cb07dbb73a..6496cdbb53 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -1,3 +1,5 @@ +//go:build fraud || integration + package tests import ( diff --git a/nodebuilder/tests/helpers_test.go b/nodebuilder/tests/helpers_test.go new file mode 100644 index 0000000000..1e7f14d823 --- /dev/null +++ b/nodebuilder/tests/helpers_test.go @@ -0,0 +1,35 @@ +//nolint:unused +package tests + +import ( + "context" + "testing" + "time" + + "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/libs/authtoken" + "github.com/celestiaorg/celestia-node/nodebuilder" + + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/stretchr/testify/require" +) + +func getAdminClient(ctx context.Context, nd *nodebuilder.Node, t *testing.T) *client.Client { + t.Helper() + + signer := nd.AdminSigner + listenAddr := "ws://" + nd.RPCServer.ListenAddr() + + jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + require.NoError(t, err) + + client, err := client.NewClient(ctx, listenAddr, jwt) + require.NoError(t, err) + + return client +} + +func setTimeInterval(cfg *nodebuilder.Config, interval time.Duration) { + cfg.P2P.RoutingTableRefreshPeriod = interval + cfg.Share.Discovery.AdvertiseInterval = interval +} diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index f3338ec294..338aa6d0c1 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -1,3 +1,5 @@ +//go:build nd || integration + package tests import ( diff --git a/nodebuilder/tests/p2p_test.go b/nodebuilder/tests/p2p_test.go index 9fe63fb931..98e9fc15b4 100644 --- a/nodebuilder/tests/p2p_test.go +++ b/nodebuilder/tests/p2p_test.go @@ -1,3 +1,5 @@ +//go:build p2p || integration + package tests import ( @@ -199,8 +201,3 @@ func TestRestartNodeDiscovery(t *testing.T) { require.NoError(t, err) assert.Equal(t, connectedness, network.Connected) } - -func setTimeInterval(cfg *nodebuilder.Config, interval time.Duration) { - cfg.P2P.RoutingTableRefreshPeriod = interval - cfg.Share.Discovery.AdvertiseInterval = interval -} diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index 89f4a0171a..d047182669 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -1,7 +1,4 @@ -// Test with light nodes spawns more goroutines than in the race detectors budget, -// and thus we're disabling the race detector. -// TODO(@Wondertan): Remove this once we move to go1.19 with unlimited race detector -//go:build !race +//go:build reconstruction || integration package tests @@ -89,6 +86,10 @@ Test-Case: Full Node reconstructs blocks from each other, after unsuccessfully s block from LN subnetworks. Analog to TestShareAvailable_DisconnectedFullNodes. */ func TestFullReconstructFromFulls(t *testing.T) { + if testing.Short() { + t.Skip() + } + light.DefaultSampleAmount = 10 // s const ( blocks = 10 @@ -255,6 +256,10 @@ Steps: 9. Check that the FN can retrieve shares from 1 to 20 blocks */ func TestFullReconstructFromLights(t *testing.T) { + if testing.Short() { + t.Skip() + } + eds.RetrieveQuadrantTimeout = time.Millisecond * 100 light.DefaultSampleAmount = 20 const ( diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index 8b939749d6..ec8386ea43 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -1,3 +1,5 @@ +//go:build sync || integration + package tests import ( @@ -16,7 +18,7 @@ import ( const ( numBlocks = 20 bsize = 16 - btime = time.Millisecond * 300 + sbtime = time.Millisecond * 300 ) /* @@ -45,7 +47,7 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(sbtime)) // wait for core network to fill 20 blocks fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, numBlocks) sw.WaitTillHeight(ctx, numBlocks) @@ -138,7 +140,7 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(sbtime)) sw.WaitTillHeight(ctx, numBlocks) // create bridge node and set it as the bootstrapper for the suite @@ -211,6 +213,10 @@ Steps: 9. Check LN is synced to height 40 */ func TestSyncStartStopLightWithBridge(t *testing.T) { + if testing.Short() { + t.Skip("skipping TestSyncStartStopLightWithBridge test in short mode.") + } + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) defer cancel() diff --git a/share/eds/retriever_no_race_test.go b/share/eds/retriever_no_race_test.go new file mode 100644 index 0000000000..15c6aa2fc4 --- /dev/null +++ b/share/eds/retriever_no_race_test.go @@ -0,0 +1,55 @@ +// go:build !race + +package eds + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" +) + +func TestRetriever_ByzantineError(t *testing.T) { + const width = 8 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + bserv := ipld.NewMemBlockservice() + shares := edstest.RandEDS(t, width).Flattened() + _, err := ipld.ImportShares(ctx, shares, bserv) + require.NoError(t, err) + + // corrupt shares so that eds erasure coding does not match + copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) + + // import corrupted eds + batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) + attackerEDS, err := rsmt2d.ImportExtendedDataSquare( + shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(width), + nmt.NodeVisitor(batchAdder.Visit)), + ) + require.NoError(t, err) + err = batchAdder.Commit() + require.NoError(t, err) + + // ensure we rcv an error + dah, err := da.NewDataAvailabilityHeader(attackerEDS) + require.NoError(t, err) + r := NewRetriever(bserv) + _, err = r.Retrieve(ctx, &dah) + var errByz *byzantine.ErrByzantine + require.ErrorAs(t, err, &errByz) +} diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index f3f7ccca64..95da345d17 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -12,8 +12,6 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" @@ -67,40 +65,6 @@ func TestRetriever_Retrieve(t *testing.T) { } } -func TestRetriever_ByzantineError(t *testing.T) { - const width = 8 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - bserv := ipld.NewMemBlockservice() - shares := edstest.RandEDS(t, width).Flattened() - _, err := ipld.ImportShares(ctx, shares, bserv) - require.NoError(t, err) - - // corrupt shares so that eds erasure coding does not match - copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) - - // import corrupted eds - batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) - attackerEDS, err := rsmt2d.ImportExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(width), - nmt.NodeVisitor(batchAdder.Visit)), - ) - require.NoError(t, err) - err = batchAdder.Commit() - require.NoError(t, err) - - // ensure we rcv an error - dah, err := da.NewDataAvailabilityHeader(attackerEDS) - require.NoError(t, err) - r := NewRetriever(bserv) - _, err = r.Retrieve(ctx, &dah) - var errByz *byzantine.ErrByzantine - require.ErrorAs(t, err, &errByz) -} - // TestRetriever_MultipleRandQuadrants asserts that reconstruction succeeds // when any three random quadrants requested. func TestRetriever_MultipleRandQuadrants(t *testing.T) { diff --git a/share/p2p/discovery/discovery_test.go b/share/p2p/discovery/discovery_test.go index c02931e1a4..1d0078196f 100644 --- a/share/p2p/discovery/discovery_test.go +++ b/share/p2p/discovery/discovery_test.go @@ -1,3 +1,5 @@ +// go:build !race + package discovery import ( diff --git a/state/core_access_test.go b/state/core_access_test.go index 68d05a955a..ad7b916ea3 100644 --- a/state/core_access_test.go +++ b/state/core_access_test.go @@ -1,3 +1,5 @@ +//go:build !race + package state import (