diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..110b5c216 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# Top-most EditorConfig file. +root = true + +# Unix-style newlines with a newline ending every file. +[*.md] +end_of_line = lf +insert_final_newline = true +max_line_length = 80 + +# 8 space indentation for Golang code. +[*.go] +indent_style = tab +indent_size = 8 +max_line_length = 80 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4811fed6..5477197c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -279,13 +279,13 @@ jobs: matrix: include: - name: bbolt - args: itest-no-backward-compat icase=terminal dbbackend=bbolt + args: itest-parallel-no-backward-compat icase=terminal dbbackend=bbolt tranches=4 - name: sqlite - args: itest-no-backward-compat icase=terminal dbbackend=sqlite + args: itest-parallel-no-backward-compat icase=terminal dbbackend=sqlite tranches=4 - name: postgres - args: itest-no-backward-compat icase=terminal dbbackend=postgres + args: itest-parallel-no-backward-compat icase=terminal dbbackend=postgres tranches=4 - name: custom-channels - args: itest-only icase=custom_channels + args: itest-parallel icase=custom_channels tranches=4 steps: - name: git checkout diff --git a/Makefile b/Makefile index e2aab9360..4c89d813e 100644 --- a/Makefile +++ b/Makefile @@ -239,6 +239,10 @@ build-itest: CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/btcd-itest -ldflags "$(ITEST_LDFLAGS)" $(BTCD_PKG) CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/lnd-itest -ldflags "$(ITEST_LDFLAGS)" $(LND_PKG)/cmd/lnd +build-itest-binary: + @$(call print, "Building itest binary.") + CGO_ENABLED=0 $(GOTEST) -v ./itest -tags="$(DEV_TAGS) $(ITEST_TAGS)" -c -o itest/itest.test + install-backward-compat-versions: @$(call print, "Installing old versions of litd for backward compatibility tests.") scripts/install-backward-compat-versions.sh '$(LITD_COMPAT_VERSIONS)' @@ -254,9 +258,19 @@ run-itest-only: itest-only: build-itest install-backward-compat-versions run-itest-only -itest: app-build build-itest itest-only +itest: app-build itest-only + +itest-no-backward-compat: app-build build-itest run-itest-only -itest-no-backward-compat: app-build build-itest build-itest run-itest-only +itest-parallel: app-build build-itest install-backward-compat-versions build-itest-binary + @$(call print, "Running integration tests in parallel.") + rm -rf itest/*.log itest/.logs*; date + scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) + +itest-parallel-no-backward-compat: app-build build-itest build-itest-binary + @$(call print, "Running integration tests in parallel (no backward compat binaries).") + rm -rf itest/*.log itest/.logs*; date + scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) # ============= # FLAKE HUNTING @@ -349,5 +363,6 @@ flakehunter-unit: .PHONY: default all yarn-install build install go-build go-build-noui \ go-install go-install-noui go-install-cli app-build release go-release \ docker-release docker-tools scratch check unit unit-cover unit-race \ - clean-itest build-itest itest-only itest flake-unit fmt lint mod mod-check \ - list rpc protos protos-check rpc-js-compile clean \ No newline at end of file + clean-itest build-itest build-itest-binary itest-only itest \ + itest-parallel itest-parallel-no-backward-compat flake-unit fmt lint \ + mod mod-check list rpc protos protos-check rpc-js-compile clean diff --git a/itest/litd_test.go b/itest/litd_test.go index a04d67c2c..ef80c1d70 100644 --- a/itest/litd_test.go +++ b/itest/litd_test.go @@ -1,6 +1,9 @@ package itest import ( + "flag" + "fmt" + "math/rand" "os" "strings" "testing" @@ -12,6 +15,44 @@ import ( "github.com/stretchr/testify/require" ) +const ( + // defaultSplitTranches is the default number of tranches to divide the + // test suite into when no override is provided. + defaultSplitTranches uint = 1 + + // defaultRunTranche is the default tranche index to execute when no + // explicit tranche is selected. + defaultRunTranche uint = 0 +) + +var ( + // testCasesSplitTranches is the number of tranches the test cases + // should be split into. By default this is set to 1, so no splitting + // happens. If this value is increased, then the -runtranche flag must + // be specified as well to indicate which part should be run in the + // current invocation. + testCasesSplitTranches = flag.Uint( + "splittranches", defaultSplitTranches, + "split the test cases in this many tranches and run the "+ + "tranche at 0-based index specified by the "+ + "-runtranche flag", + ) + + // shuffleSeedFlag enables deterministic shuffling of test cases to + // balance workload across tranches. + shuffleSeedFlag = flag.Uint64( + "shuffleseed", 0, "if set, shuffles the test cases using this "+ + "as the source of randomness", + ) + + // testCasesRunTranche selects which tranche (0-based) to execute. + testCasesRunTranche = flag.Uint( + "runtranche", defaultRunTranche, + "run the tranche of the split test cases with the given "+ + "(0-based) index", + ) +) + // TestLightningTerminal performs a series of integration tests amongst a // programmatically driven network of lnd nodes. func TestLightningTerminal(t *testing.T) { @@ -39,9 +80,18 @@ func TestLightningTerminal(t *testing.T) { "--rpcmiddleware.enable", } + testCases, trancheIndex, trancheOffset := selectTestTranche() + totalTestCases := len(allTestCases) + // Run the subset of the test cases selected in this tranche. - for _, testCase := range allTestCases { - success := t.Run(testCase.name, func(t1 *testing.T) { + for idx, testCase := range testCases { + testOrdinal := int(trancheOffset) + idx + 1 + testName := fmt.Sprintf( + "tranche%02d/%02d-of-%d/%s", int(trancheIndex), + testOrdinal, totalTestCases, testCase.name, + ) + + success := t.Run(testName, func(t1 *testing.T) { cleanTestCaseName := strings.ReplaceAll( testCase.name, " ", "_", ) @@ -107,6 +157,79 @@ func TestLightningTerminal(t *testing.T) { } } +// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is +// set and not 0. This is used by parallel test runs to even out the work +// across tranches. +func maybeShuffleTestCases() { + // Exit if not set or set to 0. + if shuffleSeedFlag == nil || *shuffleSeedFlag == 0 { + return + } + + // Init the seed and shuffle the test cases. + // #nosec G404 -- This is not for cryptographic purposes. + r := rand.New(rand.NewSource(int64(*shuffleSeedFlag))) + r.Shuffle(len(allTestCases), func(i, j int) { + allTestCases[i], allTestCases[j] = + allTestCases[j], allTestCases[i] + }) +} + +// createIndices divides the number of test cases into pairs of indices that +// specify the start and end of a tranche. +func createIndices(numCases, numTranches uint) [][2]uint { + base := numCases / numTranches + remainder := numCases % numTranches + + indices := make([][2]uint, numTranches) + start := uint(0) + + for i := uint(0); i < numTranches; i++ { + end := start + base + if i < remainder { + end++ + } + indices[i] = [2]uint{start, end} + start = end + } + + return indices +} + +// selectTestTranche returns the sub slice of the test cases that should be run +// as the current split tranche as well as the index and slice offset of the +// tranche. +func selectTestTranche() ([]*testCase, uint, uint) { + numTranches := defaultSplitTranches + if testCasesSplitTranches != nil { + numTranches = *testCasesSplitTranches + } + runTranche := defaultRunTranche + if testCasesRunTranche != nil { + runTranche = *testCasesRunTranche + } + + // There's a special flake-hunt mode where we run the same test multiple + // times in parallel. In that case the tranche index is equal to the + // thread ID, but we need to actually run all tests for the regex + // selection to work. + threadID := runTranche + if numTranches == 1 { + runTranche = 0 + } + + // Shuffle the test cases if the `shuffleseed` flag is set. + maybeShuffleTestCases() + + numCases := uint(len(allTestCases)) + indices := createIndices(numCases, numTranches) + index := indices[runTranche] + trancheOffset, trancheEnd := index[0], index[1] + + return allTestCases[trancheOffset:trancheEnd], threadID, + trancheOffset +} + func init() { logger := btclog.NewSLogger(btclog.NewDefaultHandler(os.Stdout)) UseLogger(logger.SubSystem(Subsystem)) diff --git a/make/testing_flags.mk b/make/testing_flags.mk index dfd9b3bce..9bfd0a6d1 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -3,6 +3,26 @@ include make/compile_flags.mk TEST_FLAGS = DEV_TAGS = dev +NUM_ITEST_TRANCHES = 8 +ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) +SHUFFLE_SEED = 0 + +# Scale the number of parallel running itest tranches. +ifneq ($(tranches),) +NUM_ITEST_TRANCHES = $(tranches) +ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) +endif + +# Give the ability to run the same tranche multiple times at the same time. +ifneq ($(parallel),) +ITEST_PARALLELISM = $(parallel) +endif + +# Set the seed for shuffling the test cases. +ifneq ($(shuffleseed),) +SHUFFLE_SEED = $(shuffleseed) +endif + # Define the integration test.run filter if the icase argument was provided. ifneq ($(icase),) ITEST_FLAGS += -test.run="TestLightningTerminal/$(icase)" diff --git a/scripts/itest_parallel.sh b/scripts/itest_parallel.sh new file mode 100755 index 000000000..8154c9ae1 --- /dev/null +++ b/scripts/itest_parallel.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Get all the variables. +PROCESSES=$1 +TRANCHES=$2 +SHUFFLE_SEED=$3 + +# Here we also shift 3 times and get the rest of our flags to pass on in $@. +shift 3 + +# Create a variable to hold the final exit code. +exit_code=0 + +# Run commands in parallel and track their PIDs. +pids=() +for ((i=0; i"$LOG_FILE" 2>&1 + +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo "Tranche $TRANCHE failed with exit code $exit_code" + tail -n 100 "$LOG_FILE" + exit 255 +else + echo "Tranche $TRANCHE completed successfully" +fi