From 5295661f08a70b389169c4832c578e23108a4e4a Mon Sep 17 00:00:00 2001 From: ffranr Date: Tue, 9 Dec 2025 12:44:55 +0000 Subject: [PATCH 1/4] add .editorconfig --- .editorconfig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..110b5c216 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# Top-most EditorConfig file. +root = true + +# Unix-style newlines with a newline ending every file. +[*.md] +end_of_line = lf +insert_final_newline = true +max_line_length = 80 + +# 8 space indentation for Golang code. +[*.go] +indent_style = tab +indent_size = 8 +max_line_length = 80 From 128e0b6df653a8a2506bf925c50e8353fd90fdaf Mon Sep 17 00:00:00 2001 From: ffranr Date: Fri, 5 Dec 2025 15:06:24 +0000 Subject: [PATCH 2/4] itest: add tranche-based parallel runner and clearer logs - add tranche splitting/shuffling flags to the itest harness - add itest-parallel target and scripts to run tranches concurrently - write per-tranche logs under .logs/trancheN and tail failures for clarity --- Makefile | 19 +++++- itest/litd_test.go | 127 +++++++++++++++++++++++++++++++++++++- make/testing_flags.mk | 20 ++++++ scripts/itest_parallel.sh | 38 ++++++++++++ scripts/itest_part.sh | 35 ++++++++++- 5 files changed, 233 insertions(+), 6 deletions(-) create mode 100755 scripts/itest_parallel.sh diff --git a/Makefile b/Makefile index e2aab9360..c11bf9395 100644 --- a/Makefile +++ b/Makefile @@ -239,6 +239,10 @@ build-itest: CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/btcd-itest -ldflags "$(ITEST_LDFLAGS)" $(BTCD_PKG) CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/lnd-itest -ldflags "$(ITEST_LDFLAGS)" $(LND_PKG)/cmd/lnd +build-itest-binary: + @$(call print, "Building itest binary.") + CGO_ENABLED=0 $(GOTEST) -v ./itest -tags="$(DEV_TAGS) $(ITEST_TAGS)" -c -o itest/itest.test + install-backward-compat-versions: @$(call print, "Installing old versions of litd for backward compatibility tests.") scripts/install-backward-compat-versions.sh '$(LITD_COMPAT_VERSIONS)' @@ -258,6 +262,16 @@ itest: app-build build-itest itest-only itest-no-backward-compat: app-build build-itest build-itest run-itest-only +itest-parallel: app-build build-itest install-backward-compat-versions build-itest-binary + @$(call print, "Running integration tests in parallel.") + rm -rf itest/*.log itest/.logs*; date + scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) + +itest-parallel-no-backward-compat: app-build build-itest build-itest-binary + @$(call print, "Running integration tests in parallel (no backward compat binaries).") + rm -rf itest/*.log itest/.logs*; date + scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) + # ============= # FLAKE HUNTING # ============= @@ -349,5 +363,6 @@ flakehunter-unit: .PHONY: default all yarn-install build install go-build go-build-noui \ go-install go-install-noui go-install-cli app-build release go-release \ docker-release docker-tools scratch check unit unit-cover unit-race \ - clean-itest build-itest itest-only itest flake-unit fmt lint mod mod-check \ - list rpc protos protos-check rpc-js-compile clean \ No newline at end of file + clean-itest build-itest build-itest-binary itest-only itest \ + itest-parallel itest-parallel-no-backward-compat flake-unit fmt lint \ + mod mod-check list rpc protos protos-check rpc-js-compile clean diff --git a/itest/litd_test.go b/itest/litd_test.go index a04d67c2c..ef80c1d70 100644 --- a/itest/litd_test.go +++ b/itest/litd_test.go @@ -1,6 +1,9 @@ package itest import ( + "flag" + "fmt" + "math/rand" "os" "strings" "testing" @@ -12,6 +15,44 @@ import ( "github.com/stretchr/testify/require" ) +const ( + // defaultSplitTranches is the default number of tranches to divide the + // test suite into when no override is provided. + defaultSplitTranches uint = 1 + + // defaultRunTranche is the default tranche index to execute when no + // explicit tranche is selected. + defaultRunTranche uint = 0 +) + +var ( + // testCasesSplitTranches is the number of tranches the test cases + // should be split into. By default this is set to 1, so no splitting + // happens. If this value is increased, then the -runtranche flag must + // be specified as well to indicate which part should be run in the + // current invocation. + testCasesSplitTranches = flag.Uint( + "splittranches", defaultSplitTranches, + "split the test cases in this many tranches and run the "+ + "tranche at 0-based index specified by the "+ + "-runtranche flag", + ) + + // shuffleSeedFlag enables deterministic shuffling of test cases to + // balance workload across tranches. + shuffleSeedFlag = flag.Uint64( + "shuffleseed", 0, "if set, shuffles the test cases using this "+ + "as the source of randomness", + ) + + // testCasesRunTranche selects which tranche (0-based) to execute. + testCasesRunTranche = flag.Uint( + "runtranche", defaultRunTranche, + "run the tranche of the split test cases with the given "+ + "(0-based) index", + ) +) + // TestLightningTerminal performs a series of integration tests amongst a // programmatically driven network of lnd nodes. func TestLightningTerminal(t *testing.T) { @@ -39,9 +80,18 @@ func TestLightningTerminal(t *testing.T) { "--rpcmiddleware.enable", } + testCases, trancheIndex, trancheOffset := selectTestTranche() + totalTestCases := len(allTestCases) + // Run the subset of the test cases selected in this tranche. - for _, testCase := range allTestCases { - success := t.Run(testCase.name, func(t1 *testing.T) { + for idx, testCase := range testCases { + testOrdinal := int(trancheOffset) + idx + 1 + testName := fmt.Sprintf( + "tranche%02d/%02d-of-%d/%s", int(trancheIndex), + testOrdinal, totalTestCases, testCase.name, + ) + + success := t.Run(testName, func(t1 *testing.T) { cleanTestCaseName := strings.ReplaceAll( testCase.name, " ", "_", ) @@ -107,6 +157,79 @@ func TestLightningTerminal(t *testing.T) { } } +// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is +// set and not 0. This is used by parallel test runs to even out the work +// across tranches. +func maybeShuffleTestCases() { + // Exit if not set or set to 0. + if shuffleSeedFlag == nil || *shuffleSeedFlag == 0 { + return + } + + // Init the seed and shuffle the test cases. + // #nosec G404 -- This is not for cryptographic purposes. + r := rand.New(rand.NewSource(int64(*shuffleSeedFlag))) + r.Shuffle(len(allTestCases), func(i, j int) { + allTestCases[i], allTestCases[j] = + allTestCases[j], allTestCases[i] + }) +} + +// createIndices divides the number of test cases into pairs of indices that +// specify the start and end of a tranche. +func createIndices(numCases, numTranches uint) [][2]uint { + base := numCases / numTranches + remainder := numCases % numTranches + + indices := make([][2]uint, numTranches) + start := uint(0) + + for i := uint(0); i < numTranches; i++ { + end := start + base + if i < remainder { + end++ + } + indices[i] = [2]uint{start, end} + start = end + } + + return indices +} + +// selectTestTranche returns the sub slice of the test cases that should be run +// as the current split tranche as well as the index and slice offset of the +// tranche. +func selectTestTranche() ([]*testCase, uint, uint) { + numTranches := defaultSplitTranches + if testCasesSplitTranches != nil { + numTranches = *testCasesSplitTranches + } + runTranche := defaultRunTranche + if testCasesRunTranche != nil { + runTranche = *testCasesRunTranche + } + + // There's a special flake-hunt mode where we run the same test multiple + // times in parallel. In that case the tranche index is equal to the + // thread ID, but we need to actually run all tests for the regex + // selection to work. + threadID := runTranche + if numTranches == 1 { + runTranche = 0 + } + + // Shuffle the test cases if the `shuffleseed` flag is set. + maybeShuffleTestCases() + + numCases := uint(len(allTestCases)) + indices := createIndices(numCases, numTranches) + index := indices[runTranche] + trancheOffset, trancheEnd := index[0], index[1] + + return allTestCases[trancheOffset:trancheEnd], threadID, + trancheOffset +} + func init() { logger := btclog.NewSLogger(btclog.NewDefaultHandler(os.Stdout)) UseLogger(logger.SubSystem(Subsystem)) diff --git a/make/testing_flags.mk b/make/testing_flags.mk index dfd9b3bce..9bfd0a6d1 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -3,6 +3,26 @@ include make/compile_flags.mk TEST_FLAGS = DEV_TAGS = dev +NUM_ITEST_TRANCHES = 8 +ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) +SHUFFLE_SEED = 0 + +# Scale the number of parallel running itest tranches. +ifneq ($(tranches),) +NUM_ITEST_TRANCHES = $(tranches) +ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) +endif + +# Give the ability to run the same tranche multiple times at the same time. +ifneq ($(parallel),) +ITEST_PARALLELISM = $(parallel) +endif + +# Set the seed for shuffling the test cases. +ifneq ($(shuffleseed),) +SHUFFLE_SEED = $(shuffleseed) +endif + # Define the integration test.run filter if the icase argument was provided. ifneq ($(icase),) ITEST_FLAGS += -test.run="TestLightningTerminal/$(icase)" diff --git a/scripts/itest_parallel.sh b/scripts/itest_parallel.sh new file mode 100755 index 000000000..8154c9ae1 --- /dev/null +++ b/scripts/itest_parallel.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Get all the variables. +PROCESSES=$1 +TRANCHES=$2 +SHUFFLE_SEED=$3 + +# Here we also shift 3 times and get the rest of our flags to pass on in $@. +shift 3 + +# Create a variable to hold the final exit code. +exit_code=0 + +# Run commands in parallel and track their PIDs. +pids=() +for ((i=0; i"$LOG_FILE" 2>&1 + +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo "Tranche $TRANCHE failed with exit code $exit_code" + tail -n 100 "$LOG_FILE" + exit 255 +else + echo "Tranche $TRANCHE completed successfully" +fi From 07b59f069d7ad96d8363d56eb4c5efa2db1b6813 Mon Sep 17 00:00:00 2001 From: ffranr Date: Fri, 5 Dec 2025 15:16:27 +0000 Subject: [PATCH 3/4] ci: run itest tranches in parallel - switch itest matrix jobs to use itest-parallel(-no-backward-compat) - run four tranches per job --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4811fed6..5477197c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -279,13 +279,13 @@ jobs: matrix: include: - name: bbolt - args: itest-no-backward-compat icase=terminal dbbackend=bbolt + args: itest-parallel-no-backward-compat icase=terminal dbbackend=bbolt tranches=4 - name: sqlite - args: itest-no-backward-compat icase=terminal dbbackend=sqlite + args: itest-parallel-no-backward-compat icase=terminal dbbackend=sqlite tranches=4 - name: postgres - args: itest-no-backward-compat icase=terminal dbbackend=postgres + args: itest-parallel-no-backward-compat icase=terminal dbbackend=postgres tranches=4 - name: custom-channels - args: itest-only icase=custom_channels + args: itest-parallel icase=custom_channels tranches=4 steps: - name: git checkout From 9a5e9438428b8f2f4b860ba38b19ba68927f84de Mon Sep 17 00:00:00 2001 From: ffranr Date: Fri, 5 Dec 2025 15:42:02 +0000 Subject: [PATCH 4/4] make: remove redundant job dependencies --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c11bf9395..4c89d813e 100644 --- a/Makefile +++ b/Makefile @@ -258,9 +258,9 @@ run-itest-only: itest-only: build-itest install-backward-compat-versions run-itest-only -itest: app-build build-itest itest-only +itest: app-build itest-only -itest-no-backward-compat: app-build build-itest build-itest run-itest-only +itest-no-backward-compat: app-build build-itest run-itest-only itest-parallel: app-build build-itest install-backward-compat-versions build-itest-binary @$(call print, "Running integration tests in parallel.")