Add itest framework and implement first integration tests

.github/workflows/main.yml

@@ -13,9 +13,6 @@ defaults:
     shell: bash
 
 env:
-  # go needs absolute directories, using the $HOME variable doesn't work here.
-  GOCACHE: /home/runner/work/go/pkg/build
-  GOPATH: /home/runner/work/go
   GO_VERSION: 1.23.9
 
 jobs:
@@ -27,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: git checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -42,10 +39,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: git checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v5
 
       - name: setup go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: '${{ env.GO_VERSION }}'
 
@@ -54,3 +51,24 @@ jobs:
 
       - name: run unit tests
         run: make unit
+
+  ########################
+  # run integration tests
+  ########################
+  integration-test:
+    name: run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v5
+
+      - name: setup go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+
+      - name: Install chantools
+        run: make install
+
+      - name: run integration tests
+        run: make itest

.golangci.yml

@@ -1,32 +1,11 @@
+version: "2"
+
 run:
-  # timeout for analysis
-  timeout: 4m
+  timeout: 5m
   go: "1.23"
 
-linters-settings:
-  govet:
-    # Don't report about shadowed variables
-    shadow: false
-  gofmt:
-    # simplify code: gofmt with `-s` option, true by default
-    simplify: true
-  whitespace:
-    multi-func: true
-    multi-if: true
-  tagliatelle:
-    case:
-      rules:
-        json: snake
-  staticcheck:
-    checks: ["-SA1019"]
-  gomoddirectives:
-    replace-allow-list:
-      # See go.mod for the explanation why these are needed.
-      - google.golang.org/protobuf
-
-
 linters:
-  enable-all: true
+  default: all
   disable:
     - gochecknoglobals
     - gosec
@@ -43,20 +22,50 @@ linters:
     - cyclop
     - gocyclo
     - nlreturn
-    - stylecheck
     - paralleltest
     - ireturn
     - maintidx
     - noctx
-    - gofumpt
     - exhaustive
     - protogetter
     - depguard
     - mnd
     - gomoddirectives
+    - nolintlint
+    - errcheck
+    - funcorder
+    - godoclint
+    - noinlineerr
+    - revive
+    - wsl_v5
+
+  settings:
+    govet:
+      # Don't report about shadowed variables.
+      shadow: false
+    gofmt:
+      # simplify code: gofmt with `-s` option, true by default.
+      simplify: true
+    whitespace:
+      multi-func: true
+      multi-if: true
+    tagliatelle:
+      case:
+        rules:
+          json: snake
+    staticcheck:
+      checks: [ "-SA1019" ]
+    gomoddirectives:
+      replace-allow-list:
+        # See go.mod for the explanation why these are needed.
+        - google.golang.org/protobuf
 
-issues:
-  exclude-rules:
-    - path: cmd/chantools
-      linters:
-        - lll
+  exclusions:
+    rules:
+      - path: cmd/chantools
+        linters:
+          - lll
+      - path: itest
+        linters:
+          - dupl
+          - thelper

Makefile

@@ -44,7 +44,7 @@ DOCKER_TOOLS = docker run \
 
 TEST_FLAGS = -test.timeout=20m
 
-UNIT := $(GOLIST) | $(XARGS) env $(GOTEST) $(TEST_FLAGS)
+UNIT := $(GOLIST) | grep -v "/itest" | $(XARGS) env $(GOTEST) $(TEST_FLAGS)
 LDFLAGS := -X main.Commit=$(shell git describe --tags)
 RELEASE_LDFLAGS := -s -w -buildid= $(LDFLAGS)
 
@@ -64,6 +64,10 @@ unit:
 	@$(call print, "Running unit tests.")
 	$(UNIT)
 
+itest: install
+	@$(call print, "Running integration tests.")
+	cd itest; ./itest.sh
+
 build:
 	@$(call print, "Building chantools.")
 	$(GOBUILD) -ldflags "$(LDFLAGS)" ./...
@@ -94,3 +98,5 @@ lint: docker-tools
 docs: install
 	@$(call print, "Rendering docs.")
 	chantools doc
+
+.PHONY: unit itest build install release fmt lint docs docker-tools

cmd/chantools/fakechanbackup.go

@@ -108,7 +108,7 @@ chantools fakechanbackup --from_channel_graph lncli_describegraph.json \
 			"LN channel graph in the JSON format that the "+
 			"'lncli describegraph' returns",
 	)
-	multiFileName := fmt.Sprintf("results/fake-%s.backup",
+	multiFileName := fmt.Sprintf("%s/fake-%s.backup", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	cc.cmd.Flags().StringVar(
 		&cc.MultiFile, "multi_file", multiFileName, "the fake channel "+

cmd/chantools/filterbackup.go

@@ -93,7 +93,7 @@ func filterChannelBackup(multiFile *chanbackup.MultiFile, ring keychain.KeyRing,
 	}
 	multi.StaticBackups = keep
 
-	fileName := fmt.Sprintf("results/backup-filtered-%s.backup",
+	fileName := fmt.Sprintf("%s/backup-filtered-%s.backup", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	log.Infof("Writing result to %s", fileName)
 	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)

cmd/chantools/fixoldbackup.go

@@ -108,7 +108,7 @@ func fixOldChannelBackup(multiFile *chanbackup.MultiFile,
 	}
 
 	log.Infof("Fixed shachain root of %d channels.", fixedChannels)
-	fileName := fmt.Sprintf("results/backup-fixed-%s.backup",
+	fileName := fmt.Sprintf("%s/backup-fixed-%s.backup", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	log.Infof("Writing result to %s", fileName)
 	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)

cmd/chantools/forceclose.go

@@ -230,7 +230,7 @@ func forceCloseChannels(apiURL string, extendedKey *hdkeychain.ExtendedKey,
 	if err != nil {
 		return err
 	}
-	fileName := fmt.Sprintf("results/forceclose-%s.json",
+	fileName := fmt.Sprintf("%s/forceclose-%s.json", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	log.Infof("Writing result to %s", fileName)
 	return os.WriteFile(fileName, summaryBytes, 0644)

cmd/chantools/genimportscript.go

@@ -158,7 +158,7 @@ func (c *genImportScriptCommand) Execute(_ *cobra.Command, _ []string) error {
 
 	writer := os.Stdout
 	if !c.Stdout {
-		fileName := fmt.Sprintf("results/genimportscript-%s.txt",
+		fileName := fmt.Sprintf("%s/genimportscript-%s.txt", ResultsDir,
 			time.Now().Format("2006-01-02-15-04-05"))
 		log.Infof("Writing import script with format '%s' to %s",
 			c.Format, fileName)

cmd/chantools/rescueclosed.go

@@ -322,7 +322,7 @@ outer:
 	if err != nil {
 		return err
 	}
-	fileName := fmt.Sprintf("results/rescueclosed-%s.json",
+	fileName := fmt.Sprintf("%s/rescueclosed-%s.json", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	log.Infof("Writing result to %s", fileName)
 	return os.WriteFile(fileName, summaryBytes, 0644)

cmd/chantools/root.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -47,10 +48,11 @@ const (
 )
 
 var (
-	Testnet   bool
-	Regtest   bool
-	Signet    bool
-	NoLogFile bool
+	Testnet    bool
+	Regtest    bool
+	Signet     bool
+	NoLogFile  bool
+	ResultsDir string
 
 	log btclog.Logger
 
@@ -106,6 +108,10 @@ func main() {
 			"will be created. This is useful for testing purposes "+
 			"where we don't want to create a log file.",
 	)
+	rootCmd.PersistentFlags().StringVar(
+		&ResultsDir, "resultsdir", "./results", "Directory where "+
+			"results should be stored",
+	)
 
 	rootCmd.AddCommand(
 		newChanBackupCommand(),
@@ -351,7 +357,7 @@ func setupLogging() {
 				MaxLogFiles:    3,
 				MaxLogFileSize: 10,
 			},
-			"./results/chantools.log",
+			filepath.Join(ResultsDir, "chantools.log"),
 		)
 		if err != nil {
 			panic(err)

cmd/chantools/summary.go

@@ -109,7 +109,7 @@ func summarizeChannels(apiURL string,
 	if err != nil {
 		return err
 	}
-	fileName := fmt.Sprintf("results/summary-%s.json",
+	fileName := fmt.Sprintf("%s/summary-%s.json", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	log.Infof("Writing result to %s", fileName)
 	return os.WriteFile(fileName, summaryBytes, 0644)
@@ -129,7 +129,7 @@ func summarizeAncientChannels(apiURL string,
 	if err != nil {
 		return err
 	}
-	fileName := fmt.Sprintf("results/summary-ancient-%s.json",
+	fileName := fmt.Sprintf("%s/summary-ancient-%s.json", ResultsDir,
 		time.Now().Format("2006-01-02-15-04-05"))
 	log.Infof("Writing result to %s", fileName)
 	return os.WriteFile(fileName, summaryBytes, 0644)

cmd/chantools/triggerforceclose.go

@@ -206,17 +206,17 @@ func (c *triggerForceCloseCommand) Execute(_ *cobra.Command, _ []string) error {
 		peersBytes := []byte(strings.Join(pubKeys, "\n"))
 		outputsBytes := []byte(strings.Join(outputs, "\n"))
 
-		fileName := fmt.Sprintf("results/forceclose-peers-%s.txt",
-			time.Now().Format("2006-01-02"))
+		fileName := fmt.Sprintf("%s/forceclose-peers-%s.txt",
+			ResultsDir, time.Now().Format("2006-01-02"))
 		log.Infof("Writing peers to %s", fileName)
 		err = os.WriteFile(fileName, peersBytes, 0644)
 		if err != nil {
 			return fmt.Errorf("error writing peers to file: %w",
 				err)
 		}
 
-		fileName = fmt.Sprintf("results/forceclose-addresses-%s.txt",
-			time.Now().Format("2006-01-02"))
+		fileName = fmt.Sprintf("%s/forceclose-addresses-%s.txt",
+			ResultsDir, time.Now().Format("2006-01-02"))
 		log.Infof("Writing addresses to %s", fileName)
 		return os.WriteFile(fileName, outputsBytes, 0644)
 
@@ -259,7 +259,11 @@ func closeChannel(identityPriv *btcec.PrivateKey, api *btc.ExplorerAPI,
 		return nil, fmt.Errorf("error parsing channel point: %w", err)
 	}
 
-	err = requestForceClose(peer, torProxy, *outPoint, identityECDH)
+	err = requestForceCloseLnd(peer, torProxy, *outPoint, identityECDH)
+	if err != nil {
+		return nil, fmt.Errorf("error requesting force close: %w", err)
+	}
+	err = requestForceCloseCln(peer, torProxy, *outPoint, identityECDH)
 	if err != nil {
 		return nil, fmt.Errorf("error requesting force close: %w", err)
 	}
@@ -287,6 +291,17 @@ func closeChannel(identityPriv *btcec.PrivateKey, api *btc.ExplorerAPI,
 		}
 
 		counter++
+		if counter >= 6 {
+			log.Info("Waited 30 seconds, still no spends found, " +
+				"re-triggering CLN request")
+			err = requestForceCloseCln(
+				peer, torProxy, *outPoint, identityECDH,
+			)
+			if err != nil {
+				return nil, fmt.Errorf("error requesting "+
+					"force close: %w", err)
+			}
+		}
 		if counter >= 12 {
 			return nil, errors.New("no spends found after 60 " +
 				"seconds, aborting re-try loop")
@@ -385,7 +400,7 @@ func connectPeer(peerHost, torProxy string, identity keychain.SingleKeyECDH,
 	return p, cleanup, nil
 }
 
-func requestForceClose(peerHost, torProxy string, channelPoint wire.OutPoint,
+func requestForceCloseLnd(peerHost, torProxy string, channelPoint wire.OutPoint,
 	identity keychain.SingleKeyECDH) error {
 
 	p, cleanup, err := connectPeer(
@@ -415,6 +430,32 @@ func requestForceClose(peerHost, torProxy string, channelPoint wire.OutPoint,
 		return err
 	}
 
+	// Wait a few seconds to give the peer time to process the message.
+	time.Sleep(5 * time.Second)
+
+	return nil
+}
+
+func requestForceCloseCln(peerHost, torProxy string, channelPoint wire.OutPoint,
+	identity keychain.SingleKeyECDH) error {
+
+	p, cleanup, err := connectPeer(
+		peerHost, torProxy, identity, dialTimeout,
+	)
+	defer func() {
+		_ = cleanup()
+	}()
+
+	if err != nil {
+		return fmt.Errorf("error connecting to peer: %w", err)
+	}
+
+	channelID := lnwire.NewChanIDFromOutPoint(channelPoint)
+
+	// Channel ID (32 byte) + u16 for the data length (which will be 0).
+	data := make([]byte, 34)
+	copy(data[:32], channelID[:])
+
 	log.Infof("Sending channel error message to peer to trigger force "+
 		"close of channel %v", channelPoint)
 

cmd/chantools/zombierecovery_findmatches.go

@@ -316,7 +316,7 @@ func (c *zombieRecoveryFindMatchesCommand) Execute(_ *cobra.Command,
 			Node1:   node1,
 		}
 
-		folder := "results/match-" + node1
+		folder := fmt.Sprintf("%s/match-%s", ResultsDir, node1)
 		today := time.Now().Format("2006-01-02")
 		for node2, match := range node1map {
 			err = os.MkdirAll(folder, 0755)