From d05155cadb9588fb051887b296e0ac14bd384105 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 08:52:57 +0200
Subject: [PATCH 01/14] app: add teku simnet integration test

---
 app/app.go                        |   3 +-
 app/lifecycle/manager.go          |   2 +-
 app/lifecycle/order.go            |   1 +
 app/lifecycle/orderstop_string.go |   7 +-
 app/simnet_test.go                | 186 +++++++++++++++++++++++++++---
 testutil/beaconmock/beaconmock.go |  24 +++-
 testutil/beaconmock/options.go    |   9 +-
 testutil/beaconmock/server.go     |  14 ++-
 8 files changed, 216 insertions(+), 30 deletions(-)

diff --git a/app/app.go b/app/app.go
index eac8c19af..0dec28533 100644
--- a/app/app.go
+++ b/app/app.go
@@ -252,7 +252,7 @@ func wireSimNetCoreWorkflow(life *lifecycle.Manager, conf Config, manifest Manif
 	if err != nil {
 		return err
 	}
-	conf.BeaconNodeAddr = bmock.HTTPServerAddr
+	conf.BeaconNodeAddr = bmock.HTTPAddr()
 
 	sched, err := scheduler.New(corePubkeys, bmock)
 	if err != nil {
@@ -316,6 +316,7 @@ func wireSimNetCoreWorkflow(life *lifecycle.Manager, conf Config, manifest Manif
 	life.RegisterStart(lifecycle.AsyncAppCtx, lifecycle.StartLeaderCast, lifecycle.HookFunc(consensus.Run))
 	life.RegisterStart(lifecycle.AsyncBackground, lifecycle.StartScheduler, lifecycle.HookFuncErr(sched.Run))
 	life.RegisterStop(lifecycle.StopScheduler, lifecycle.HookFuncMin(sched.Stop))
+	life.RegisterStop(lifecycle.StopBeaconMock, lifecycle.HookFuncErr(bmock.Close))
 
 	return nil
 }
diff --git a/app/lifecycle/manager.go b/app/lifecycle/manager.go
index 0bd456af7..f8bed7e02 100644
--- a/app/lifecycle/manager.go
+++ b/app/lifecycle/manager.go
@@ -225,7 +225,7 @@ func run(appCtx context.Context, startHooks, stopHooks []hook) error {
 	stop := func(hook hook) {
 		err := hook.Func.Call(stopCtx)
 		if errors.Is(stopCtx.Err(), context.DeadlineExceeded) {
-			cacheErr(errors.New("shutdown timeout"))
+			cacheErr(errors.New("shutdown timeout", z.Str("hook", hook.Label)))
 		} else if err != nil && !errors.Is(err, context.Canceled) {
 			cacheErr(errors.Wrap(err, "stop hook", z.Str("hook", hook.Label)))
 			cancel() // Cancel the graceful stop context.
diff --git a/app/lifecycle/order.go b/app/lifecycle/order.go
index 75a9faaf7..f2f2c1004 100644
--- a/app/lifecycle/order.go
+++ b/app/lifecycle/order.go
@@ -38,5 +38,6 @@ const (
 	StopP2PTCPNode
 	StopP2PUDPNode
 	StopMonitoringAPI
+	StopBeaconMock // Need to close this before validator API, since it can hold long lived connections.
 	StopValidatorAPI
 )
diff --git a/app/lifecycle/orderstop_string.go b/app/lifecycle/orderstop_string.go
index 23da88958..d1ba1f411 100644
--- a/app/lifecycle/orderstop_string.go
+++ b/app/lifecycle/orderstop_string.go
@@ -28,12 +28,13 @@ func _() {
 	_ = x[StopP2PTCPNode-9]
 	_ = x[StopP2PUDPNode-10]
 	_ = x[StopMonitoringAPI-11]
-	_ = x[StopValidatorAPI-12]
+	_ = x[StopBeaconMock-12]
+	_ = x[StopValidatorAPI-13]
 }
 
-const _OrderStop_name = "TracingSchedulerP2PPeerDBP2PTCPNodeP2PUDPNodeMonitoringAPIValidatorAPI"
+const _OrderStop_name = "TracingSchedulerP2PPeerDBP2PTCPNodeP2PUDPNodeMonitoringAPIBeaconMockValidatorAPI"
 
-var _OrderStop_index = [...]uint8{0, 7, 16, 25, 35, 45, 58, 70}
+var _OrderStop_index = [...]uint8{0, 7, 16, 25, 35, 45, 58, 68, 80}
 
 func (i OrderStop) String() string {
 	i -= 6
diff --git a/app/simnet_test.go b/app/simnet_test.go
index d67a7b957..c765d3fe6 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -16,6 +16,14 @@ package app_test
 
 import (
 	"context"
+	"crypto/ecdsa"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
 	"testing"
 	"time"
 
@@ -31,11 +39,39 @@ import (
 	"github.com/obolnetwork/charon/tbls/tblsconv"
 	"github.com/obolnetwork/charon/testutil"
 	"github.com/obolnetwork/charon/testutil/beaconmock"
+	"github.com/obolnetwork/charon/testutil/keystore"
 )
 
-func TestSimnetNoNetwork(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+var integration = flag.Bool("integration", false, "Forces docker based integration test")
+
+func TestSimnetNoNetwork_TekuVC(t *testing.T) {
+	hasDocker := exec.Command("docker", "--version").Run() == nil
+	if !hasDocker && !*integration {
+		t.Skip("Skipping Teku integration test since no docker found")
+	} else if !hasDocker {
+		t.Fatal("docker command not found")
+	}
+
+	args := newSimnetArgs(t)
+	args = startTeku(t, args)
+	testSimnet(t, args)
+}
+
+func TestSimnetNoNetwork_MockVCs(t *testing.T) {
+	testSimnet(t, newSimnetArgs(t))
+}
+
+type simnetArgs struct {
+	N          int
+	VMocks     []bool
+	VAPIAddrs  []string
+	P2PKeys    []*ecdsa.PrivateKey
+	SimnetKeys []*bls_sig.SecretKey
+	Manifest   app.Manifest
+}
+
+func newSimnetArgs(t *testing.T) simnetArgs {
+	t.Helper()
 
 	const n = 3
 	manifest, p2pKeys, secretShares := app.NewClusterForT(t, 1, n, n, 99)
@@ -47,6 +83,37 @@ func TestSimnetNoNetwork(t *testing.T) {
 		secrets = append(secrets, secret)
 	}
 
+	var (
+		vmocks    []bool
+		vapiAddrs []string
+	)
+	for i := 0; i < n; i++ {
+		vmocks = append(vmocks, true)
+		vapiAddrs = append(vapiAddrs, testutil.AvailableAddr(t).String())
+	}
+
+	return simnetArgs{
+		N:          n,
+		VMocks:     vmocks,
+		VAPIAddrs:  vapiAddrs,
+		P2PKeys:    p2pKeys,
+		SimnetKeys: secrets,
+		Manifest:   manifest,
+	}
+}
+
+type simResult struct {
+	Duty   core.Duty
+	Pubkey core.PubKey
+	Data   core.AggSignedData
+}
+
+// testSimnet spins of a simnet cluster or N charon nodes connected via in-memory transports.
+// It asserts successful end-2-end attestation broadcast from all nodes for 2 slots.
+func testSimnet(t *testing.T, args simnetArgs) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+
 	parSigExFunc := parsigex.NewMemExFunc()
 	lcastTransportFunc := leadercast.NewMemTransportFunc(ctx)
 
@@ -54,16 +121,16 @@ func TestSimnetNoNetwork(t *testing.T) {
 		eg      errgroup.Group
 		results = make(chan simResult)
 	)
-	for i := 0; i < n; i++ {
+	for i := 0; i < args.N; i++ {
 		conf := app.Config{
-			SimnetVMock:      true,
+			SimnetVMock:      args.VMocks[i],
 			MonitoringAddr:   testutil.AvailableAddr(t).String(), // Random monitoring address
-			ValidatorAPIAddr: testutil.AvailableAddr(t).String(), // Random validatorapi address
+			ValidatorAPIAddr: args.VAPIAddrs[i],
 			TestConfig: app.TestConfig{
-				Manifest:           &manifest,
-				P2PKey:             p2pKeys[i],
+				Manifest:           &args.Manifest,
+				P2PKey:             args.P2PKeys[i],
 				DisablePing:        true,
-				SimnetKeys:         []*bls_sig.SecretKey{secrets[i]},
+				SimnetKeys:         []*bls_sig.SecretKey{args.SimnetKeys[i]},
 				ParSigExFunc:       parSigExFunc,
 				LcastTransportFunc: lcastTransportFunc,
 				BroadcastCallback: func(ctx context.Context, duty core.Duty, key core.PubKey, data core.AggSignedData) error {
@@ -83,7 +150,7 @@ func TestSimnetNoNetwork(t *testing.T) {
 		})
 	}
 
-	pubkey, err := tblsconv.KeyToCore(manifest.PublicKeys()[0])
+	pubkey, err := tblsconv.KeyToCore(args.Manifest.PublicKeys()[0])
 	require.NoError(t, err)
 
 	// Assert results
@@ -107,7 +174,7 @@ func TestSimnetNoNetwork(t *testing.T) {
 
 			// Assert we get results from all peers.
 			counts[res.Duty]++
-			if counts[res.Duty] == n {
+			if counts[res.Duty] == args.N {
 				remaining--
 			}
 			if remaining != 0 {
@@ -123,8 +190,97 @@ func TestSimnetNoNetwork(t *testing.T) {
 	require.NoError(t, eg.Wait())
 }
 
-type simResult struct {
-	Duty   core.Duty
-	Pubkey core.PubKey
-	Data   core.AggSignedData
+// startTeku starts a teku validator client for node0 and returns updated args.
+func startTeku(t *testing.T, args simnetArgs) simnetArgs {
+	t.Helper()
+
+	// Configure teku as VC for node0
+	args.VMocks[0] = false
+
+	// Write private share keystore and password
+	tempDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	err = keystore.StoreSimnetKeys([]*bls_sig.SecretKey{args.SimnetKeys[0]}, tempDir)
+	require.NoError(t, err)
+	err = os.WriteFile(path.Join(tempDir, "keystore-simnet-0.txt"), []byte("simnet"), 0o644)
+	require.NoError(t, err)
+
+	// Change VAPI bind address to host external IP
+	args.VAPIAddrs[0] = strings.Replace(args.VAPIAddrs[0], "127.0.0.1", externalIP(t), 1)
+
+	tekuArgs := []string{
+		"validator-client",
+		"--network=auto",
+		"--validator-keys=/keys:/keys",
+		fmt.Sprintf("--beacon-node-api-endpoint=http://%s", args.VAPIAddrs[0]),
+	}
+
+	// Configure docker
+	name := fmt.Sprint(time.Now().UnixNano())
+	dockerArgs := []string{
+		"run",
+		"--rm",
+		fmt.Sprintf("--name=%s", name),
+		fmt.Sprintf("--volume=%s:/keys", tempDir),
+		"consensys/teku:latest",
+	}
+	dockerArgs = append(dockerArgs, tekuArgs...)
+
+	// Start teku
+	ctx, cancel := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, "docker", dockerArgs...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	require.NoError(t, cmd.Start())
+
+	// Kill the container when done.
+	t.Cleanup(func() {
+		cancel()
+		_ = exec.Command("docker", "kill", name).Run() // Teku in docker doesn't support sig term...
+		_ = cmd.Wait()
+	})
+
+	return args
+}
+
+// externalIP returns the hosts external IP.
+// Copied from https://stackoverflow.com/questions/23558425/how-do-i-get-the-local-ip-address-in-go.
+func externalIP(t *testing.T) string {
+	t.Helper()
+
+	ifaces, err := net.Interfaces()
+	require.NoError(t, err)
+
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagUp == 0 {
+			continue // interface down
+		}
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue // loopback interface
+		}
+		addrs, err := iface.Addrs()
+		require.NoError(t, err)
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() {
+				continue
+			}
+			ip = ip.To4()
+			if ip == nil {
+				continue // not an ipv4 address
+			}
+
+			return ip.String()
+		}
+	}
+
+	t.Fatal("no network?")
+
+	return ""
 }
diff --git a/testutil/beaconmock/beaconmock.go b/testutil/beaconmock/beaconmock.go
index 0da7e8bd4..c57a6409b 100644
--- a/testutil/beaconmock/beaconmock.go
+++ b/testutil/beaconmock/beaconmock.go
@@ -37,12 +37,15 @@ package beaconmock
 
 import (
 	"context"
+	"net/http"
 	"time"
 
 	eth2client "github.com/attestantio/go-eth2-client"
 	eth2v1 "github.com/attestantio/go-eth2-client/api/v1"
 	eth2p0 "github.com/attestantio/go-eth2-client/spec/phase0"
 	"github.com/jonboulle/clockwork"
+
+	"github.com/obolnetwork/charon/app/errors"
 )
 
 // Interface assertions.
@@ -69,7 +72,7 @@ func New(opts ...Option) (Mock, error) {
 	}
 
 	// Then configure the mock
-	mock := defaultMock(httpMock, "http://"+httpServer.Addr, temp.clock)
+	mock := defaultMock(httpMock, httpServer, temp.clock)
 	for _, opt := range opts {
 		opt(&mock)
 	}
@@ -81,9 +84,9 @@ func New(opts ...Option) (Mock, error) {
 // Create a new instance with default behaviour via New and then override any function.
 type Mock struct {
 	HTTPMock
-	HTTPServerAddr string
-	overrides      []staticOverride
-	clock          clockwork.Clock
+	httpServer *http.Server
+	overrides  []staticOverride
+	clock      clockwork.Clock
 
 	AttestationDataFunc    func(context.Context, eth2p0.Slot, eth2p0.CommitteeIndex) (*eth2p0.AttestationData, error)
 	AttesterDutiesFunc     func(context.Context, eth2p0.Epoch, []eth2p0.ValidatorIndex) ([]*eth2v1.AttesterDuty, error)
@@ -134,3 +137,16 @@ func (Mock) Name() string {
 func (Mock) Address() string {
 	return "mock-address"
 }
+
+func (m Mock) HTTPAddr() string {
+	return "http://" + m.httpServer.Addr
+}
+
+func (m Mock) Close() error {
+	err := m.httpServer.Close()
+	if err != nil {
+		return errors.Wrap(err, "close server")
+	}
+
+	return nil
+}
diff --git a/testutil/beaconmock/options.go b/testutil/beaconmock/options.go
index 7e3de6121..d96936f75 100644
--- a/testutil/beaconmock/options.go
+++ b/testutil/beaconmock/options.go
@@ -20,6 +20,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"hash/fnv"
+	"net/http"
 	"sort"
 	"strings"
 	"time"
@@ -265,11 +266,11 @@ func WithClock(clock clockwork.Clock) Option {
 }
 
 // defaultMock returns a minimum viable mock that doesn't panic and returns mostly empty responses.
-func defaultMock(httpMock HTTPMock, addr string, clock clockwork.Clock) Mock {
+func defaultMock(httpMock HTTPMock, httpServer *http.Server, clock clockwork.Clock) Mock {
 	return Mock{
-		clock:          clock,
-		HTTPMock:       httpMock,
-		HTTPServerAddr: addr,
+		clock:      clock,
+		HTTPMock:   httpMock,
+		httpServer: httpServer,
 		ProposerDutiesFunc: func(ctx context.Context, epoch eth2p0.Epoch, indices []eth2p0.ValidatorIndex) ([]*eth2v1.ProposerDuty, error) {
 			return []*eth2v1.ProposerDuty{}, nil
 		},
diff --git a/testutil/beaconmock/server.go b/testutil/beaconmock/server.go
index 33db94295..480dd2dc1 100644
--- a/testutil/beaconmock/server.go
+++ b/testutil/beaconmock/server.go
@@ -62,6 +62,8 @@ type staticOverride struct {
 
 // newHTTPServer returns a beacon API mock http server.
 func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, error) {
+	shutdown := make(chan struct{})
+
 	endpoints := []struct {
 		Path    string
 		Handler http.HandlerFunc
@@ -96,7 +98,10 @@ func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, erro
 			Path: "/eth/v1/events",
 			Handler: func(w http.ResponseWriter, r *http.Request) {
 				// TODO(corver): Send keep alives
-				<-r.Context().Done()
+				select {
+				case <-shutdown:
+				case <-r.Context().Done():
+				}
 			},
 		},
 	}
@@ -146,7 +151,12 @@ func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, erro
 		_, _ = w.Write(resp)
 	}))
 
-	return &http.Server{Addr: addr, Handler: r}, nil
+	s := http.Server{Addr: addr, Handler: r}
+	s.RegisterOnShutdown(func() {
+		close(shutdown)
+	})
+
+	return &s, nil
 }
 
 // newHTTPMock starts and returns a static beacon mock http server and client.

From ee4b04d19a9a127ac8527c693e1572e1c497d62c Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 09:21:47 +0200
Subject: [PATCH 02/14] cleanup

---
 .github/workflows/test.yml |  2 +-
 app/simnet_test.go         | 27 ++++++++++++++++++++++++---
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1f2ac15ac..5f4886a59 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test -coverprofile=coverage.out -covermode=atomic ./...
+    - run: go test -coverprofile=coverage.out -covermode=atomic -short ./...
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2.1.0
       with:
diff --git a/app/simnet_test.go b/app/simnet_test.go
index c765d3fe6..7d2ecc600 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -32,6 +32,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/obolnetwork/charon/app"
+	"github.com/obolnetwork/charon/app/errors"
 	"github.com/obolnetwork/charon/core"
 	"github.com/obolnetwork/charon/core/leadercast"
 	"github.com/obolnetwork/charon/core/parsigex"
@@ -45,6 +46,9 @@ import (
 var integration = flag.Bool("integration", false, "Forces docker based integration test")
 
 func TestSimnetNoNetwork_TekuVC(t *testing.T) {
+	if testing.Short() {
+		return
+	}
 	hasDocker := exec.Command("docker", "--version").Run() == nil
 	if !hasDocker && !*integration {
 		t.Skip("Skipping Teku integration test since no docker found")
@@ -68,6 +72,7 @@ type simnetArgs struct {
 	P2PKeys    []*ecdsa.PrivateKey
 	SimnetKeys []*bls_sig.SecretKey
 	Manifest   app.Manifest
+	ErrChan    chan error
 }
 
 func newSimnetArgs(t *testing.T) simnetArgs {
@@ -99,6 +104,7 @@ func newSimnetArgs(t *testing.T) simnetArgs {
 		P2PKeys:    p2pKeys,
 		SimnetKeys: secrets,
 		Manifest:   manifest,
+		ErrChan:    make(chan error, 1),
 	}
 }
 
@@ -187,6 +193,17 @@ func testSimnet(t *testing.T, args simnetArgs) {
 		}
 	}()
 
+	// Wire err channel (for docker errors)
+	eg.Go(func() error {
+		select {
+		case <-ctx.Done():
+			return nil
+		case err := <-args.ErrChan:
+			cancel()
+			return err
+		}
+	})
+
 	require.NoError(t, eg.Wait())
 }
 
@@ -231,13 +248,17 @@ func startTeku(t *testing.T, args simnetArgs) simnetArgs {
 	cmd := exec.CommandContext(ctx, "docker", dockerArgs...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	require.NoError(t, cmd.Start())
-
+	go func() {
+		err = cmd.Run()
+		if ctx.Err() != nil {
+			return
+		}
+		args.ErrChan <- errors.Wrap(err, "docker command failed (see logging)")
+	}()
 	// Kill the container when done.
 	t.Cleanup(func() {
 		cancel()
 		_ = exec.Command("docker", "kill", name).Run() // Teku in docker doesn't support sig term...
-		_ = cmd.Wait()
 	})
 
 	return args

From 21bf709adccdc61fc06b8e8cfe75cbc3eb60d0d4 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 09:23:27 +0200
Subject: [PATCH 03/14] cleanup

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5f4886a59..d5d2560c0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test -coverprofile=coverage.out -covermode=atomic -short ./...
+    - run: go test -coverprofile=coverage.out -covermode=atomic -v ./...
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2.1.0
       with:

From 220fb103e6e9f46ef6867d23c22941054f4e6849 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 09:32:11 +0200
Subject: [PATCH 04/14] cleanup

---
 .github/workflows/test.yml | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d5d2560c0..71a06f9d5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,9 +20,28 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test -coverprofile=coverage.out -covermode=atomic -v ./...
+    - run: go test -coverprofile=coverage.out -covermode=atomic -short ./...
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2.1.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         files: coverage.out
+
+  integration_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.1'
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - run: which docker
+      - run: docker pull consensys/teku:latest
+      - run: go test github.com/obolnetwork/charon/app -integration -slow

From d028fe277c7fc69a3ce772d7f48eb8679f4138a0 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 09:35:17 +0200
Subject: [PATCH 05/14] cleanup

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 71a06f9d5..5bc2b949b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -44,4 +44,4 @@ jobs:
             ${{ runner.os }}-go-
       - run: which docker
       - run: docker pull consensys/teku:latest
-      - run: go test github.com/obolnetwork/charon/app -integration -slow
+      - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=1m

From 6d89dbfdc0a7774718f71baaec99eaaece1b48d3 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 09:48:49 +0200
Subject: [PATCH 06/14] cleanup

---
 .github/workflows/test.yml    |  1 -
 testutil/beaconmock/server.go | 11 ++++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5bc2b949b..4ee9fb24d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -42,6 +42,5 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-      - run: which docker
       - run: docker pull consensys/teku:latest
       - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=1m
diff --git a/testutil/beaconmock/server.go b/testutil/beaconmock/server.go
index 480dd2dc1..6c264f511 100644
--- a/testutil/beaconmock/server.go
+++ b/testutil/beaconmock/server.go
@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
 	eth2client "github.com/attestantio/go-eth2-client"
@@ -62,6 +63,7 @@ type staticOverride struct {
 
 // newHTTPServer returns a beacon API mock http server.
 func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, error) {
+	debug := os.Getenv("BEACONMOCK_DEBUG") == "true"
 	shutdown := make(chan struct{})
 
 	endpoints := []struct {
@@ -114,7 +116,9 @@ func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, erro
 		r.Handle(e.Path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			ctx := log.WithTopic(r.Context(), "bmock")
 			ctx = log.WithCtx(ctx, z.Str("path", e.Path))
-			log.Debug(ctx, "Serving mocked endpoint")
+			if debug {
+				log.Debug(ctx, "Serving mocked endpoint")
+			}
 			e.Handler(w, r)
 		}))
 	}
@@ -146,8 +150,9 @@ func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, erro
 
 			return
 		}
-
-		log.Debug(ctx, "Serving static endpoint")
+		if debug {
+			log.Debug(ctx, "Serving static endpoint")
+		}
 		_, _ = w.Write(resp)
 	}))
 

From aa7680d1be6b292ea60381f59a7b127d17b254f7 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 09:56:48 +0200
Subject: [PATCH 07/14] cleanup

---
 app/simnet_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/simnet_test.go b/app/simnet_test.go
index 7d2ecc600..98e088b97 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -242,7 +242,7 @@ func startTeku(t *testing.T, args simnetArgs) simnetArgs {
 		"consensys/teku:latest",
 	}
 	dockerArgs = append(dockerArgs, tekuArgs...)
-
+	t.Logf("docker args: %v", dockerArgs)
 	// Start teku
 	ctx, cancel := context.WithCancel(context.Background())
 	cmd := exec.CommandContext(ctx, "docker", dockerArgs...)

From 928b7773f6be98d64c04ad70dafb6a3e5cbbfbc7 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 10:03:30 +0200
Subject: [PATCH 08/14] cleanup

---
 .github/workflows/test.yml |  2 +-
 app/app.go                 |  7 ++++---
 app/simnet_test.go         | 12 +++---------
 3 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4ee9fb24d..985e7a13d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test -coverprofile=coverage.out -covermode=atomic -short ./...
+    - run: go test -coverprofile=coverage.out -covermode=atomic ./...
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2.1.0
       with:
diff --git a/app/app.go b/app/app.go
index 0dec28533..a67613b56 100644
--- a/app/app.go
+++ b/app/app.go
@@ -442,14 +442,15 @@ func wireValidatorMock(conf Config, pubshares []eth2p0.BLSPubKey, sched core.Sch
 			addr := "http://" + conf.ValidatorAPIAddr
 			cl, err := eth2http.New(ctx, eth2http.WithLogLevel(1), eth2http.WithAddress(addr))
 			if err != nil {
-				log.Warn(ctx, "validatorapi client", z.Err(err))
+				log.Warn(ctx, "Cannot connect to validatorapi", z.Err(err))
+				return
 			}
 
 			err = validatormock.Attest(ctx, cl.(*eth2http.Service), signer, eth2p0.Slot(duty.Slot), pubshares...)
 			if err != nil {
-				log.Warn(ctx, "attestation failed", z.Err(err))
+				log.Warn(ctx, "Attestation failed", z.Err(err))
 			} else {
-				log.Info(ctx, "attestation success", z.I64("slot", duty.Slot))
+				log.Info(ctx, "Attestation success", z.I64("slot", duty.Slot))
 			}
 		}()
 
diff --git a/app/simnet_test.go b/app/simnet_test.go
index 98e088b97..37f617820 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -43,17 +43,11 @@ import (
 	"github.com/obolnetwork/charon/testutil/keystore"
 )
 
-var integration = flag.Bool("integration", false, "Forces docker based integration test")
+var integration = flag.Bool("integration", false, "Enable docker based integration test")
 
 func TestSimnetNoNetwork_TekuVC(t *testing.T) {
-	if testing.Short() {
-		return
-	}
-	hasDocker := exec.Command("docker", "--version").Run() == nil
-	if !hasDocker && !*integration {
-		t.Skip("Skipping Teku integration test since no docker found")
-	} else if !hasDocker {
-		t.Fatal("docker command not found")
+	if !*integration {
+		t.Skip("Skipping Teku integration test")
 	}
 
 	args := newSimnetArgs(t)

From 53ecf8c5031a4ee49008cdd258369f24bf060a1a Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 10:08:23 +0200
Subject: [PATCH 09/14] cleanup

---
 .github/workflows/test.yml | 3 ++-
 app/simnet_test.go         | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 985e7a13d..53be5deab 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -29,6 +29,7 @@ jobs:
 
   integration_tests:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -43,4 +44,4 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
       - run: docker pull consensys/teku:latest
-      - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=1m
+      - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=5m
diff --git a/app/simnet_test.go b/app/simnet_test.go
index 37f617820..4a3cadd8e 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -43,6 +43,7 @@ import (
 	"github.com/obolnetwork/charon/testutil/keystore"
 )
 
+//go:generate go test . -run=TestSimnetNoNetwork_TekuVC -integration
 var integration = flag.Bool("integration", false, "Enable docker based integration test")
 
 func TestSimnetNoNetwork_TekuVC(t *testing.T) {

From 824f565b10fd866b70348a1a9d7c88fe6d91e3b3 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 12:15:43 +0200
Subject: [PATCH 10/14] cleanup

---
 app/simnet_test.go | 46 ++++++++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/app/simnet_test.go b/app/simnet_test.go
index 4a3cadd8e..9d9e89d00 100644
--- a/app/simnet_test.go
+++ b/app/simnet_test.go
@@ -43,7 +43,7 @@ import (
 	"github.com/obolnetwork/charon/testutil/keystore"
 )
 
-//go:generate go test . -run=TestSimnetNoNetwork_TekuVC -integration
+//go:generate go test . -run=TestSimnetNoNetwork_TekuVC -integration -v
 var integration = flag.Bool("integration", false, "Enable docker based integration test")
 
 func TestSimnetNoNetwork_TekuVC(t *testing.T) {
@@ -52,7 +52,7 @@ func TestSimnetNoNetwork_TekuVC(t *testing.T) {
 	}
 
 	args := newSimnetArgs(t)
-	args = startTeku(t, args)
+	args = startTeku(t, args, 0)
 	testSimnet(t, args)
 }
 
@@ -70,6 +70,7 @@ type simnetArgs struct {
 	ErrChan    chan error
 }
 
+// newSimnetArgs defines the default simnet test args.
 func newSimnetArgs(t *testing.T) simnetArgs {
 	t.Helper()
 
@@ -103,12 +104,6 @@ func newSimnetArgs(t *testing.T) simnetArgs {
 	}
 }
 
-type simResult struct {
-	Duty   core.Duty
-	Pubkey core.PubKey
-	Data   core.AggSignedData
-}
-
 // testSimnet spins of a simnet cluster or N charon nodes connected via in-memory transports.
 // It asserts successful end-2-end attestation broadcast from all nodes for 2 slots.
 func testSimnet(t *testing.T, args simnetArgs) {
@@ -118,6 +113,12 @@ func testSimnet(t *testing.T, args simnetArgs) {
 	parSigExFunc := parsigex.NewMemExFunc()
 	lcastTransportFunc := leadercast.NewMemTransportFunc(ctx)
 
+	type simResult struct {
+		Duty   core.Duty
+		Pubkey core.PubKey
+		Data   core.AggSignedData
+	}
+
 	var (
 		eg      errgroup.Group
 		results = make(chan simResult)
@@ -202,29 +203,30 @@ func testSimnet(t *testing.T, args simnetArgs) {
 	require.NoError(t, eg.Wait())
 }
 
-// startTeku starts a teku validator client for node0 and returns updated args.
-func startTeku(t *testing.T, args simnetArgs) simnetArgs {
+// startTeku starts a teku validator client for the provided node and returns updated args.
+func startTeku(t *testing.T, args simnetArgs, node int) simnetArgs {
 	t.Helper()
 
 	// Configure teku as VC for node0
-	args.VMocks[0] = false
+	args.VMocks[node] = false
 
 	// Write private share keystore and password
 	tempDir, err := os.MkdirTemp("", "")
 	require.NoError(t, err)
-	err = keystore.StoreSimnetKeys([]*bls_sig.SecretKey{args.SimnetKeys[0]}, tempDir)
+	err = keystore.StoreSimnetKeys([]*bls_sig.SecretKey{args.SimnetKeys[node]}, tempDir)
 	require.NoError(t, err)
 	err = os.WriteFile(path.Join(tempDir, "keystore-simnet-0.txt"), []byte("simnet"), 0o644)
 	require.NoError(t, err)
 
 	// Change VAPI bind address to host external IP
-	args.VAPIAddrs[0] = strings.Replace(args.VAPIAddrs[0], "127.0.0.1", externalIP(t), 1)
+	args.VAPIAddrs[node] = strings.Replace(args.VAPIAddrs[node], "127.0.0.1", externalIP(t), 1)
 
+	// Teku arguments
 	tekuArgs := []string{
 		"validator-client",
 		"--network=auto",
 		"--validator-keys=/keys:/keys",
-		fmt.Sprintf("--beacon-node-api-endpoint=http://%s", args.VAPIAddrs[0]),
+		fmt.Sprintf("--beacon-node-api-endpoint=http://%s", args.VAPIAddrs[node]),
 	}
 
 	// Configure docker
@@ -234,26 +236,30 @@ func startTeku(t *testing.T, args simnetArgs) simnetArgs {
 		"--rm",
 		fmt.Sprintf("--name=%s", name),
 		fmt.Sprintf("--volume=%s:/keys", tempDir),
+		"--user=root", // Root required to read volume files in GitHub actions.
 		"consensys/teku:latest",
 	}
 	dockerArgs = append(dockerArgs, tekuArgs...)
 	t.Logf("docker args: %v", dockerArgs)
+
 	// Start teku
 	ctx, cancel := context.WithCancel(context.Background())
-	cmd := exec.CommandContext(ctx, "docker", dockerArgs...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
 	go func() {
-		err = cmd.Run()
+		c := exec.CommandContext(ctx, "docker", dockerArgs...)
+		c.Stdout = os.Stdout
+		c.Stderr = os.Stderr
+		err = c.Run()
 		if ctx.Err() != nil {
+			// Expected shutdown
 			return
 		}
 		args.ErrChan <- errors.Wrap(err, "docker command failed (see logging)")
 	}()
-	// Kill the container when done.
+
+	// Kill the container when done (context cancel is not enough for some reason).
 	t.Cleanup(func() {
 		cancel()
-		_ = exec.Command("docker", "kill", name).Run() // Teku in docker doesn't support sig term...
+		_ = exec.Command("docker", "kill", name).Run()
 	})
 
 	return args

From b50efad2b461d3d9685bd92f7a3181762e57f69a Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 12:20:03 +0200
Subject: [PATCH 11/14] cleanup

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 53be5deab..75c3339ef 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -43,5 +43,7 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+      - run: ls -la /var/lib/docker
+      - run: ls -la /usr/bin
       - run: docker pull consensys/teku:latest
       - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=5m

From c1636ec3a7c67587d5ae044c90ffc4db72bb5a72 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 12:21:57 +0200
Subject: [PATCH 12/14] cleanup

---
 .github/workflows/test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 75c3339ef..53be5deab 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -43,7 +43,5 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-      - run: ls -la /var/lib/docker
-      - run: ls -la /usr/bin
       - run: docker pull consensys/teku:latest
       - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=5m

From 5d3f7501f8d00d3d2aff5bf27410c78f5baa9ec0 Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 12:24:42 +0200
Subject: [PATCH 13/14] cleanup

---
 testutil/beaconmock/server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testutil/beaconmock/server.go b/testutil/beaconmock/server.go
index 6c264f511..7510acf7b 100644
--- a/testutil/beaconmock/server.go
+++ b/testutil/beaconmock/server.go
@@ -63,7 +63,7 @@ type staticOverride struct {
 
 // newHTTPServer returns a beacon API mock http server.
 func newHTTPServer(addr string, overrides ...staticOverride) (*http.Server, error) {
-	debug := os.Getenv("BEACONMOCK_DEBUG") == "true"
+	debug := os.Getenv("BEACONMOCK_DEBUG") == "true" // NOTE: These logs are verbose, so disabled by default.
 	shutdown := make(chan struct{})
 
 	endpoints := []struct {

From d428745d37918715406043db1ee920f8029f757c Mon Sep 17 00:00:00 2001
From: corverroos <corver.roos@gmail.com>
Date: Thu, 24 Mar 2022 12:26:20 +0200
Subject: [PATCH 14/14] cleanup

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 53be5deab..35c654fb8 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test -coverprofile=coverage.out -covermode=atomic ./...
+    - run: go test -coverprofile=coverage.out -covermode=atomic -timeout=5m ./...
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2.1.0
       with:
@@ -44,4 +44,4 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
       - run: docker pull consensys/teku:latest
-      - run: go test github.com/obolnetwork/charon/app -integration -slow -v -timeout=5m
+      - run: go test -timeout=5m -v github.com/obolnetwork/charon/app -integration -slow