feat (tests): enable multiple upgrades for automated upgrade tests

tests/e2e/e2e_suite_test.go

@@ -19,16 +19,19 @@ const (
 	defaultChainID        = "evmos_9000-1"
 	defaultManagerNetwork = "evmos-local"
 	tharsisRepo           = "tharsishq/evmos"
-	firstUpgradeHeight    = 25
+
+	// blocksAfterUpgrade defines how many blocks must be produced after an upgrade is
+	// considered successful
+	blocksAfterUpgrade = 5
+	// upgradeHeightDelta defines the number of blocks after the proposal and the scheduled upgrade
+	upgradeHeightDelta = 10
 
 	relatedBuildPath = "../../build/"
 )
 
 type upgradeParams struct {
-	InitialVersion         string
-	TargetVersion          string
-	SoftwareUpgradeVersion string
-	MountPath              string
+	MountPath string
+	Upgrades  []upgradeConfig
 
 	ChainID     string
 	TargetRepo  string
@@ -61,105 +64,118 @@ func (s *IntegrationTestSuite) SetupSuite() {
 	}
 }
 
-func (s *IntegrationTestSuite) runInitialNode() {
+// runInitialNode builds a docker image capable of running an Evmos node with the given version.
+// After a successful build, it runs the container and checks if the node can produce blocks.
+func (s *IntegrationTestSuite) runInitialNode(version string) {
 	err := s.upgradeManager.BuildImage(
 		localRepository,
-		s.upgradeParams.InitialVersion,
+		version,
 		"./upgrade/Dockerfile.init",
 		".",
-		map[string]string{"INITIAL_VERSION": s.upgradeParams.InitialVersion},
+		map[string]string{"INITIAL_VERSION": version},
 	)
-	s.Require().NoError(err, "can't build initial container")
+	s.Require().NoError(err, "can't build container with Evmos version: %s", version)
 
-	node := upgrade.NewNode(localRepository, s.upgradeParams.InitialVersion)
+	node := upgrade.NewNode(localRepository, version)
 	node.SetEnvVars([]string{fmt.Sprintf("CHAIN_ID=%s", s.upgradeParams.ChainID)})
 
 	err = s.upgradeManager.RunNode(node)
-	s.Require().NoError(err, "can't run initial node")
+	s.Require().NoError(err, "can't run node with Evmos version: %s", version)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	// wait until node starts and produce some blocks
-	err = s.upgradeManager.WaitForHeight(ctx, 5)
+	err = s.upgradeManager.WaitForHeight(ctx, s.upgradeManager.HeightBeforeStop+5)
 	s.Require().NoError(err)
 
-	s.T().Logf("successfully started initial node version: [%s]", s.upgradeParams.InitialVersion)
+	s.T().Logf("successfully started node with Evmos version: [%s]", version)
 }
 
-func (s *IntegrationTestSuite) proposeUpgrade() {
+// proposeUpgrade submits an upgrade proposal to the chain that schedules an upgrade to
+// the given target version.
+func (s *IntegrationTestSuite) proposeUpgrade(name, target string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	// calculate upgrade height for the proposal
+	nodeHeight, err := s.upgradeManager.GetNodeHeight(ctx)
+	s.Require().NoError(err, "can't get block height from running node")
+	s.upgradeManager.UpgradeHeight = uint(nodeHeight + upgradeHeightDelta)
+
+	// if Evmos is lower than v10.x.x no need to use the legacy proposal
+	currentVersion, err := s.upgradeManager.GetNodeVersion(ctx)
+	s.Require().NoError(err, "can't get evmosd version from running node")
+	currentVersion = strings.TrimSpace(currentVersion)
+	if !strings.HasPrefix(currentVersion, "v") {
+		currentVersion = "v" + currentVersion
+	}
+	s.T().Logf("current version: '%s'", currentVersion)
+	cmp := upgrade.ByVersion([]string{currentVersion, "v10.0.0"})
+	legacyProposal := !cmp.Less(0, 1)
+	s.T().Logf("legacy proposal: %v", legacyProposal)
+
+	// create the proposal
 	exec, err := s.upgradeManager.CreateSubmitProposalExec(
-		s.upgradeParams.SoftwareUpgradeVersion,
+		name,
 		s.upgradeParams.ChainID,
-		firstUpgradeHeight,
+		s.upgradeManager.UpgradeHeight,
+		legacyProposal,
 	)
-	s.Require().NoError(err, "can't create submit proposal exec")
-	outBuf, errBuf, err := s.upgradeManager.RunExec(ctx, exec)
 	s.Require().NoErrorf(
 		err,
-		"failed to submit upgrade proposal; stdout: %s, stderr: %s", outBuf, errBuf,
+		"can't create the proposal to upgrade Evmos to %s at height %d with name %s",
+		target, s.upgradeManager.UpgradeHeight, name,
 	)
 
-	s.Require().Truef(
-		strings.Contains(outBuf.String(), "code: 0"),
-		"tx returned non code 0: %s %s", outBuf, errBuf,
-	)
-
-	s.T().Logf(
-		"successfully submitted upgrade proposal: upgrade height: [%d] upgrade version: [%s]",
-		firstUpgradeHeight,
-		s.upgradeParams.SoftwareUpgradeVersion,
-	)
-}
-
-func (s *IntegrationTestSuite) depositToProposal() {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	exec, err := s.upgradeManager.CreateDepositProposalExec(s.upgradeParams.ChainID)
-	s.Require().NoError(err, "can't create deposit to proposal tx exec")
 	outBuf, errBuf, err := s.upgradeManager.RunExec(ctx, exec)
 	s.Require().NoErrorf(
 		err,
-		"failed to submit deposit to proposal tx; stdout: %s, stderr: %s", outBuf, errBuf,
+		"failed to submit proposal to upgrade Evmos to %s at height %d\nstdout: %s,\nstderr: %s",
+		target, s.upgradeManager.UpgradeHeight, outBuf.String(), errBuf.String(),
 	)
 
 	s.Require().Truef(
 		strings.Contains(outBuf.String(), "code: 0"),
-		"tx returned non code 0: %s %s", outBuf, errBuf,
+		"tx returned non code 0:\nstdout: %s\nstderr: %s", outBuf.String(), errBuf.String(),
 	)
 
-	s.T().Logf("successfully deposited to proposal")
+	s.T().Logf(
+		"successfully submitted upgrade proposal: height: %d, name: %s",
+		s.upgradeManager.UpgradeHeight,
+		target,
+	)
 }
 
-func (s *IntegrationTestSuite) voteForProposal() {
+// voteForProposal votes for the upgrade proposal with the given id.
+func (s *IntegrationTestSuite) voteForProposal(id int) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	exec, err := s.upgradeManager.CreateVoteProposalExec(s.upgradeParams.ChainID)
+	exec, err := s.upgradeManager.CreateVoteProposalExec(s.upgradeParams.ChainID, id)
 	s.Require().NoError(err, "can't create vote for proposal exec")
 	outBuf, errBuf, err := s.upgradeManager.RunExec(ctx, exec)
 	s.Require().NoErrorf(
 		err,
-		"failed to vote for proposal tx; stdout: %s, stderr: %s", outBuf, errBuf,
+		"failed to vote for proposal tx;\nstdout: %s,\nstderr: %s", outBuf.String(), errBuf.String(),
 	)
 
 	s.Require().Truef(
 		strings.Contains(outBuf.String(), "code: 0"),
-		"tx returned non code 0: %s %s", outBuf, errBuf,
+		"tx returned non code 0:\nstdout: %s\nstderr: %s", outBuf.String(), errBuf.String(),
 	)
 
 	s.T().Logf("successfully voted for upgrade proposal")
 }
 
-func (s *IntegrationTestSuite) upgrade() {
+// upgrade upgrades the node to the given version using the given repo. The repository
+// can either be a local path or a remote repository.
+func (s *IntegrationTestSuite) upgrade(targetRepo, targetVersion string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	s.T().Log("wait for node to reach upgrade height...")
 	// wait for proposed upgrade height
-	err := s.upgradeManager.WaitForHeight(ctx, firstUpgradeHeight)
+	err := s.upgradeManager.WaitForHeight(ctx, int(s.upgradeManager.UpgradeHeight))
 	s.Require().NoError(err, "can't reach upgrade height")
 	buildDir := strings.Split(s.upgradeParams.MountPath, ":")[0]
 
@@ -168,37 +184,43 @@ func (s *IntegrationTestSuite) upgrade() {
 	err = s.upgradeManager.ExportState(buildDir)
 	s.Require().NoError(err, "can't export node container state to local")
 
-	s.T().Log("killing initial node...")
+	s.T().Log("killing node before upgrade...")
 	err = s.upgradeManager.KillCurrentNode()
 	s.Require().NoError(err, "can't kill current node")
 
 	s.T().Logf(
-		"starting upgraded node: version: [%s] mount point: [%s]",
-		s.upgradeParams.TargetVersion,
+		"starting upgraded node: repo: [%s] version: [%s] mount point: [%s]",
+		targetRepo,
+		targetVersion,
 		s.upgradeParams.MountPath,
 	)
 
-	node := upgrade.NewNode(s.upgradeParams.TargetRepo, s.upgradeParams.TargetVersion)
+	node := upgrade.NewNode(targetRepo, targetVersion)
 	node.Mount(s.upgradeParams.MountPath)
 	node.SetCmd([]string{"evmosd", "start", fmt.Sprintf("--chain-id=%s", s.upgradeParams.ChainID)})
 	err = s.upgradeManager.RunNode(node)
 	s.Require().NoError(err, "can't mount and run upgraded node container")
 
-	s.T().Log("node started! waiting for node to produce 10 blocks")
+	s.T().Logf("node started! waiting for node to produce %d blocks", blocksAfterUpgrade)
 	// make sure node produce blocks after upgrade
-	err = s.upgradeManager.WaitForHeight(ctx, firstUpgradeHeight+10)
-	s.Require().NoError(err, "node not produce blocks")
+	err = s.upgradeManager.WaitForHeight(ctx, int(s.upgradeManager.UpgradeHeight)+blocksAfterUpgrade)
+	s.Require().NoError(err, "node does not produce blocks after upgrade")
 }
 
+// TearDownSuite kills the running container, removes the network and mount path
 func (s *IntegrationTestSuite) TearDownSuite() {
 	if s.upgradeParams.SkipCleanup {
+		s.T().Logf("skipping cleanup... container %s will be left running", s.upgradeManager.ContainerID())
 		return
 	}
 	s.T().Log("tearing down e2e integration test suite...")
 
-	s.Require().NoError(s.upgradeManager.KillCurrentNode())
+	err := s.upgradeManager.KillCurrentNode()
+	if err != nil && !strings.Contains(err.Error(), "is not running") {
+		s.Require().NoError(err, "can't kill current node")
+	}
 
-	s.Require().NoError(s.upgradeManager.RemoveNetwork())
+	s.Require().NoError(s.upgradeManager.RemoveNetwork(), "can't remove network")
 
-	s.Require().NoError(os.RemoveAll(strings.Split(s.upgradeParams.MountPath, ":")[0]))
+	s.Require().NoError(os.RemoveAll(strings.Split(s.upgradeParams.MountPath, ":")[0]), "can't remove mount path")
 }

tests/e2e/e2e_test.go

@@ -1,10 +1,21 @@
 package e2e
 
+// TestUpgrade tests if an Evmos node can be upgraded from one version to another.
+// It iterates through the list of scheduled upgrades, that are defined using the input
+// arguments to the make command. The function then submits a proposal to upgrade the chain,
+// and finally upgrades the chain.
+// If the chain can be restarted after the upgrade(s), the test passes.
 func (s *IntegrationTestSuite) TestUpgrade() {
-	s.runInitialNode()
-	s.proposeUpgrade()
-	s.depositToProposal()
-	s.voteForProposal()
-	s.upgrade()
+	for idx, upgrade := range s.upgradeParams.Upgrades {
+		if idx == 0 {
+			// start initial node
+			s.runInitialNode(upgrade.version)
+			continue
+		}
+		s.T().Logf("(upgrade %d): UPGRADING TO %s WITH PROPOSAL NAME %s", idx, upgrade.version, upgrade.name)
+		s.proposeUpgrade(upgrade.name, upgrade.version)
+		s.voteForProposal(idx)
+		s.upgrade(upgrade.repo, upgrade.version)
+	}
 	s.T().Logf("SUCCESS")
 }

tests/e2e/e2e_utils_test.go

@@ -6,35 +6,77 @@ import (
 	"strings"
 )
 
+// versionSeparator is used to separate versions in the INITIAL_VERSION and TARGET_VERSION
+// environment vars
+const versionSeparator = "/"
+
+// upgradeConfig defines a struct that contains the version and the source repository for an upgrade
+type upgradeConfig struct {
+	name    string
+	version string
+	repo    string
+}
+
+// loadUpgradeParams loads the parameters for the upgrade test suite from the environment
+// variables
 func (s *IntegrationTestSuite) loadUpgradeParams() {
 	var err error
+	// name defines the upgrade name to use in the proposal
+	var name string
+	// targetRepo is assigned to the remote repository by default and is changed to local if no
+	// target version is given
+	targetRepo := tharsisRepo
+	// upgradesList contains the available upgrades in the app/upgrades folder
+	var upgradesList []string
+	// upgrades contains the slice of all upgrades that shall be executed
+	var upgrades []upgradeConfig //nolint:prealloc
+	// version defines the version to run the Evmos node with
+	var version string
+	// versions contains the slice of all versions that are run during the upgrade tests
+	var versions []string
 
 	initialV := os.Getenv("INITIAL_VERSION")
 	if initialV == "" {
-		upgradesList, err := s.upgradeManager.RetrieveUpgradesList()
+		upgradesList, err = s.upgradeManager.RetrieveUpgradesList()
 		s.Require().NoError(err)
-		// set the pre-last upgrade is upgrade list
-		s.upgradeParams.InitialVersion = upgradesList[len(upgradesList)-2]
+		// set the second-to-last upgrade as initial version
+		versions = []string{upgradesList[len(upgradesList)-2]}
 	} else {
-		s.upgradeParams.InitialVersion = initialV
+		versions = strings.Split(initialV, versionSeparator)
+	}
+
+	// for all initial versions define the docker hub repo as the source
+	for _, version := range versions {
+		upgrades = append(upgrades, upgradeConfig{
+			version,
+			version,
+			targetRepo,
+		})
 	}
 
 	// Target version loading, if not specified manager gets the last one from app/upgrades folder
 	// and sets target repository to local, otherwise tharsishq repo will be used
 	targetV := os.Getenv("TARGET_VERSION")
 	if targetV == "" {
-		upgradesList, err := s.upgradeManager.RetrieveUpgradesList()
-		s.Require().NoError(err)
-		// set the last upgrade is upgrade list
-		s.upgradeParams.SoftwareUpgradeVersion = upgradesList[len(upgradesList)-1]
-		s.upgradeParams.TargetVersion = localVersionTag
-		s.upgradeParams.TargetRepo = tharsisRepo
+		if upgradesList == nil {
+			upgradesList, err = s.upgradeManager.RetrieveUpgradesList()
+			s.Require().NoError(err)
+		}
+		name = upgradesList[len(upgradesList)-1]
+		version = localVersionTag
 	} else {
-		s.upgradeParams.TargetVersion = targetV
-		s.upgradeParams.SoftwareUpgradeVersion = targetV
-		s.upgradeParams.TargetRepo = tharsisRepo
+		name = targetV
+		version = targetV
 	}
 
+	// Add the target version to the upgrades slice
+	upgrades = append(upgrades, upgradeConfig{
+		name,
+		version,
+		targetRepo,
+	})
+	s.upgradeParams.Upgrades = upgrades
+
 	// If chain ID is not specified, 'evmos_9000-1' will be used in upgrade-init.sh
 	chID := os.Getenv("CHAIN_ID")
 	if chID == "" {