[7.0.x] upgrade/resume backport (#1524)

* Hook 'system cluster-info' command * Use self-binary for collecting kubernetes logs * Backport of #1495.
gravitational · May 11, 2020 · 7aa83a7 · 7aa83a7
1 parent 620e004
commit 7aa83a7
Show file tree

Hide file tree

Showing 16 changed files with 298 additions and 191 deletions.
diff --git a/lib/expand/join.go b/lib/expand/join.go
@@ -854,7 +854,12 @@ func (p *Peer) tryConnect(operationID string) (ctx *operationContext, err error)
 			return nil, utils.Abort(err)
 		}
 		if trace.IsCompareFailed(err) {
-			p.printStep("Waiting for another operation to finish at %v", addr)
+			p.Warnf("Waiting for precondition to create expand operation: %v.", err)
+			if utils.IsClusterDegradedError(err) {
+				p.printStep("Cluster is degraded, waiting for it to become healthy")
+			} else {
+				p.printStep("Waiting for another operation to complete at %v", addr)
+			}
 		}
 	}
 	return ctx, trace.Wrap(err)

diff --git a/lib/install/client/resume.go b/lib/install/client/resume.go
@@ -61,8 +61,8 @@ func (r *ResumeStrategy) checkAndSetDefaults() (err error) {
 		r.ServicePath, err = environ.GetServicePath(stateDir)
 		if err != nil {
 			if trace.IsNotFound(err) {
-				return trace.Wrap(err,
-					"failed to find installer service. Start the installation with 'gravity install'")
+				return trace.Wrap(err, "failed to find installer service. "+
+					"Use 'gravity install' to start new installation or 'gravity join' to join an existing cluster.")
 			}
 			return trace.Wrap(err)
 		}

diff --git a/lib/ops/opsservice/operationgroup.go b/lib/ops/opsservice/operationgroup.go
@@ -168,8 +168,11 @@ func (g *operationGroup) canCreateExpandOperation(site ops.Site, operation ops.S
 	}
 
 	// cluster is not active, but there are no expand operations so there is either
-	// other type of operation is in progress, or it's degraded
+	// other type of operation in progress, or it's degraded
 	if len(operations) == 0 {
+		if site.State == ops.SiteStateDegraded {
+			return utils.ClusterDegradedError{}
+		}
 		return trace.CompareFailed("cannot expand %v cluster", site.State)
 	}
 

diff --git a/lib/report/k8s.go b/lib/report/k8s.go
@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/gravitational/gravity/lib/constants"
 	"github.com/gravitational/gravity/lib/defaults"
 	"github.com/gravitational/gravity/lib/utils"
 	"github.com/gravitational/gravity/lib/utils/kubectl"
@@ -38,7 +37,7 @@ func NewKubernetesCollector(ctx context.Context, runner utils.CommandRunner) Col
 		Cmd("k8s-nodes", utils.PlanetCommand(kubectl.Command("get", "nodes", "--output", "wide"))...),
 		Cmd("k8s-describe-nodes", utils.PlanetCommand(kubectl.Command("describe", "nodes"))...),
 		Cmd("k8s-cluster-info-dump.tgz",
-			constants.GravityBin, "system", "cluster-info"),
+			utils.Exe.Path, "system", "cluster-info"),
 	}
 	for _, resourceType := range defaults.KubernetesReportResourceTypes {
 		commands = append(commands,

diff --git a/lib/system/environ/validate.go b/lib/system/environ/validate.go
@@ -70,7 +70,7 @@ func ValidateInstall(env *localenv.LocalEnvironment) func() error {
 				"(see https://gravitational.com/gravity/docs/cluster/#deleting-a-cluster for more details)",
 				stateDir)
 		}
-		if err := validateNoPackageState(env.Packages, env.StateDir); err != nil {
+		if err := ValidateNoPackageState(env.Packages, env.StateDir); err != nil {
 			return trace.Wrap(err)
 		}
 		return nil
@@ -126,7 +126,9 @@ func validateNoActiveService(stateDir string) error {
 	}
 }
 
-func validateNoPackageState(packages pack.PackageService, stateDir string) error {
+// ValidateNoPackageState checks whether the specified package service
+// has state (i.e. has packages) and returns an error in this case.
+func ValidateNoPackageState(packages pack.PackageService, stateDir string) error {
 	// make sure that there are no packages in the local state left from
 	// some improperly cleaned up installation
 	installedPackages, err := packages.GetPackages(defaults.SystemAccountOrg)

diff --git a/lib/utils/error.go b/lib/utils/error.go
@@ -444,12 +444,36 @@ func ExitStatusFromError(err error) *int {
 	return nil
 }
 
+// IsCompareFailedError returns true to indicate this error
+// complies with compare failed error protocol
+func (ClusterDegradedError) IsCompareFailedError() bool {
+	return true
+}
+
+// Error returns the text representation of this error
+func (ClusterDegradedError) Error() string {
+	return "cluster is degraded"
+}
+
+// ClusterDegradedError indicates that the cluster is degraded
+type ClusterDegradedError struct{}
+
 // ExitCode interprets this value as exit code.
 // Implements ExitCodeError
 func (r exitCodeError) ExitCode() int {
 	return r.code
 }
 
+// IsClusterDegradedError determines if the error indicates that
+// the cluster is degraded
+func IsClusterDegradedError(err error) bool {
+	if _, ok := trace.Unwrap(err).(ClusterDegradedError); ok {
+		return true
+	}
+	// Handle the case when the error has come over the wire
+	return strings.Contains(err.Error(), "cluster is degraded")
+}
+
 // Error returns this exit code as error string.
 // Implements error
 func (r exitCodeError) Error() string {

diff --git a/tool/gravity/cli/clusterconfig.go b/tool/gravity/cli/clusterconfig.go
@@ -83,7 +83,7 @@ func newConfigUpdater(ctx context.Context, localEnv, updateEnv *localenv.LocalEn
 	return newUpdater(ctx, localEnv, updateEnv, init)
 }
 
-func executeConfigPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
+func executeConfigPhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -98,7 +98,7 @@ func executeConfigPhase(env *localenv.LocalEnvironment, environ LocalEnvironment
 	return trace.Wrap(err)
 }
 
-func setConfigPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params SetPhaseParams, operation ops.SiteOperation) error {
+func setConfigPhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params SetPhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -112,7 +112,7 @@ func setConfigPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFact
 	return updater.SetPhase(context.TODO(), params.PhaseID, params.State)
 }
 
-func rollbackConfigPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
+func rollbackConfigPhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -127,7 +127,7 @@ func rollbackConfigPhase(env *localenv.LocalEnvironment, environ LocalEnvironmen
 	return trace.Wrap(err)
 }
 
-func completeConfigPlan(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, operation ops.SiteOperation) error {
+func completeConfigPlanForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)

diff --git a/tool/gravity/cli/clusterupdate.go b/tool/gravity/cli/clusterupdate.go
@@ -129,7 +129,21 @@ func newClusterUpdater(
 	return updater, nil
 }
 
-func executeUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
+func executeUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams) error {
+	operation, err := getActiveOperation(env, environ, params.OperationID)
+	if err != nil {
+		if trace.IsNotFound(err) {
+			return trace.NotFound("no active update operation found")
+		}
+		return trace.Wrap(err)
+	}
+	if operation.Type != ops.OperationUpdate {
+		return trace.NotFound("no active update operation found")
+	}
+	return executeUpdatePhaseForOperation(env, environ, params, operation.SiteOperation)
+}
+
+func executeUpdatePhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -144,7 +158,7 @@ func executeUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironment
 	return trace.Wrap(err)
 }
 
-func rollbackUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
+func rollbackUpdatePhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -159,7 +173,7 @@ func rollbackUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironmen
 	return trace.Wrap(err)
 }
 
-func setUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params SetPhaseParams, operation ops.SiteOperation) error {
+func setUpdatePhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params SetPhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -173,7 +187,7 @@ func setUpdatePhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFact
 	return updater.SetPhase(context.TODO(), params.PhaseID, params.State)
 }
 
-func completeUpdatePlan(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, operation ops.SiteOperation) error {
+func completeUpdatePlanForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)

diff --git a/tool/gravity/cli/commands.go b/tool/gravity/cli/commands.go
@@ -264,6 +264,8 @@ type Application struct {
 	SystemReinstallCmd SystemReinstallCmd
 	// SystemHistoryCmd displays system update history
 	SystemHistoryCmd SystemHistoryCmd
+	// SystemClusterInfoCmd dumps cluster info suitable for debugging
+	SystemClusterInfoCmd SystemClusterInfoCmd
 	// SystemStepDownCmd asks active gravity master to step down
 	SystemStepDownCmd SystemStepDownCmd
 	// SystemRollbackCmd rolls back last system update
@@ -1514,6 +1516,12 @@ type SystemHistoryCmd struct {
 	*kingpin.CmdClause
 }
 
+// SystemClusterInfoCmd dumps kubernetes cluster info suitable for debugging.
+// It is a convenience wrapper around 'kubectl cluster-info dump --all-namespaces'
+type SystemClusterInfoCmd struct {
+	*kingpin.CmdClause
+}
+
 // SystemStepDownCmd asks active gravity master to step down
 type SystemStepDownCmd struct {
 	*kingpin.CmdClause

diff --git a/tool/gravity/cli/environ.go b/tool/gravity/cli/environ.go
@@ -72,7 +72,7 @@ func newEnvironUpdater(ctx context.Context, localEnv, updateEnv *localenv.LocalE
 	return newUpdater(ctx, localEnv, updateEnv, init)
 }
 
-func executeEnvironPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
+func executeEnvironPhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -87,7 +87,7 @@ func executeEnvironPhase(env *localenv.LocalEnvironment, environ LocalEnvironmen
 	return trace.Wrap(err)
 }
 
-func setEnvironPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params SetPhaseParams, operation ops.SiteOperation) error {
+func setEnvironPhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params SetPhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -101,7 +101,7 @@ func setEnvironPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFac
 	return updater.SetPhase(context.TODO(), params.PhaseID, params.State)
 }
 
-func rollbackEnvironPhase(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
+func rollbackEnvironPhaseForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, params PhaseParams, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)
@@ -116,7 +116,7 @@ func rollbackEnvironPhase(env *localenv.LocalEnvironment, environ LocalEnvironme
 	return trace.Wrap(err)
 }
 
-func completeEnvironPlan(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, operation ops.SiteOperation) error {
+func completeEnvironPlanForOperation(env *localenv.LocalEnvironment, environ LocalEnvironmentFactory, operation ops.SiteOperation) error {
 	updateEnv, err := environ.NewUpdateEnv()
 	if err != nil {
 		return trace.Wrap(err)

diff --git a/tool/gravity/cli/gc.go b/tool/gravity/cli/gc.go
@@ -254,15 +254,15 @@ func getGarbageCollector(env *localenv.LocalEnvironment, operation ops.SiteOpera
 	})
 }
 
-func executeGarbageCollectPhase(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
+func executeGarbageCollectPhaseForOperation(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
 	collector, err := getGarbageCollector(env, operation)
 	if err != nil {
 		return trace.Wrap(err)
 	}
 	return collector.RunPhase(context.TODO(), params.PhaseID, params.Timeout, params.Force)
 }
 
-func setGarbageCollectPhase(env *localenv.LocalEnvironment, params SetPhaseParams, operation ops.SiteOperation) error {
+func setGarbageCollectPhaseForOperation(env *localenv.LocalEnvironment, params SetPhaseParams, operation ops.SiteOperation) error {
 	collector, err := getGarbageCollector(env, operation)
 	if err != nil {
 		return trace.Wrap(err)

diff --git a/tool/gravity/cli/install.go b/tool/gravity/cli/install.go
@@ -534,32 +534,32 @@ func agent(env *localenv.LocalEnvironment, config agentConfig) error {
 	return trace.Wrap(agent.Serve())
 }
 
-func executeInstallPhase(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
+func executeInstallPhaseForOperation(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
 	return trace.Wrap(executePhaseFromService(
 		env, params, operation, "Connecting to installer", "Connected to installer"))
 }
 
-func rollbackInstallPhase(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
+func rollbackInstallPhaseForOperation(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
 	return trace.Wrap(rollbackPhaseFromService(
 		env, params, operation, "Connecting to installer", "Connected to installer"))
 }
 
-func completeInstallPlan(env *localenv.LocalEnvironment, operation ops.SiteOperation) error {
+func completeInstallPlanForOperation(env *localenv.LocalEnvironment, operation ops.SiteOperation) error {
 	return trace.Wrap(completePlanFromService(
 		env, operation, "Connecting to installer", "Connected to installer"))
 }
 
-func executeJoinPhase(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
+func executeJoinPhaseForOperation(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
 	return trace.Wrap(executePhaseFromService(
 		env, params, operation, "Connecting to agent", "Connected to agent"))
 }
 
-func rollbackJoinPhase(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
+func rollbackJoinPhaseForOperation(env *localenv.LocalEnvironment, params PhaseParams, operation ops.SiteOperation) error {
 	return trace.Wrap(rollbackPhaseFromService(
 		env, params, operation, "Connecting to agent", "Connected to agent"))
 }
 
-func completeJoinPlan(env *localenv.LocalEnvironment, operation ops.SiteOperation) error {
+func completeJoinPlanForOperation(env *localenv.LocalEnvironment, operation ops.SiteOperation) error {
 	err := completePlanFromService(
 		env, operation, "Connecting to agent", "Connected to agent")
 	if err == nil {