Skip to content

Commit

Permalink
fix: cluster set doesnt get into a weird state of no cluster being set (
Browse files Browse the repository at this point in the history 
#1055)

## Description:
Don't restart the engine in cluster set
  • Loading branch information
@h4ck3rk3y
h4ck3rk3y committed Aug 30, 2023
1 parent f83e513 commit c647035
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 34 deletions.
91 changes: 57 additions & 34 deletions cli/cli/commands/cluster/set/set.go
View file
Expand Up @@ -8,14 +8,18 @@ import (
"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
"github.com/kurtosis-tech/kurtosis/cli/cli/defaults"
"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/engine_manager"
"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/kurtosis_config_getter"
"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_cluster_setting"
"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config"
"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/resolved_config"
"github.com/kurtosis-tech/kurtosis/contexts-config-store/store"
"github.com/kurtosis-tech/stacktrace"
"github.com/sirupsen/logrus"
)

const (
clusterNameArgKey = "cluster-name"
clusterNameArgKey = "cluster-name"
emptyClusterFromNeverHavingClusterSet = ""
)

var SetCmd = &lowlevel.LowlevelKurtosisCommand{
Expand Down Expand Up @@ -47,7 +51,15 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
return stacktrace.Propagate(err, "'%s' is not a valid name for Kurtosis cluster", clusterName)
}

clusterUpdateSuccessful := false
contextsConfigStore := store.GetContextsConfigStore()
currentKurtosisContext, err := contextsConfigStore.GetCurrentContext()
if err != nil {
return stacktrace.Propagate(err, "tried fetching the current Kurtosis context but failed, we can't switch clusters without this information. This is a bug in Kurtosis")
}
if store.IsRemote(currentKurtosisContext) {
return stacktrace.NewError("Switching clusters on a remote context is not a permitted operation, please switch to the local context using `kurtosis %s %s default` before switching clusters", command_str_consts.ContextCmdStr, command_str_consts.ContextSwitchCmdStr)
}

clusterSettingStore := kurtosis_cluster_setting.GetKurtosisClusterSettingStore()
clusterPriorToUpdate, err := clusterSettingStore.GetClusterSetting()
if err != nil {
Expand All @@ -60,51 +72,62 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
return nil
}

engineManagerOldCluster, err := engine_manager.NewEngineManager(ctx)
if err != nil {
return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
// at the moment docker clusters are local only, we stop engine in them in favor of switching to any other cluster
stopOldEngine := false
if clusterPriorToUpdate == emptyClusterFromNeverHavingClusterSet {
stopOldEngine = true
} else {
clusterSettingPriorToUpdate, err := kurtosis_config_getter.GetKurtosisClusterConfig()
if err != nil {
return stacktrace.Propagate(err, "An error occurred while fetching cluster setting for current cluster '%s'; not proceeding further to ensure that Kurtosis doesn't get into a bad state", clusterPriorToUpdate)
} else if clusterSettingPriorToUpdate.GetClusterType() == resolved_config.KurtosisClusterType_Docker {
stopOldEngine = true
}
}
if err = engineManagerOldCluster.StopEngineIdempotently(ctx); err != nil {
logrus.Warnf("Kurtosis was not able to stop a potentially running engine on the cluster '%s'. This is not "+
"critical, it might be that the current cluster is not reachable anymore. In the worst case scenario, the "+
"engine will continue running in cluster '%s' until Kurtosis is switched back to it.", clusterPriorToUpdate,
clusterPriorToUpdate)

if stopOldEngine {
logrus.Infof("Current cluster seems to be a local cluster of type '%s'; will stop the engine if its running so that it doesn't interfere with the updated cluster", resolved_config.KurtosisClusterType_Docker.String())
engineManagerOldCluster, err := engine_manager.NewEngineManager(ctx)
if err != nil {
return stacktrace.Propagate(err, "an error occurred while creating an engine manager for current cluster")
}
if err := engineManagerOldCluster.StopEngineIdempotently(ctx); err != nil {
return stacktrace.Propagate(err, "Tried stopping engine for current cluster but failed; not proceeding further as current cluster is a local cluster and there might be port clashes with the updated cluster. Its status can be retrieved "+
"running 'kurtosis %s %s' and it can potentially be started running 'kurtosis %s %s'",
command_str_consts.EngineCmdStr, command_str_consts.EngineStatusCmdStr, command_str_consts.EngineCmdStr,
command_str_consts.EngineStartCmdStr)
}
}

if err = clusterSettingStore.SetClusterSetting(clusterName); err != nil {
return stacktrace.Propagate(err, "Failed to set cluster name to '%v'.", clusterName)
}
defer func() {
if clusterUpdateSuccessful {
return
}
if err = clusterSettingStore.SetClusterSetting(clusterPriorToUpdate); err != nil {
logrus.Errorf("An error happened updating cluster to '%s'. Kurtosis tried to roll back to the "+
"previous value '%s' but the roll back failed. You have to roll back manually running "+
"'kurtosis %s %s %s'", clusterName, clusterPriorToUpdate, command_str_consts.ClusterCmdStr,
command_str_consts.ClusterSetCmdStr, clusterPriorToUpdate)
}
}()
logrus.Infof("Cluster set to '%s', Kurtosis engine will now be started", clusterName)
logrus.Infof("Cluster set to '%s'", clusterName)
engineManagerNewCluster, err := engine_manager.NewEngineManager(ctx)
if err != nil {
return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
}
// Start the engine inside the new cluster
_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize)
engineStatus, _, _, err := engineManagerNewCluster.GetEngineStatus(ctx)
if err != nil {
return stacktrace.Propagate(err, "Engine could not be started after cluster was updated. The cluster "+
"will be rolled back, but it is possible the engine will remain stopped. Its status can be retrieved "+
"running 'kurtosis %s %s' and it can potentially be started running 'kurtosis %s %s'",
command_str_consts.EngineCmdStr, command_str_consts.EngineStatusCmdStr, command_str_consts.EngineCmdStr,
command_str_consts.EngineStartCmdStr)
return stacktrace.Propagate(err, "an error occurred while getting the status of the engine in the current cluster")
}
defer func() {
if err = engineClientCloseFunc(); err != nil {
logrus.Warnf("Error closing the engine client:\n'%v'", err)
// we only start in a stopped state, the idempotent visitor gets stuck with engine_manager.EngineStatus_ContainerRunningButServerNotResponding if the gateway isn't running
// TODO - fix the idempotent starter longer term
if engineStatus == engine_manager.EngineStatus_Stopped {
_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize)
if err != nil {
return stacktrace.Propagate(err, "Engine could not be started after cluster was updated. Its status can be retrieved "+
"running 'kurtosis %s %s' and it can potentially be started running 'kurtosis %s %s'",
command_str_consts.EngineCmdStr, command_str_consts.EngineStatusCmdStr, command_str_consts.EngineCmdStr,
command_str_consts.EngineStartCmdStr)
}
}()
clusterUpdateSuccessful = true
defer func() {
if err = engineClientCloseFunc(); err != nil {
logrus.Warnf("Error closing the engine client:\n'%v'", err)
}
}()
}

return nil
}

Expand Down
2 changes: 2 additions & 0 deletions contexts-config-store/go.sum
View file
Expand Up @@ -33,8 +33,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
Expand Down
6 changes: 6 additions & 0 deletions grpc-file-transfer/golang/go.sum
View file
Expand Up @@ -23,12 +23,18 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
Expand Down

0 comments on commit c647035

Please sign in to comment.