diff --git a/client/nomad.go b/client/nomad.go new file mode 100644 index 000000000..4b1f33759 --- /dev/null +++ b/client/nomad.go @@ -0,0 +1,21 @@ +package client + +import ( + nomad "github.com/hashicorp/nomad/api" +) + +// NewNomadClient is used to create a new client to interact with Nomad. +func NewNomadClient(addr string) (*nomad.Client, error) { + config := nomad.DefaultConfig() + + if addr != "" { + config.Address = addr + } + + c, err := nomad.NewClient(config) + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/command/deploy.go b/command/deploy.go index 3f819bc98..7c06a634a 100644 --- a/command/deploy.go +++ b/command/deploy.go @@ -129,7 +129,7 @@ func (c *DeployCommand) Run(args []string) int { } } - success := levant.TriggerDeployment(config) + success := levant.TriggerDeployment(config, nil) if !success { return 1 } diff --git a/command/scale_in.go b/command/scale_in.go new file mode 100644 index 000000000..493cfa214 --- /dev/null +++ b/command/scale_in.go @@ -0,0 +1,118 @@ +package command + +import ( + "strings" + + "github.com/jrasell/levant/levant/structs" + "github.com/jrasell/levant/logging" + "github.com/jrasell/levant/scale" +) + +// ScaleInCommand is the command implementation that allows users to scale a +// Nomad job out. +type ScaleInCommand struct { + Meta +} + +// Help provides the help information for the scale-in command. +func (c *ScaleInCommand) Help() string { + helpText := ` +Usage: levant scale-in [options] + + Scale a Nomad job and optional task group out. + +General Options: + + -address= + The Nomad HTTP API address including port which Levant will use to make + calls. + + -log-level= + Specify the verbosity level of Levant's logs. Valid values include DEBUG, + INFO, and WARN, in decreasing order of verbosity. The default is INFO. + + -log-format= + Specify the format of Levant's logs. Valid values are HUMAN or JSON. The + default is HUMAN. + +Scale In Options: + + -count= + The count by which the job and task groups should be scaled in by. Only + one of count or percent can be passed. + + -percent= + A percentage value by which the job and task groups should be scaled in + by. Counts will be rounded up, to ensure required capacity is met. Only + one of count or percent can be passed. + + -task-group= + The name of the task group you wish to target for scaling. Is this is not + speicified all task groups within the job will be scaled. +` + return strings.TrimSpace(helpText) +} + +// Synopsis is provides a brief summary of the scale-in command. +func (c *ScaleInCommand) Synopsis() string { + return "Scale in a Nomad job" +} + +// Run triggers a run of the Levant scale-in functions. +func (c *ScaleInCommand) Run(args []string) int { + + var err error + var logL, logF string + + config := &structs.ScalingConfig{} + config.Direction = structs.ScalingDirectionIn + + flags := c.Meta.FlagSet("scale-in", FlagSetVars) + flags.Usage = func() { c.UI.Output(c.Help()) } + + flags.StringVar(&config.Addr, "address", "", "") + flags.StringVar(&logL, "log-level", "INFO", "") + flags.StringVar(&logF, "log-format", "HUMAN", "") + flags.IntVar(&config.Count, "count", 0, "") + flags.IntVar(&config.Percent, "percent", 0, "") + flags.StringVar(&config.TaskGroup, "task-group", "", "") + + if err = flags.Parse(args); err != nil { + return 1 + } + + args = flags.Args() + + if len(args) != 1 { + c.UI.Error("This command takes one argument: ") + return 1 + } + + config.JobID = args[0] + + if config.Count == 0 && config.Percent == 0 || config.Count > 0 && config.Percent > 0 || + config.Count < 0 || config.Percent < 0 { + c.UI.Error("You must set either -count or -percent flag to scale-in") + return 1 + } + + if config.Count > 0 { + config.DirectionType = structs.ScalingDirectionTypeCount + } + + if config.Percent > 0 { + config.DirectionType = structs.ScalingDirectionTypePercent + } + + if err = logging.SetupLogger(logL, logF); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + success := scale.TriggerScalingEvent(config) + if !success { + return 1 + } + + return 0 +} diff --git a/command/scale_out.go b/command/scale_out.go new file mode 100644 index 000000000..7e96e8245 --- /dev/null +++ b/command/scale_out.go @@ -0,0 +1,117 @@ +package command + +import ( + "strings" + + "github.com/jrasell/levant/levant/structs" + "github.com/jrasell/levant/logging" + "github.com/jrasell/levant/scale" +) + +// ScaleOutCommand is the command implementation that allows users to scale a +// Nomad job out. +type ScaleOutCommand struct { + Meta +} + +// Help provides the help information for the scale-out command. +func (c *ScaleOutCommand) Help() string { + helpText := ` +Usage: levant scale-out [options] + + Scale a Nomad job and optional task group out. + +General Options: + + -address= + The Nomad HTTP API address including port which Levant will use to make + calls. + + -log-level= + Specify the verbosity level of Levant's logs. Valid values include DEBUG, + INFO, and WARN, in decreasing order of verbosity. The default is INFO. + + -log-format= + Specify the format of Levant's logs. Valid values are HUMAN or JSON. The + default is HUMAN. + +Scale Out Options: + + -count= + The count by which the job and task groups should be scaled out by. Only + one of count or percent can be passed. + + -percent= + A percentage value by which the job and task groups should be scaled out + by. Counts will be rounded up, to ensure required capacity is met. Only + one of count or percent can be passed. + + -task-group= + The name of the task group you wish to target for scaling. Is this is not + speicified all task groups within the job will be scaled. +` + return strings.TrimSpace(helpText) +} + +// Synopsis is provides a brief summary of the scale-out command. +func (c *ScaleOutCommand) Synopsis() string { + return "Scale out a Nomad job" +} + +// Run triggers a run of the Levant scale-out functions. +func (c *ScaleOutCommand) Run(args []string) int { + + var err error + var logL, logF string + + config := &structs.ScalingConfig{} + config.Direction = structs.ScalingDirectionOut + + flags := c.Meta.FlagSet("scale-out", FlagSetVars) + flags.Usage = func() { c.UI.Output(c.Help()) } + + flags.StringVar(&config.Addr, "address", "", "") + flags.StringVar(&logL, "log-level", "INFO", "") + flags.StringVar(&logF, "log-format", "HUMAN", "") + flags.IntVar(&config.Count, "count", 0, "") + flags.IntVar(&config.Percent, "percent", 0, "") + flags.StringVar(&config.TaskGroup, "task-group", "", "") + + if err = flags.Parse(args); err != nil { + return 1 + } + + args = flags.Args() + + if len(args) != 1 { + c.UI.Error("This command takes one argument: ") + return 1 + } + + config.JobID = args[0] + + if config.Count == 0 && config.Percent == 0 || config.Count > 0 && config.Percent > 0 { + c.UI.Error("You must set either -count or -percent flag to scale-out") + return 1 + } + + if config.Count > 0 { + config.DirectionType = structs.ScalingDirectionTypeCount + } + + if config.Percent > 0 { + config.DirectionType = structs.ScalingDirectionTypePercent + } + + if err = logging.SetupLogger(logL, logF); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + success := scale.TriggerScalingEvent(config) + if !success { + return 1 + } + + return 0 +} diff --git a/commands.go b/commands.go index aec5ea0fa..d3b9960d9 100644 --- a/commands.go +++ b/commands.go @@ -41,6 +41,16 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "scale-in": func() (cli.Command, error) { + return &command.ScaleInCommand{ + Meta: meta, + }, nil + }, + "scale-out": func() (cli.Command, error) { + return &command.ScaleOutCommand{ + Meta: meta, + }, nil + }, "version": func() (cli.Command, error) { ver := version.Version rel := version.VersionPrerelease diff --git a/levant/deploy.go b/levant/deploy.go index de4bf2672..d6b2ad995 100644 --- a/levant/deploy.go +++ b/levant/deploy.go @@ -7,14 +7,11 @@ import ( nomad "github.com/hashicorp/nomad/api" nomadStructs "github.com/hashicorp/nomad/nomad/structs" + "github.com/jrasell/levant/client" "github.com/jrasell/levant/levant/structs" "github.com/rs/zerolog/log" ) -const ( - jobIDContextField = "job_id" -) - // levantDeployment is the all deployment related objects for this Levant // deployment invoction. type levantDeployment struct { @@ -22,48 +19,36 @@ type levantDeployment struct { config *structs.Config } -// newNomadClient is used to create a new client to interact with Nomad. -func newNomadClient(addr string) (*nomad.Client, error) { - config := nomad.DefaultConfig() - - if addr != "" { - config.Address = addr - } - - c, err := nomad.NewClient(config) - if err != nil { - return nil, err - } - - return c, nil -} - // newLevantDeployment sets up the Levant deployment object and Nomad client // to interact with the Nomad API. -func newLevantDeployment(config *structs.Config) (*levantDeployment, error) { +func newLevantDeployment(config *structs.Config, nomadClient *nomad.Client) (*levantDeployment, error) { var err error dep := &levantDeployment{} dep.config = config - dep.nomad, err = newNomadClient(config.Addr) - if err != nil { - return nil, err + if nomadClient == nil { + dep.nomad, err = client.NewNomadClient(config.Addr) + if err != nil { + return nil, err + } + } else { + dep.nomad = nomadClient } // Add the JobID as a log context field. - log.Logger = log.With().Str(jobIDContextField, *config.Job.ID).Logger() + log.Logger = log.With().Str(structs.JobIDContextField, *config.Job.ID).Logger() return dep, nil } // TriggerDeployment provides the main entry point into a Levant deployment and // is used to setup the clients before triggering the deployment process. -func TriggerDeployment(config *structs.Config) bool { +func TriggerDeployment(config *structs.Config, nomadClient *nomad.Client) bool { // Create our new deployment object. - levantDep, err := newLevantDeployment(config) + levantDep, err := newLevantDeployment(config, nomadClient) if err != nil { log.Error().Err(err).Msg("levant/deploy: unable to setup Levant deployment") return false diff --git a/levant/dispatch.go b/levant/dispatch.go index 0bbbc79c9..a14e9a69f 100644 --- a/levant/dispatch.go +++ b/levant/dispatch.go @@ -2,6 +2,7 @@ package levant import ( nomad "github.com/hashicorp/nomad/api" + "github.com/jrasell/levant/client" "github.com/jrasell/levant/levant/structs" "github.com/rs/zerolog/log" ) @@ -10,7 +11,7 @@ import ( // is used to setup the clients before triggering the dispatch process. func TriggerDispatch(job string, metaMap map[string]string, payload []byte, address string) bool { - client, err := newNomadClient(address) + client, err := client.NewNomadClient(address) if err != nil { log.Error().Msgf("levant/dispatch: unable to setup Levant dispatch: %v", err) return false diff --git a/levant/structs/config.go b/levant/structs/config.go index fa161b41c..1eec53f77 100644 --- a/levant/structs/config.go +++ b/levant/structs/config.go @@ -2,6 +2,12 @@ package structs import nomad "github.com/hashicorp/nomad/api" +const ( + // JobIDContextField is the logging context feild added when interacting + // with jobs. + JobIDContextField = "job_id" +) + // Config is the main struct used to configure and run a Levant deployment on // a given target job. type Config struct { diff --git a/levant/structs/scaling.go b/levant/structs/scaling.go new file mode 100644 index 000000000..b7979d8a7 --- /dev/null +++ b/levant/structs/scaling.go @@ -0,0 +1,40 @@ +package structs + +// These consts represent configuration indentifers to use when performing +// either a scale-out or scale-in operation. +const ( + ScalingDirectionOut = "Out" + ScalingDirectionIn = "In" + ScalingDirectionTypeCount = "Count" + ScalingDirectionTypePercent = "Percent" +) + +// ScalingConfig is an internal config struct used to track configuration +// details when performing a scale-out or scale-in operation. +type ScalingConfig struct { + // Addr is the Nomad API address to use for all calls and must include both + // protocol and port. + Addr string + + // Count is the count by which the operator has asked to scale the Nomad job + // and optional taskgroup by. + Count int + + // Direction is the direction in which the scaling will take place and is + // populated by consts. + Direction string + + // DirectionType is an identifier on whether the operator has specified to + // scale using a count increase or percentage. + DirectionType string + + // JobID is the Nomad job which will be interacted with for scaling. + JobID string + + // Percent is the percentage by which the operator has asked to scale the + // Nomad job and optional taskgroup by. + Percent int + + // TaskGroup is the Nomad job taskgroup which has been selected for scaling. + TaskGroup string +} diff --git a/scale/scale.go b/scale/scale.go new file mode 100644 index 000000000..f0e8d12c7 --- /dev/null +++ b/scale/scale.go @@ -0,0 +1,130 @@ +package scale + +import ( + nomad "github.com/hashicorp/nomad/api" + nomadStructs "github.com/hashicorp/nomad/nomad/structs" + + "github.com/jrasell/levant/client" + "github.com/jrasell/levant/levant" + "github.com/jrasell/levant/levant/structs" + "github.com/rs/zerolog/log" +) + +// TriggerScalingEvent provides the exported entry point into performing a job +// scale based on user inputs. +func TriggerScalingEvent(config *structs.ScalingConfig) bool { + + // Add the JobID as a log context field. + log.Logger = log.With().Str(structs.JobIDContextField, config.JobID).Logger() + + nomadClient, err := client.NewNomadClient(config.Addr) + if err != nil { + log.Error().Msg("levant/scale: unable to setup Levant scaling event") + return false + } + + job := updateJob(nomadClient, config) + if job == nil { + log.Error().Msg("levant/scale: unable to perform job count update") + return false + } + + // Setup a deployment object, as a scaling event is a deployment and should + // go through the same process and code upgrades. + deploymentConfig := &structs.Config{} + deploymentConfig.Job = job + deploymentConfig.ForceCount = true + + log.Info().Msg("levant/scale: job will now be deployed with updated counts") + + // Trigger a deployment of the updated job which results in the scaling of + // the job and will go through all the deployment tracking until an end + // state is reached. + success := levant.TriggerDeployment(deploymentConfig, nomadClient) + if !success { + return false + } + + return true +} + +// updateJob gathers information on the current state of the running job and +// along with the user defined input updates the in-memory job specification +// to reflect the desired scaled state. +func updateJob(client *nomad.Client, config *structs.ScalingConfig) *nomad.Job { + + job, _, err := client.Jobs().Info(config.JobID, nil) + if err != nil { + log.Error().Err(err).Msg("levant/scale: unable to obtain job information from Nomad") + return nil + } + + // You can't scale a job that isn't running; or at least you shouldn't in + // my current opinion. + if *job.Status != nomadStructs.JobStatusRunning { + log.Error().Msgf("levant/scale: job is not in %s state", nomadStructs.JobStatusRunning) + return nil + } + + for _, group := range job.TaskGroups { + + // If the user has specified a taskgroup to scale, ensure we only change + // the specific of this. + if config.TaskGroup != "" { + if *group.Name == config.TaskGroup { + log.Debug().Msgf("levant/scale: scaling action to be requested on taskgroup %s only", + config.TaskGroup) + updateTaskGroup(config, group) + } + + // If no taskgroup has been specified, all found will have their + // count updated. + } else { + log.Debug().Msg("levant/scale: scaling action requested on all taskgroups") + updateTaskGroup(config, group) + } + } + + return job +} + +// updateTaskGroup is tasked with performing the count update based on the user +// configuration when a group is identified as being marked for scaling. +func updateTaskGroup(config *structs.ScalingConfig, group *nomad.TaskGroup) { + + var c int + + // If a percentage scale value has been passed, we must convert this to an + // int which represents the count to scale by as Nomad job submissions must + // be done with group counts as desired ints. + switch config.DirectionType { + case structs.ScalingDirectionTypeCount: + c = config.Count + case structs.ScalingDirectionTypePercent: + c = calculateCountBasedOnPercent(*group.Count, config.Percent) + } + + // Depending on whether we are scaling-out or scaling-in we need to perform + // the correct maths. There is a little duplication here, but that is to + // provide better logging. + switch config.Direction { + case structs.ScalingDirectionOut: + nc := *group.Count + c + log.Info().Msgf("levant/scale: task group %s will scale-out from %v to %v", + *group.Name, *group.Count, nc) + *group.Count = nc + + case structs.ScalingDirectionIn: + nc := *group.Count - c + log.Info().Msgf("levant/scale: task group %s will scale-in from %v to %v", + *group.Name, *group.Count, nc) + *group.Count = nc + } +} + +// calculateCountBasedOnPercent is a small helper function to turn a percentage +// based scale event into a relative count. +func calculateCountBasedOnPercent(count, percent int) int { + n := (float64(count) / 100) * float64(percent) + return int(n + 0.5) +} diff --git a/scale/scale_test.go b/scale/scale_test.go new file mode 100644 index 000000000..8d9ce9f37 --- /dev/null +++ b/scale/scale_test.go @@ -0,0 +1,112 @@ +package scale + +import ( + "testing" + + nomad "github.com/hashicorp/nomad/api" + "github.com/jrasell/levant/levant/structs" +) + +func TestScale_updateTaskGroup(t *testing.T) { + + sOut := structs.ScalingDirectionOut + sIn := structs.ScalingDirectionIn + sCount := structs.ScalingDirectionTypeCount + sPercent := structs.ScalingDirectionTypePercent + + cases := []struct { + Config *structs.ScalingConfig + Group *nomad.TaskGroup + EndCount int + }{ + { + buildScalingConfig(sOut, sCount, 100), + buildTaskGroup(1000), + 1100, + }, + { + buildScalingConfig(sOut, sPercent, 25), + buildTaskGroup(100), + 125, + }, + { + buildScalingConfig(sIn, sCount, 900), + buildTaskGroup(901), + 1, + }, + { + buildScalingConfig(sIn, sPercent, 90), + buildTaskGroup(100), + 10, + }, + } + + for _, tc := range cases { + updateTaskGroup(tc.Config, tc.Group) + + if tc.EndCount != *tc.Group.Count { + t.Fatalf("got: %#v, expected %#v", *tc.Group.Count, tc.EndCount) + } + } +} + +func TestScale_calculateCountBasedOnPercent(t *testing.T) { + + cases := []struct { + Count int + Percent int + Output int + }{ + { + 100, + 50, + 50, + }, + { + 3, + 33, + 1, + }, + { + 3, + 10, + 0, + }, + } + + for _, tc := range cases { + output := calculateCountBasedOnPercent(tc.Count, tc.Percent) + + if output != tc.Output { + t.Fatalf("got: %#v, expected %#v", output, tc.Output) + } + } +} + +func buildScalingConfig(direction, dType string, number int) *structs.ScalingConfig { + + c := &structs.ScalingConfig{} + c.Direction = direction + c.DirectionType = dType + + switch dType { + case structs.ScalingDirectionTypeCount: + c.Count = number + case structs.ScalingDirectionTypePercent: + c.Percent = number + } + + return c +} + +func buildTaskGroup(count int) *nomad.TaskGroup { + + n := "LevantTest" + c := count + + t := &nomad.TaskGroup{} + t.Name = &n + t.Count = &c + + return t +}