Skip to content

Implement new format of the health report #3492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 26, 2020
Merged

Implement new format of the health report #3492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d70c33d
Implement new format of the health report
anjalshireesh Nov 3, 2020
53fb224
Parallelize the mapping functions
anjalshireesh Nov 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 91 additions & 95 deletions cmd/admin-subnet-health.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package cmd
import (
"context"
gojson "encoding/json"
"errors"
"flag"
"fmt"
"os"
Expand All @@ -30,23 +31,21 @@ import (
"github.com/fatih/color"
"github.com/klauspost/compress/gzip"
"github.com/minio/cli"
json "github.com/minio/mc/pkg/colorjson"
"github.com/minio/mc/pkg/probe"
"github.com/minio/minio/pkg/console"
"github.com/minio/minio/pkg/madmin"
)

var adminOBDFlags = []cli.Flag{
OBDDataTypeFlag{
var adminHealthFlags = []cli.Flag{
HealthDataTypeFlag{
Name: "test",
Usage: "choose OBD tests to run [" + options.String() + "]",
Usage: "choose health tests to run [" + options.String() + "]",
Value: nil,
EnvVar: "MC_HEALTH_TEST,MC_OBD_TEST",
Hidden: true,
},
cli.DurationFlag{
Name: "deadline",
Usage: "maximum duration that OBD tests should be allowed to run",
Usage: "maximum duration that health tests should be allowed to run",
Value: 3600 * time.Second,
EnvVar: "MC_HEALTH_DEADLINE,MC_OBD_DEADLINE",
},
Expand All @@ -55,9 +54,9 @@ var adminOBDFlags = []cli.Flag{
var adminSubnetHealthCmd = cli.Command{
Name: "health",
Usage: "run health check for Subnet",
Action: mainAdminOBD,
Action: mainAdminHealth,
Before: setGlobalsFromContext,
Flags: append(adminOBDFlags, globalFlags...),
Flags: append(adminHealthFlags, globalFlags...),
CustomHelpTemplate: `NAME:
{{.HelpName}} - {{.Usage}}

Expand All @@ -73,33 +72,15 @@ EXAMPLES:
`,
}

type clusterOBDStruct struct {
Status string `json:"status"`
Error string `json:"error,omitempty"`
Info madmin.OBDInfo `json:"obdInfo,omitempty"`
}

func (u clusterOBDStruct) String() string {
return u.JSON()
}

// JSON jsonifies service status message.
func (u clusterOBDStruct) JSON() string {
statusJSONBytes, e := json.MarshalIndent(u, " ", " ")
fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

return string(statusJSONBytes)
}

// checkAdminInfoSyntax - validate arguments passed by a user
func checkAdminOBDSyntax(ctx *cli.Context) {
func checkAdminHealthSyntax(ctx *cli.Context) {
if len(ctx.Args()) == 0 || len(ctx.Args()) > 1 {
cli.ShowCommandHelpAndExit(ctx, "health", 1) // last argument is exit code
}
}

//compress and tar obd output
func tarGZ(c clusterOBDStruct, alias string) error {
//compress and tar health report output
func tarGZ(c HealthReportInfo, alias string) error {
filename := fmt.Sprintf("%s-health_%s.json.gz", filepath.Clean(alias), time.Now().Format("20060102150405"))
f, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
Expand All @@ -115,6 +96,19 @@ func tarGZ(c clusterOBDStruct, alias string) error {
defer gzWriter.Close()

enc := gojson.NewEncoder(gzWriter)

header := HealthReportHeader{
Subnet: Health{
Health: SchemaVersion{
Version: "v1",
},
},
}

if err := enc.Encode(header); err != nil {
return err
}

if err := enc.Encode(c); err != nil {
return err
}
Expand Down Expand Up @@ -146,8 +140,8 @@ func warnText(s string) string {
return console.Colorize("WARN", s)
}

func mainAdminOBD(ctx *cli.Context) error {
checkAdminOBDSyntax(ctx)
func mainAdminHealth(ctx *cli.Context) error {
checkAdminHealthSyntax(ctx)

// Get the alias parameter from cli
args := ctx.Args()
Expand All @@ -157,12 +151,29 @@ func mainAdminOBD(ctx *cli.Context) error {
client, err := newAdminClient(aliasedURL)
fatalIf(err, "Unable to initialize admin connection.")

opts := GetOBDDataTypeSlice(ctx, "test")
healthInfo, e := fetchServerHealthInfo(ctx, client)
clusterHealthInfo := MapHealthInfoToV1(healthInfo, e)

if globalJSON {
printMsg(clusterHealthInfo)
return nil
}

if clusterHealthInfo.GetError() != "" {
console.Println(warnText("unable to obtain health information:"), clusterHealthInfo.GetError())
return nil
}

return tarGZ(clusterHealthInfo, aliasedURL)
}

func fetchServerHealthInfo(ctx *cli.Context, client *madmin.AdminClient) (madmin.HealthInfo, error) {
opts := GetHealthDataTypeSlice(ctx, "test")
if len(*opts) == 0 {
opts = &options
}

optsMap := make(map[madmin.OBDDataType]struct{})
optsMap := make(map[madmin.HealthDataType]struct{})
for _, opt := range *opts {
optsMap[opt] = struct{}{}
}
Expand Down Expand Up @@ -214,8 +225,7 @@ func mainAdminOBD(ctx *cli.Context) error {
}
}
}

spinner := func(resource string, opt madmin.OBDDataType) func(bool) bool {
spinner := func(resource string, opt madmin.HealthDataType) func(bool) bool {
var spinStopper func()
done := false

Expand All @@ -241,19 +251,17 @@ func mainAdminOBD(ctx *cli.Context) error {
}
}

clusterOBDInfo := clusterOBDStruct{}

admin := spinner("Admin Info", madmin.OBDDataTypeMinioInfo)
cpu := spinner("CPU Info", madmin.OBDDataTypeSysCPU)
diskHw := spinner("Disk Info", madmin.OBDDataTypeSysDiskHw)
osInfo := spinner("OS Info", madmin.OBDDataTypeSysOsInfo)
mem := spinner("Mem Info", madmin.OBDDataTypeSysMem)
process := spinner("Process Info", madmin.OBDDataTypeSysLoad)
config := spinner("Server Config", madmin.OBDDataTypeMinioConfig)
drive := spinner("Drive Test", madmin.OBDDataTypePerfDrive)
net := spinner("Network Test", madmin.OBDDataTypePerfNet)

progress := func(info madmin.OBDInfo) {
admin := spinner("Admin Info", madmin.HealthDataTypeMinioInfo)
cpu := spinner("CPU Info", madmin.HealthDataTypeSysCPU)
diskHw := spinner("Disk Info", madmin.HealthDataTypeSysDiskHw)
osInfo := spinner("OS Info", madmin.HealthDataTypeSysOsInfo)
mem := spinner("Mem Info", madmin.HealthDataTypeSysMem)
process := spinner("Process Info", madmin.HealthDataTypeSysLoad)
config := spinner("Server Config", madmin.HealthDataTypeMinioConfig)
drive := spinner("Drive Test", madmin.HealthDataTypePerfDrive)
net := spinner("Network Test", madmin.HealthDataTypePerfNet)

progress := func(info madmin.HealthInfo) {
_ = admin(len(info.Minio.Info.Servers) > 0) &&
cpu(len(info.Sys.CPUInfo) > 0) &&
diskHw(len(info.Sys.DiskHwInfo) > 0) &&
Expand All @@ -265,45 +273,33 @@ func mainAdminOBD(ctx *cli.Context) error {
net(len(info.Perf.Net) > 1 && len(info.Perf.NetParallel.Addr) > 0)
}

var err error
var healthInfo madmin.HealthInfo

// Fetch info of all servers (cluster or single server)
obdChan := client.ServerOBDInfo(cont, *opts, ctx.Duration("deadline"))
for adminOBDInfo := range obdChan {
if adminOBDInfo.Error != "" {
clusterOBDInfo.Status = "Error"
clusterOBDInfo.Error = adminOBDInfo.Error
clusterOBDInfo.Info.Error = ""
clusterOBDInfo.Info.Minio.Info = madmin.InfoMessage{}
obdChan := client.ServerHealthInfo(cont, *opts, ctx.Duration("deadline"))
for adminHealthInfo := range obdChan {
if adminHealthInfo.Error != "" {
err = errors.New(adminHealthInfo.Error)
break
}

clusterOBDInfo.Status = "Success"
clusterOBDInfo.Info = adminOBDInfo
progress(adminOBDInfo)
healthInfo = adminHealthInfo
progress(adminHealthInfo)
}

// cancel the context if obdChan has returned.
cancel()

if globalJSON {
printMsg(clusterOBDInfo)
return nil
}

if clusterOBDInfo.Error != "" {
console.Println(warnText("unable to obtain health information:"), clusterOBDInfo.Error)
return nil
}

return tarGZ(clusterOBDInfo, aliasedURL)
return healthInfo, err
}

// OBDDataTypeSlice is a typed list of OBD tests
type OBDDataTypeSlice []madmin.OBDDataType
// HealthDataTypeSlice is a typed list of health tests
type HealthDataTypeSlice []madmin.HealthDataType

// Set - sets the flag to the given value
func (d *OBDDataTypeSlice) Set(value string) error {
func (d *HealthDataTypeSlice) Set(value string) error {
for _, v := range strings.Split(value, ",") {
if obdData, ok := madmin.OBDDataTypesMap[strings.Trim(v, " ")]; ok {
if obdData, ok := madmin.HealthDataTypesMap[strings.Trim(v, " ")]; ok {
*d = append(*d, obdData)
} else {
return fmt.Errorf("valid options include %s", options.String())
Expand All @@ -312,8 +308,8 @@ func (d *OBDDataTypeSlice) Set(value string) error {
return nil
}

// String - returns the string representation of the OBD datatypes
func (d *OBDDataTypeSlice) String() string {
// String - returns the string representation of the health datatypes
func (d *HealthDataTypeSlice) String() string {
val := ""
for _, obdData := range *d {
formatStr := "%s"
Expand All @@ -328,68 +324,68 @@ func (d *OBDDataTypeSlice) String() string {
}

// Value - returns the value
func (d *OBDDataTypeSlice) Value() []madmin.OBDDataType {
func (d *HealthDataTypeSlice) Value() []madmin.HealthDataType {
return *d
}

// Get - returns the value
func (d *OBDDataTypeSlice) Get() interface{} {
func (d *HealthDataTypeSlice) Get() interface{} {
return *d
}

// OBDDataTypeFlag is a typed flag to represent OBD datatypes
type OBDDataTypeFlag struct {
// HealthDataTypeFlag is a typed flag to represent health datatypes
type HealthDataTypeFlag struct {
Name string
Usage string
EnvVar string
Hidden bool
Value *OBDDataTypeSlice
Value *HealthDataTypeSlice
}

// String - returns the string to be shown in the help message
func (f OBDDataTypeFlag) String() string {
func (f HealthDataTypeFlag) String() string {
return fmt.Sprintf("--%s %s", f.Name, f.Usage)
}

// GetName - returns the name of the flag
func (f OBDDataTypeFlag) GetName() string {
func (f HealthDataTypeFlag) GetName() string {
return f.Name
}

// GetOBDDataTypeSlice - returns the list of set OBD tests
func GetOBDDataTypeSlice(c *cli.Context, name string) *OBDDataTypeSlice {
// GetHealthDataTypeSlice - returns the list of set health tests
func GetHealthDataTypeSlice(c *cli.Context, name string) *HealthDataTypeSlice {
generic := c.Generic(name)
if generic == nil {
return nil
}
return generic.(*OBDDataTypeSlice)
return generic.(*HealthDataTypeSlice)
}

// GetGlobalOBDDataTypeSlice - returns the list of set OBD tests set globally
func GetGlobalOBDDataTypeSlice(c *cli.Context, name string) *OBDDataTypeSlice {
// GetGlobalHealthDataTypeSlice - returns the list of set health tests set globally
func GetGlobalHealthDataTypeSlice(c *cli.Context, name string) *HealthDataTypeSlice {
generic := c.GlobalGeneric(name)
if generic == nil {
return nil
}
return generic.(*OBDDataTypeSlice)
return generic.(*HealthDataTypeSlice)
}

// Apply - applies the flag
func (f OBDDataTypeFlag) Apply(set *flag.FlagSet) {
func (f HealthDataTypeFlag) Apply(set *flag.FlagSet) {
f.ApplyWithError(set)
}

// ApplyWithError - applies with error
func (f OBDDataTypeFlag) ApplyWithError(set *flag.FlagSet) error {
func (f HealthDataTypeFlag) ApplyWithError(set *flag.FlagSet) error {
if f.EnvVar != "" {
for _, envVar := range strings.Split(f.EnvVar, ",") {
envVar = strings.TrimSpace(envVar)
if envVal, ok := syscall.Getenv(envVar); ok {
newVal := &OBDDataTypeSlice{}
newVal := &HealthDataTypeSlice{}
for _, s := range strings.Split(envVal, ",") {
s = strings.TrimSpace(s)
if err := newVal.Set(s); err != nil {
return fmt.Errorf("could not parse %s as OBD datatype value for flag %s: %s", envVal, f.Name, err)
return fmt.Errorf("could not parse %s as health datatype value for flag %s: %s", envVal, f.Name, err)
}
}
f.Value = newVal
Expand All @@ -401,11 +397,11 @@ func (f OBDDataTypeFlag) ApplyWithError(set *flag.FlagSet) error {
for _, name := range strings.Split(f.Name, ",") {
name = strings.Trim(name, " ")
if f.Value == nil {
f.Value = &OBDDataTypeSlice{}
f.Value = &HealthDataTypeSlice{}
}
set.Var(f.Value, name, f.Usage)
}
return nil
}

var options = OBDDataTypeSlice(madmin.OBDDataTypesList)
var options = HealthDataTypeSlice(madmin.HealthDataTypesList)
Loading