diff --git a/docs/config.toml b/docs/config.toml index 161c83b..c2d5195 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -20,7 +20,7 @@ pygmentsUseClasses = true repo_url = "https://github.com/adejoux/nmon2influxdb" repo_id = "nmon2influxdb" - version = "1.1.2" + version = "2.0.0" logo = "images/logo.png" favicon = "" diff --git a/docs/content/configuration/file.md b/docs/content/configuration/file.md index b198823..b64caf7 100644 --- a/docs/content/configuration/file.md +++ b/docs/content/configuration/file.md @@ -46,6 +46,14 @@ import_log_retention="1d" # dashboard dashboard_write_file = false + +# HMC parameters +hmc_server="mylab" +hmc_user="hscroot" +hmc_password="abc123" +hmc_managed_system="mysystem" +hmc_database="nmon2influxdbHMC" +hmc_data_retention="40d" {{< /highlight >}} # Additional parameters diff --git a/docs/content/getting-started/hmc.md b/docs/content/getting-started/hmc.md new file mode 100644 index 0000000..54c5385 --- /dev/null +++ b/docs/content/getting-started/hmc.md @@ -0,0 +1,23 @@ +--- +date: 2016-11-21T15:04:24+01:00 +title: HMC +menu: + main: + parent: Getting started + identifier: /getting-started/hmc + weight: 40 +--- + +# Enabling PCM data collection + +By default, PCM data are not collected. It's needed to enable it at the HMC level. + + {{< gallery image="hmc_pcm.png" >}} + +You can enable PCM data collection by managed system on enable it for all of them. + +{{< gallery image="hmc_pcm_settings.png" >}} + +# firewall settings + +To fetch PCM data from the HMC, you need to have the port **12443** allowed. diff --git a/docs/content/news/2.0.0_version_released.md b/docs/content/news/2.0.0_version_released.md new file mode 100644 index 0000000..b62c9e4 --- /dev/null +++ b/docs/content/news/2.0.0_version_released.md @@ -0,0 +1,75 @@ +--- +date: 2016-11-21T13:38:55+01:00 +title: 2.0.0 version released +type: "news" +--- + +New version released adding import of HMC PCM(Performance and capacity Monitoring) data. +Get a view of all your partitions and systems in a single panel. + + + +# HMC PCM data import + +It's possible to import PCM data from all systems where PCM data collection is set. + +{{< highlight batch >}} +nmon2influxdb hmc import --hmc myhmc --hmcuser hscroot --hmcpass abc123 +{{}} + +# Dashboards + +Currently no new dashboards are hardcoded in nmon2influxdb. It's because the import functionality is still new and it's not fixed what is the best way to display this metrics in Grafana. So the dashboard will evolve a lot in the following weeks. It's easier to share the dashboards outside of the tool for now. + +With this release you have 2 dashboards available for download: + + * **hmc_partition.json** + + This dashboard use grafana templating to display partition informations. + + You can import the dashboard with **nmon2influxdb**: + {{< highlight batch >}} + nmon2influxdb dashboard hmc_partition.json + {{}} + + {{< gallery image="hmc_partition.png" >}} + + + * **hmc_system.json** + + You can import the dashboard with **nmon2influxdb**: + {{< highlight batch >}} + nmon2influxdb dashboard hmc_partition.json + {{}} + +{{< gallery image="hmc_system.png" >}} + +### New configuration parameters + +{{< highlight toml >}} +hmc_server="mylab" +hmc_user="hscroot" +hmc_password="abc123" +hmc_managed_system="mysystem" +hmc_database="nmon2influxdbHMC" +hmc_data_retention="40d" +{{< /highlight >}} + +**hmc_user** and **hmc_password** provides credentials to login on the HMC specified by **hmc_server**. + +**Note:** Connection is done on port **12443**. + +By default, another influxdb database is used to store HMC metrics, **hmc_database** allows to change the default value(**nmon2influxdbHMC**). + +No data retention is set by default but it's possible to make data expires with the parameter **hmc_data_retention**. + +# Example + +{{< highlight batch >}} +nmon2influxdb hmc import +############################### +File /log/nmon/lpar02_151104_1204.nmon imported : 316320 points ! +file not changed since last import: /log/nmon/lpar01_151104_1116.nmon +file not changed since last import: /log/nmon/lpar01_151104_1200.nmon +file not changed since last import: /log/nmon/lpar02_110415.nmon +{{}} diff --git a/docs/content/usage/hmc_import.md b/docs/content/usage/hmc_import.md new file mode 100644 index 0000000..9ab4d97 --- /dev/null +++ b/docs/content/usage/hmc_import.md @@ -0,0 +1,78 @@ +--- +date: 2016-11-21T11:24:01+01:00 +title: HMC import +menu: + main: + parent: Usage + identifier: /usage/hmc_import + weight: 15 +--- + +{{< highlight batch >}} +NAME: + nmon2influxdb hmc import - import hmc PCM data + +USAGE: + nmon2influxdb hmc import [command options] [arguments...] + +OPTIONS: + --hmc "myhmc" hmc server + --hmcuser "hscroot" hmc user + --hmcpass "abc123" hmc password + --managed_system, -m only import this managed system +{{< /highlight >}} + +# Parameters + + * **hmc**: HMC to use to fetch PCM data + * **hmcuser**: HMC user to use for connection + * **hmcpass**: add per cpu metrics + * **managed_system**: fetch HMC PCM data only for this managed system + +# Environment variables + +Environment variables can be specified to setup default parameter values. + + * **NMON2INFLUXDB_HMC_SERVER** + * **NMON2INFLUXDB_HMC_USER** + +**Note:** HMC password cannot be set by environment variables. + +# Configuration file parameters + + +{{< highlight toml >}} +hmc_server="mylab" +hmc_user="hscroot" +hmc_password="abc123" +hmc_managed_system="mysystem" +hmc_database="nmon2influxdbHMC" +hmc_data_retention="40d" +{{< /highlight >}} + +It's possible to set all CLI parameters. It's also possible to change the InfluxDB database name with **hmc_database** and change the data retention with **hmc_data_retention**. + +# Examples + +Loading HMC metrics from HMC **myhmc**: + +{{< highlight batch >}} +nmon2influxdb hmc import +Getting list of managed systems +MANAGED SYSTEM: p750A +partition powerVC: 2940 points +MANAGED SYSTEM: p720-NIM_RETIRED +Error getting PCM data +{{< /highlight >}} + +Note: parameters can also be set in the configuration file **~/.nmon2influxdb.cfg**. + +Loading HMC metrics from HMC **myhmc** for system **mysystem** only: + +{{< highlight batch >}} +nmon2influxdb hmc import -m p750A +Getting list of managed systems +MANAGED SYSTEM: p750A +partition powerVC: 2940 points +Skipping system: p720-NIM_RETIRED +{{< /highlight >}} diff --git a/docs/static/images/hmc_partition.png b/docs/static/images/hmc_partition.png new file mode 100644 index 0000000..4f055c1 Binary files /dev/null and b/docs/static/images/hmc_partition.png differ diff --git a/docs/static/images/hmc_pcm.png b/docs/static/images/hmc_pcm.png new file mode 100644 index 0000000..ea060f1 Binary files /dev/null and b/docs/static/images/hmc_pcm.png differ diff --git a/docs/static/images/hmc_pcm_settings.png b/docs/static/images/hmc_pcm_settings.png new file mode 100644 index 0000000..6bbac4c Binary files /dev/null and b/docs/static/images/hmc_pcm_settings.png differ diff --git a/docs/static/images/hmc_system.png b/docs/static/images/hmc_system.png new file mode 100644 index 0000000..924fa56 Binary files /dev/null and b/docs/static/images/hmc_system.png differ diff --git a/hmc/hmc.go b/hmc/hmc.go index e923f3e..8b9d419 100644 --- a/hmc/hmc.go +++ b/hmc/hmc.go @@ -51,6 +51,10 @@ type Point struct { VlanID string VswitchID string SharedEthernetAdapterID string + DrcIndex string + PhysicalLocation string + PhysicalDrcIndex string + PhysicalPortID string Value interface{} Timestamp time.Time } @@ -93,11 +97,11 @@ func (hmc *HMC) AddPoint(point Point) { tags := map[string]string{"system": hmc.GlobalPoint.System, "name": point.Metric} if len(hmc.GlobalPoint.Pool) > 0 { - tags["pool"] = point.Pool + tags["pool"] = hmc.GlobalPoint.Pool } if len(hmc.GlobalPoint.Type) > 0 { - tags["type"] = point.Type + tags["type"] = hmc.GlobalPoint.Type } if len(hmc.GlobalPoint.Device) > 0 { tags["device"] = hmc.GlobalPoint.Device diff --git a/hmc/import.go b/hmc/import.go index c7e42fb..78073f8 100644 --- a/hmc/import.go +++ b/hmc/import.go @@ -24,6 +24,7 @@ func Import(c *cli.Context) { hmc := NewHMC(c) + fmt.Printf("Getting list of managed systems\n") systems, GetSysErr := hmc.GetManagedSystems() nmon2influxdblib.CheckError(GetSysErr) @@ -201,6 +202,33 @@ func Import(c *cli.Context) { hmc.GlobalPoint.SharedEthernetAdapterID = "" hmc.GlobalPoint.ViosID = "" } + + for _, net := range lpar.Network.SriovLogicalPorts { + hmc.GlobalPoint.DrcIndex = net.DrcIndex + hmc.GlobalPoint.PhysicalLocation = net.PhysicalLocation + hmc.GlobalPoint.PhysicalDrcIndex = net.PhysicalDrcIndex + hmc.GlobalPoint.PhysicalPortID = strconv.Itoa(net.PhysicalPortID) + hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts", + Metric: "receivedPackets", + Value: net.ReceivedPackets[0]}) + hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts", + Metric: "sentPackets", + Value: net.SentPackets[0]}) + hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts", + Metric: "droppedPackets", + Value: net.DroppedPackets[0]}) + hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts", + Metric: "sentBytes", + Value: net.SentBytes[0]}) + hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts", + Metric: "ReceivedBytes", + Value: net.ReceivedBytes[0]}) + + hmc.GlobalPoint.DrcIndex = "" + hmc.GlobalPoint.PhysicalLocation = "" + hmc.GlobalPoint.PhysicalDrcIndex = "" + hmc.GlobalPoint.PhysicalPortID = "" + } } } fmt.Printf(" %d points\n", hmc.InfluxDB.PointsCount()) @@ -248,23 +276,20 @@ func Import(c *cli.Context) { hmc.GlobalPoint.Pool = spp.Name hmc.AddPoint(Point{Name: "SystemSharedProcessorPool", Metric: "assignedProcUnits", - Value: spp.AssignedProcUnits[0], - Pool: spp.Name}) + Value: spp.AssignedProcUnits[0]}) hmc.AddPoint(Point{Name: "SystemSharedProcessorPool", Metric: "utilizedProcUnits", - Pool: spp.Name, Value: spp.UtilizedProcUnits[0]}) hmc.AddPoint(Point{Name: "SystemSharedProcessorPool", Metric: "availableProcUnits", - Value: spp.AvailableProcUnits[0], - Pool: spp.Name}) + Value: spp.AvailableProcUnits[0]}) hmc.GlobalPoint.Pool = "" } for _, vios := range sample.ViosUtil { hmc.GlobalPoint.Partition = vios.Name for _, scsi := range vios.Storage.GenericPhysicalAdapters { hmc.GlobalPoint.Device = scsi.ID - hmc.AddPoint(Point{Name: "genericPhysicalAdapters", + hmc.AddPoint(Point{Name: "SystemgenericPhysicalAdapters", Metric: "transmittedBytes", Value: scsi.TransmittedBytes[0]}) hmc.AddPoint(Point{Name: "SystemGenericPhysicalAdapters", diff --git a/hmc/pcm.go b/hmc/pcm.go index 0f66fb3..26b952e 100644 --- a/hmc/pcm.go +++ b/hmc/pcm.go @@ -225,6 +225,22 @@ type PCMData struct { ViosID int `json:"viosId,omitempty"` SharedEthernetAdapterID string `json:"sharedEthernetAdapterId,omitempty"` } `json:"virtualEthernetAdapters"` + SriovLogicalPorts []struct { + DrcIndex string `json:"drcIndex"` + PhysicalLocation string `json:"physicalLocation"` + PhysicalDrcIndex string `json:"physicalDrcIndex"` + PhysicalPortID int `json:"physicalPortId"` + TransferredBytes []float64 `json:"transferredBytes"` + VlanID int `json:"vlanId"` + VswitchID int `json:"vswitchId"` + IsPortVlanID bool `json:"isPortVlanId"` + ReceivedPackets []float64 `json:"receivedPackets"` + SentPackets []float64 `json:"sentPackets"` + DroppedPackets []float64 `json:"droppedPackets"` + SentBytes []float64 `json:"sentBytes"` + ReceivedBytes []float64 `json:"receivedBytes"` + ViosID int `json:"viosId,omitempty"` + } `json:"sriovLogicalPorts"` } `json:"network"` State string `json:"state"` UUID string `json:"uuid"` diff --git a/main.go b/main.go index 5ae2cdb..a692e4e 100644 --- a/main.go +++ b/main.go @@ -43,6 +43,15 @@ func main() { if len(config.ImportSkipMetrics) > 0 { os.Setenv("NMON2INFLUXDB_SKIP_METRICS", config.ImportSkipMetrics) } + + if len(config.HMCServer) > 0 { + os.Setenv("NMON2INFLUXDB_HMC_SERVER", config.HMCServer) + } + + if len(config.ImportSkipMetrics) > 0 { + os.Setenv("NMON2INFLUXDB_HMC_USER", config.HMCServer) + } + app := cli.NewApp() app.Name = "nmon2influxdb" app.Usage = "upload NMON stats to InfluxDB database" @@ -195,26 +204,27 @@ func main() { Subcommands: []cli.Command{ { Name: "import", + Usage: "import HMC PCM data", Action: hmc.Import, Flags: []cli.Flag{ cli.StringFlag{ - Name: "hmc", - Usage: "hmc server", - Value: config.HMCServer, + Name: "hmc", + Usage: "HMC server", + EnvVar: "NMON2INFLUXDB_HMC_SERVER", }, cli.StringFlag{ Name: "hmcuser", - Usage: "hmc user", + Usage: "HMC user", Value: config.HMCUser, }, cli.StringFlag{ Name: "hmcpass", - Usage: "hmc password", + Usage: "HMC password", Value: config.HMCPassword, }, cli.StringFlag{ Name: "managed_system,m", - Usage: "only import this managed system", + Usage: "only import from this managed system", Value: config.HMCManagedSystem, }, }, diff --git a/nmon/dashboard.go b/nmon/dashboard.go index 988fbaa..26258e1 100644 --- a/nmon/dashboard.go +++ b/nmon/dashboard.go @@ -6,7 +6,6 @@ package nmon import ( "bufio" - "bytes" "encoding/json" "fmt" "os" @@ -83,6 +82,10 @@ func DashboardTemplate(config *nmon2influxdblib.Config, file string) { fmt.Printf("Cannot convert template !\n") nmon2influxdblib.CheckError(err) } + // if config.DashboardWriteFile { + // nmon2influxdblib.PrintPrettyJSON() + // return + // } err = nmon.UploadDashboard(dashboard) nmon2influxdblib.CheckError(err) return @@ -110,7 +113,8 @@ func (nmon *Nmon) WriteDashboard() { // make a write buffer writer := bufio.NewWriter(file) b, _ := json.Marshal(dashboard) - r := bytes.NewReader(b) + + r := nmon2influxdblib.GetPrettyJSON(b) r.WriteTo(writer) writer.Flush() diff --git a/nmon/list.go b/nmon/list.go index 85e7ef6..42ded71 100644 --- a/nmon/list.go +++ b/nmon/list.go @@ -1,4 +1,4 @@ -// nmon2influxdb +// Package nmon provides wrapper on nmon reltaed commands // import nmon report in InfluxDB // author: adejoux@djouxtech.net package nmon diff --git a/nmon/stats.go b/nmon/stats.go index 829caa4..5ea5528 100644 --- a/nmon/stats.go +++ b/nmon/stats.go @@ -1,4 +1,4 @@ -// nmon2influxdb +// Package nmon provides wrapper on nmon reltaed commands // import nmon report in InfluxDB // author: adejoux@djouxtech.net package nmon diff --git a/nmon2influxdblib/helpers.go b/nmon2influxdblib/helpers.go index 27c6eed..a7fed91 100644 --- a/nmon2influxdblib/helpers.go +++ b/nmon2influxdblib/helpers.go @@ -56,12 +56,17 @@ func PrintHTTPRequest(request *http.Request) { //PrintPrettyJSON helper used to display JSON output in a nicer way func PrintPrettyJSON(contents []byte) { + text := GetPrettyJSON(contents) + log.Println("output:", string(text.Bytes())) +} + +//GetPrettyJSON returns pretty json string +func GetPrettyJSON(contents []byte) bytes.Buffer { var prettyJSON bytes.Buffer error := json.Indent(&prettyJSON, contents, "", "\t") if error != nil { log.Println("JSON parse error: ", error) - } - log.Println("output:", string(prettyJSON.Bytes())) + return prettyJSON }