HMC import completed

docs/config.toml

@@ -20,7 +20,7 @@ pygmentsUseClasses = true
 	repo_url = "https://github.com/adejoux/nmon2influxdb"
 	repo_id = "nmon2influxdb"
 
-	version = "1.1.2"
+	version = "2.0.0"
 	logo = "images/logo.png"
 	favicon = ""
 

docs/content/configuration/file.md

@@ -46,6 +46,14 @@ import_log_retention="1d"
 
 # dashboard
 dashboard_write_file = false
+
+# HMC parameters
+hmc_server="mylab"
+hmc_user="hscroot"
+hmc_password="abc123"
+hmc_managed_system="mysystem"
+hmc_database="nmon2influxdbHMC"
+hmc_data_retention="40d"
 {{< /highlight >}}
 
 # Additional parameters

docs/content/getting-started/hmc.md

@@ -0,0 +1,23 @@
+---
+date: 2016-11-21T15:04:24+01:00
+title: HMC
+menu:
+  main:
+    parent: Getting started
+    identifier: /getting-started/hmc
+    weight: 40
+---
+
+# Enabling PCM data collection
+
+By default, PCM data are not collected. It's needed to enable it at the HMC level.
+
+  {{< gallery image="hmc_pcm.png" >}}
+
+You can enable PCM data collection by managed system on enable it for all of them.
+
+{{< gallery image="hmc_pcm_settings.png" >}}
+
+# firewall settings
+
+To fetch PCM data from the HMC, you need to have the port **12443** allowed.

docs/content/news/2.0.0_version_released.md

@@ -0,0 +1,75 @@
+---
+date: 2016-11-21T13:38:55+01:00
+title: 2.0.0 version released
+type: "news"
+---
+
+New version released adding import of HMC PCM(Performance and capacity Monitoring) data.
+Get a view of all your partitions and systems in a single panel.
+
+<!--more-->
+
+# HMC PCM data import
+
+It's possible to import PCM data from all systems where PCM data collection is set.
+
+{{< highlight batch >}}
+nmon2influxdb hmc import --hmc myhmc --hmcuser hscroot --hmcpass abc123
+{{</highlight >}}
+
+# Dashboards
+
+Currently no new dashboards are hardcoded in nmon2influxdb. It's because the import functionality is still new and it's not fixed what is the best way to display this metrics in Grafana. So the dashboard will evolve a lot in the following weeks. It's easier to share the dashboards outside of the tool for now.
+
+With this release you have 2 dashboards available for download:
+
+  * **hmc_partition.json**
+
+  This dashboard use grafana templating to display partition informations.
+
+  You can import the dashboard with **nmon2influxdb**:
+  {{< highlight batch >}}
+  nmon2influxdb dashboard hmc_partition.json
+  {{</highlight >}}
+
+  {{< gallery image="hmc_partition.png" >}}
+
+
+  * **hmc_system.json**
+
+  You can import the dashboard with **nmon2influxdb**:
+  {{< highlight batch >}}
+  nmon2influxdb dashboard hmc_partition.json
+  {{</highlight >}}
+
+{{< gallery image="hmc_system.png" >}}
+
+### New configuration parameters
+
+{{< highlight toml >}}
+hmc_server="mylab"
+hmc_user="hscroot"
+hmc_password="abc123"
+hmc_managed_system="mysystem"
+hmc_database="nmon2influxdbHMC"
+hmc_data_retention="40d"
+{{< /highlight >}}
+
+**hmc_user** and **hmc_password** provides credentials to login on the HMC specified by **hmc_server**.
+
+**Note:** Connection is done on port **12443**.
+
+By default, another influxdb database is used to store HMC metrics, **hmc_database** allows to change the default value(**nmon2influxdbHMC**).
+
+No data retention is set by default but it's possible to make data expires with the parameter **hmc_data_retention**.
+
+# Example
+
+{{< highlight batch >}}
+nmon2influxdb hmc import
+###############################
+File /log/nmon/lpar02_151104_1204.nmon imported : 316320 points !
+file not changed since last import: /log/nmon/lpar01_151104_1116.nmon
+file not changed since last import: /log/nmon/lpar01_151104_1200.nmon
+file not changed since last import: /log/nmon/lpar02_110415.nmon
+{{</highlight >}}

docs/content/usage/hmc_import.md

@@ -0,0 +1,78 @@
+---
+date: 2016-11-21T11:24:01+01:00
+title: HMC import
+menu:
+  main:
+    parent: Usage
+    identifier: /usage/hmc_import
+    weight: 15
+---
+
+{{< highlight batch >}}
+NAME:
+   nmon2influxdb hmc import - import hmc PCM data
+
+USAGE:
+   nmon2influxdb hmc import [command options] [arguments...]
+
+OPTIONS:
+   --hmc "myhmc"		hmc server
+   --hmcuser "hscroot"		hmc user
+   --hmcpass "abc123"		hmc password
+   --managed_system, -m 	only import this managed system
+{{< /highlight >}}
+
+# Parameters
+
+  * **hmc**: HMC to use to fetch PCM data
+  * **hmcuser**: HMC user to use for connection
+  * **hmcpass**: add per cpu metrics
+  * **managed_system**: fetch HMC PCM data only for this managed system
+
+# Environment variables
+
+Environment variables can be specified to setup default parameter values.
+
+  * **NMON2INFLUXDB_HMC_SERVER**
+  * **NMON2INFLUXDB_HMC_USER**
+
+**Note:** HMC password cannot be set by environment variables.
+
+# Configuration file parameters
+
+
+{{< highlight toml >}}
+hmc_server="mylab"
+hmc_user="hscroot"
+hmc_password="abc123"
+hmc_managed_system="mysystem"
+hmc_database="nmon2influxdbHMC"
+hmc_data_retention="40d"
+{{< /highlight >}}
+
+It's possible to set all CLI parameters. It's also possible to change the InfluxDB database name with **hmc_database** and change the data retention with **hmc_data_retention**.
+
+# Examples
+
+Loading HMC metrics from HMC **myhmc**:
+
+{{< highlight batch >}}
+nmon2influxdb hmc import
+Getting list of managed systems
+MANAGED SYSTEM: p750A
+partition powerVC: 2940 points
+MANAGED SYSTEM: p720-NIM_RETIRED
+Error getting PCM data
+{{< /highlight >}}
+
+Note: parameters can also be set in the configuration file **~/.nmon2influxdb.cfg**.
+
+Loading HMC metrics from HMC **myhmc** for system **mysystem** only:
+
+{{< highlight batch >}}
+nmon2influxdb hmc import -m p750A
+Getting list of managed systems
+MANAGED SYSTEM: p750A
+partition powerVC: 2940 points
+Skipping system: p720-NIM_RETIRED
+{{< /highlight >}}

hmc/hmc.go

@@ -51,6 +51,10 @@ type Point struct {
 	VlanID                  string
 	VswitchID               string
 	SharedEthernetAdapterID string
+	DrcIndex                string
+	PhysicalLocation        string
+	PhysicalDrcIndex        string
+	PhysicalPortID          string
 	Value                   interface{}
 	Timestamp               time.Time
 }
@@ -93,11 +97,11 @@ func (hmc *HMC) AddPoint(point Point) {
 
 	tags := map[string]string{"system": hmc.GlobalPoint.System, "name": point.Metric}
 	if len(hmc.GlobalPoint.Pool) > 0 {
-		tags["pool"] = point.Pool
+		tags["pool"] = hmc.GlobalPoint.Pool
 	}
 
 	if len(hmc.GlobalPoint.Type) > 0 {
-		tags["type"] = point.Type
+		tags["type"] = hmc.GlobalPoint.Type
 	}
 	if len(hmc.GlobalPoint.Device) > 0 {
 		tags["device"] = hmc.GlobalPoint.Device

hmc/import.go

@@ -24,6 +24,7 @@ func Import(c *cli.Context) {
 
 	hmc := NewHMC(c)
 
+	fmt.Printf("Getting list of managed systems\n")
 	systems, GetSysErr := hmc.GetManagedSystems()
 	nmon2influxdblib.CheckError(GetSysErr)
 
@@ -201,6 +202,33 @@ func Import(c *cli.Context) {
 							hmc.GlobalPoint.SharedEthernetAdapterID = ""
 							hmc.GlobalPoint.ViosID = ""
 						}
+
+						for _, net := range lpar.Network.SriovLogicalPorts {
+							hmc.GlobalPoint.DrcIndex = net.DrcIndex
+							hmc.GlobalPoint.PhysicalLocation = net.PhysicalLocation
+							hmc.GlobalPoint.PhysicalDrcIndex = net.PhysicalDrcIndex
+							hmc.GlobalPoint.PhysicalPortID = strconv.Itoa(net.PhysicalPortID)
+							hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts",
+								Metric: "receivedPackets",
+								Value:  net.ReceivedPackets[0]})
+							hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts",
+								Metric: "sentPackets",
+								Value:  net.SentPackets[0]})
+							hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts",
+								Metric: "droppedPackets",
+								Value:  net.DroppedPackets[0]})
+							hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts",
+								Metric: "sentBytes",
+								Value:  net.SentBytes[0]})
+							hmc.AddPoint(Point{Name: "PartitionSriovLogicalPorts",
+								Metric: "ReceivedBytes",
+								Value:  net.ReceivedBytes[0]})
+
+							hmc.GlobalPoint.DrcIndex = ""
+							hmc.GlobalPoint.PhysicalLocation = ""
+							hmc.GlobalPoint.PhysicalDrcIndex = ""
+							hmc.GlobalPoint.PhysicalPortID = ""
+						}
 					}
 				}
 				fmt.Printf(" %d points\n", hmc.InfluxDB.PointsCount())
@@ -248,23 +276,20 @@ func Import(c *cli.Context) {
 				hmc.GlobalPoint.Pool = spp.Name
 				hmc.AddPoint(Point{Name: "SystemSharedProcessorPool",
 					Metric: "assignedProcUnits",
-					Value:  spp.AssignedProcUnits[0],
-					Pool:   spp.Name})
+					Value:  spp.AssignedProcUnits[0]})
 				hmc.AddPoint(Point{Name: "SystemSharedProcessorPool",
 					Metric: "utilizedProcUnits",
-					Pool:   spp.Name,
 					Value:  spp.UtilizedProcUnits[0]})
 				hmc.AddPoint(Point{Name: "SystemSharedProcessorPool",
 					Metric: "availableProcUnits",
-					Value:  spp.AvailableProcUnits[0],
-					Pool:   spp.Name})
+					Value:  spp.AvailableProcUnits[0]})
 				hmc.GlobalPoint.Pool = ""
 			}
 			for _, vios := range sample.ViosUtil {
 				hmc.GlobalPoint.Partition = vios.Name
 				for _, scsi := range vios.Storage.GenericPhysicalAdapters {
 					hmc.GlobalPoint.Device = scsi.ID
-					hmc.AddPoint(Point{Name: "genericPhysicalAdapters",
+					hmc.AddPoint(Point{Name: "SystemgenericPhysicalAdapters",
 						Metric: "transmittedBytes",
 						Value:  scsi.TransmittedBytes[0]})
 					hmc.AddPoint(Point{Name: "SystemGenericPhysicalAdapters",

hmc/pcm.go

@@ -225,6 +225,22 @@ type PCMData struct {
 						ViosID                   int       `json:"viosId,omitempty"`
 						SharedEthernetAdapterID  string    `json:"sharedEthernetAdapterId,omitempty"`
 					} `json:"virtualEthernetAdapters"`
+					SriovLogicalPorts []struct {
+						DrcIndex         string    `json:"drcIndex"`
+						PhysicalLocation string    `json:"physicalLocation"`
+						PhysicalDrcIndex string    `json:"physicalDrcIndex"`
+						PhysicalPortID   int       `json:"physicalPortId"`
+						TransferredBytes []float64 `json:"transferredBytes"`
+						VlanID           int       `json:"vlanId"`
+						VswitchID        int       `json:"vswitchId"`
+						IsPortVlanID     bool      `json:"isPortVlanId"`
+						ReceivedPackets  []float64 `json:"receivedPackets"`
+						SentPackets      []float64 `json:"sentPackets"`
+						DroppedPackets   []float64 `json:"droppedPackets"`
+						SentBytes        []float64 `json:"sentBytes"`
+						ReceivedBytes    []float64 `json:"receivedBytes"`
+						ViosID           int       `json:"viosId,omitempty"`
+					} `json:"sriovLogicalPorts"`
 				} `json:"network"`
 				State string `json:"state"`
 				UUID  string `json:"uuid"`

main.go

@@ -43,6 +43,15 @@ func main() {
 	if len(config.ImportSkipMetrics) > 0 {
 		os.Setenv("NMON2INFLUXDB_SKIP_METRICS", config.ImportSkipMetrics)
 	}
+
+	if len(config.HMCServer) > 0 {
+		os.Setenv("NMON2INFLUXDB_HMC_SERVER", config.HMCServer)
+	}
+
+	if len(config.ImportSkipMetrics) > 0 {
+		os.Setenv("NMON2INFLUXDB_HMC_USER", config.HMCServer)
+	}
+
 	app := cli.NewApp()
 	app.Name = "nmon2influxdb"
 	app.Usage = "upload NMON stats to InfluxDB database"
@@ -195,26 +204,27 @@ func main() {
 			Subcommands: []cli.Command{
 				{
 					Name:   "import",
+					Usage:  "import HMC PCM data",
 					Action: hmc.Import,
 					Flags: []cli.Flag{
 						cli.StringFlag{
-							Name:  "hmc",
-							Usage: "hmc server",
-							Value: config.HMCServer,
+							Name:   "hmc",
+							Usage:  "HMC server",
+							EnvVar: "NMON2INFLUXDB_HMC_SERVER",
 						},
 						cli.StringFlag{
 							Name:  "hmcuser",
-							Usage: "hmc user",
+							Usage: "HMC user",
 							Value: config.HMCUser,
 						},
 						cli.StringFlag{
 							Name:  "hmcpass",
-							Usage: "hmc password",
+							Usage: "HMC password",
 							Value: config.HMCPassword,
 						},
 						cli.StringFlag{
 							Name:  "managed_system,m",
-							Usage: "only import this managed system",
+							Usage: "only import from this managed system",
 							Value: config.HMCManagedSystem,
 						},
 					},

nmon/dashboard.go

@@ -6,7 +6,6 @@ package nmon
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -83,6 +82,10 @@ func DashboardTemplate(config *nmon2influxdblib.Config, file string) {
 		fmt.Printf("Cannot convert template !\n")
 		nmon2influxdblib.CheckError(err)
 	}
+	// if config.DashboardWriteFile {
+	// 	nmon2influxdblib.PrintPrettyJSON()
+	// 	return
+	// }
 	err = nmon.UploadDashboard(dashboard)
 	nmon2influxdblib.CheckError(err)
 	return
@@ -110,7 +113,8 @@ func (nmon *Nmon) WriteDashboard() {
 	// make a write buffer
 	writer := bufio.NewWriter(file)
 	b, _ := json.Marshal(dashboard)
-	r := bytes.NewReader(b)
+
+	r := nmon2influxdblib.GetPrettyJSON(b)
 	r.WriteTo(writer)
 	writer.Flush()