Send container stats over API on a per-interface basis

This mirrors how the Docker API handles things, allowing us to be
more compatible with Docker and more verbose on the Libpod API.
Stats are given as per network interface in the container, but
still aggregated for `podman stats` and `podman pod stats`
display (so the CLI does not change, only the Libpod and Compat
APIs).

Signed-off-by: Matt Heon <mheon@redhat.com>

cmd/podman/containers/stats.go

@@ -215,7 +215,15 @@ func (s *containerStats) MemPerc() string {
 }
 
 func (s *containerStats) NetIO() string {
-	return combineHumanValues(s.NetInput, s.NetOutput)
+	var netInput uint64
+	var netOutput uint64
+
+	for _, net := range s.Network {
+		netInput += net.RxBytes
+		netOutput += net.TxBytes
+	}
+
+	return combineHumanValues(netInput, netOutput)
 }
 
 func (s *containerStats) BlockIO() string {

docs/source/markdown/podman-stats.1.md.in

@@ -50,9 +50,8 @@ Valid placeholders for the Go template are listed below:
 | .MemUsage           | Memory usage                                     |
 | .MemUsageBytes      | Memory usage (IEC)                               |
 | .Name               | Container Name                                   |
-| .NetInput           | Network Input                                    |
 | .NetIO              | Network IO                                       |
-| .NetOutput          | Network Output                                   |
+| .Network            | Network I/O, separated by network interface      |
 | .PerCPU             | CPU time consumed by all tasks [1]               |
 | .PIDs               | Number of PIDs                                   |
 | .PIDS               | Number of PIDs (yes, we know this is a dup)      |

libpod/define/containerstate.go

@@ -141,11 +141,23 @@ type ContainerStats struct {
 	MemUsage      uint64
 	MemLimit      uint64
 	MemPerc       float64
-	NetInput      uint64
-	NetOutput     uint64
-	BlockInput    uint64
-	BlockOutput   uint64
-	PIDs          uint64
-	UpTime        time.Duration
-	Duration      uint64
+	// Map of interface name to network statistics for that interface.
+	Network     map[string]ContainerNetworkStats
+	BlockInput  uint64
+	BlockOutput uint64
+	PIDs        uint64
+	UpTime      time.Duration
+	Duration    uint64
+}
+
+// Statistics for an individual container network interface
+type ContainerNetworkStats struct {
+	RxBytes   uint64
+	RxDropped uint64
+	RxErrors  uint64
+	RxPackets uint64
+	TxBytes   uint64
+	TxDropped uint64
+	TxErrors  uint64
+	TxPackets uint64
 }

libpod/networking_freebsd.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/containers/buildah/pkg/jail"
 	"github.com/containers/common/libnetwork/types"
+	"github.com/containers/podman/v4/libpod/define"
 	"github.com/containers/storage/pkg/lockfile"
 	"github.com/sirupsen/logrus"
 )
@@ -45,33 +46,6 @@ type NetstatAddress struct {
 	Collisions uint64 `json:"collisions"`
 }
 
-// copied from github.com/vishvanada/netlink which does not build on freebsd
-type LinkStatistics64 struct {
-	RxPackets         uint64
-	TxPackets         uint64
-	RxBytes           uint64
-	TxBytes           uint64
-	RxErrors          uint64
-	TxErrors          uint64
-	RxDropped         uint64
-	TxDropped         uint64
-	Multicast         uint64
-	Collisions        uint64
-	RxLengthErrors    uint64
-	RxOverErrors      uint64
-	RxCrcErrors       uint64
-	RxFrameErrors     uint64
-	RxFifoErrors      uint64
-	RxMissedErrors    uint64
-	TxAbortedErrors   uint64
-	TxCarrierErrors   uint64
-	TxFifoErrors      uint64
-	TxHeartbeatErrors uint64
-	TxWindowErrors    uint64
-	RxCompressed      uint64
-	TxCompressed      uint64
-}
-
 type RootlessNetNS struct {
 	dir  string
 	Lock *lockfile.LockFile
@@ -223,7 +197,7 @@ func (r *Runtime) teardownNetNS(ctr *Container) error {
 
 // TODO (5.0): return the statistics per network interface
 // This would allow better compat with docker.
-func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) {
+func getContainerNetIO(ctr *Container) (map[string]define.ContainerNetworkStats, error) {
 	if ctr.state.NetNS == "" {
 		// If NetNS is nil, it was set as none, and no netNS
 		// was set up this is a valid state and thus return no
@@ -249,8 +223,9 @@ func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) {
 		return nil, err
 	}
 
+	res := make(map[string]define.ContainerNetworkStats)
+
 	// Sum all the interface stats - in practice only Tx/TxBytes are needed
-	res := &LinkStatistics64{}
 	for _, ifaddr := range stats.Statistics.Interface {
 		// Each interface has two records, one for link-layer which has
 		// an MTU field and one for IP which doesn't. We only want the
@@ -260,14 +235,16 @@ func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) {
 		// if we move to per-interface stats in future, this can be
 		// reported separately.
 		if ifaddr.Mtu > 0 {
-			res.RxPackets += ifaddr.ReceivedPackets
-			res.TxPackets += ifaddr.SentPackets
-			res.RxBytes += ifaddr.ReceivedBytes
-			res.TxBytes += ifaddr.SentBytes
-			res.RxErrors += ifaddr.ReceivedErrors
-			res.TxErrors += ifaddr.SentErrors
-			res.RxDropped += ifaddr.DroppedPackets
-			res.Collisions += ifaddr.Collisions
+			linkStats := define.ContainerNetworkStats{
+				RxPackets: ifaddr.ReceivedPackets,
+				TxPackets: ifaddr.SentPackets,
+				RxBytes:   ifaddr.ReceivedBytes,
+				TxBytes:   ifaddr.SentBytes,
+				RxErrors:  ifaddr.ReceivedErrors,
+				TxErrors:  ifaddr.SentErrors,
+				RxDropped: ifaddr.DroppedPackets,
+			}
+			res[ifaddr.Name] = linkStats
 		}
 	}
 

libpod/networking_linux.go

@@ -13,6 +13,7 @@ import (
 	"github.com/containers/common/libnetwork/types"
 	netUtil "github.com/containers/common/libnetwork/util"
 	"github.com/containers/common/pkg/netns"
+	"github.com/containers/podman/v4/libpod/define"
 	"github.com/containers/podman/v4/pkg/rootless"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
@@ -186,10 +187,9 @@ func getContainerNetNS(ctr *Container) (string, *Container, error) {
 	return "", nil, nil
 }
 
-// TODO (5.0): return the statistics per network interface
-// This would allow better compat with docker.
-func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) {
-	var netStats *netlink.LinkStatistics
+// Returns a map of interface name to statistics for that interface.
+func getContainerNetIO(ctr *Container) (map[string]define.ContainerNetworkStats, error) {
+	perNetworkStats := make(map[string]define.ContainerNetworkStats)
 
 	netNSPath, otherCtr, netPathErr := getContainerNetNS(ctr)
 	if netPathErr != nil {
@@ -222,21 +222,26 @@ func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) {
 				if err != nil {
 					return err
 				}
-				if netStats == nil {
-					netStats = link.Attrs().Statistics
-					continue
-				}
-				// Currently only Tx/RxBytes are used.
-				// In the future we should return all stats per interface so that
-				// api users have a better options.
 				stats := link.Attrs().Statistics
-				netStats.TxBytes += stats.TxBytes
-				netStats.RxBytes += stats.RxBytes
+				if stats != nil {
+					newStats := define.ContainerNetworkStats{
+						RxBytes:   stats.RxBytes,
+						RxDropped: stats.RxDropped,
+						RxErrors:  stats.RxErrors,
+						RxPackets: stats.RxPackets,
+						TxBytes:   stats.TxBytes,
+						TxDropped: stats.TxDropped,
+						TxErrors:  stats.TxErrors,
+						TxPackets: stats.TxPackets,
+					}
+
+					perNetworkStats[dev] = newStats
+				}
 			}
 		}
 		return nil
 	})
-	return netStats, err
+	return perNetworkStats, err
 }
 
 // joinedNetworkNSPath returns netns path and bool if netns was set

libpod/stats_common.go

@@ -41,6 +41,12 @@ func (c *Container) GetContainerStats(previousStats *define.ContainerStats) (*de
 		}
 	}
 
+	netStats, err := getContainerNetIO(c)
+	if err != nil {
+		return nil, err
+	}
+	stats.Network = netStats
+
 	if err := c.getPlatformContainerStats(stats, previousStats); err != nil {
 		return nil, err
 	}

libpod/stats_freebsd.go

@@ -80,20 +80,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev
 	stats.MemLimit = c.getMemLimit()
 	stats.SystemNano = now
 
-	netStats, err := getContainerNetIO(c)
-	if err != nil {
-		return err
-	}
-
-	// Handle case where the container is not in a network namespace
-	if netStats != nil {
-		stats.NetInput = netStats.RxBytes
-		stats.NetOutput = netStats.TxBytes
-	} else {
-		stats.NetInput = 0
-		stats.NetOutput = 0
-	}
-
 	return nil
 }
 

libpod/stats_linux.go

@@ -39,10 +39,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev
 		return fmt.Errorf("unable to obtain cgroup stats: %w", err)
 	}
 	conState := c.state.State
-	netStats, err := getContainerNetIO(c)
-	if err != nil {
-		return err
-	}
 
 	// If the current total usage in the cgroup is less than what was previously
 	// recorded then it means the container was restarted and runs in a new cgroup
@@ -69,14 +65,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev
 	stats.CPUSystemNano = cgroupStats.CpuStats.CpuUsage.UsageInKernelmode
 	stats.SystemNano = now
 	stats.PerCPU = cgroupStats.CpuStats.CpuUsage.PercpuUsage
-	// Handle case where the container is not in a network namespace
-	if netStats != nil {
-		stats.NetInput = netStats.RxBytes
-		stats.NetOutput = netStats.TxBytes
-	} else {
-		stats.NetInput = 0
-		stats.NetOutput = 0
-	}
 
 	return nil
 }

pkg/api/handlers/compat/containers_stats_linux.go

@@ -119,23 +119,20 @@ streamLabel: // A label to flatten the scope
 			return
 		}
 
-		// FIXME: network inspection does not yet work entirely
 		net := make(map[string]docker.NetworkStats)
-		networkName := inspect.NetworkSettings.EndpointID
-		if networkName == "" {
-			networkName = "network"
-		}
-		net[networkName] = docker.NetworkStats{
-			RxBytes:    stats.NetInput,
-			RxPackets:  0,
-			RxErrors:   0,
-			RxDropped:  0,
-			TxBytes:    stats.NetOutput,
-			TxPackets:  0,
-			TxErrors:   0,
-			TxDropped:  0,
-			EndpointID: inspect.NetworkSettings.EndpointID,
-			InstanceID: "",
+		for netName, netStats := range stats.Network {
+			net[netName] = docker.NetworkStats{
+				RxBytes:    netStats.RxBytes,
+				RxPackets:  netStats.RxPackets,
+				RxErrors:   netStats.RxErrors,
+				RxDropped:  netStats.RxDropped,
+				TxBytes:    netStats.TxBytes,
+				TxPackets:  netStats.TxPackets,
+				TxErrors:   netStats.TxErrors,
+				TxDropped:  netStats.TxDropped,
+				EndpointID: inspect.NetworkSettings.EndpointID,
+				InstanceID: "",
+			}
 		}
 
 		resources := ctnr.LinuxResources()

pkg/domain/infra/abi/pods_stats.go

@@ -44,12 +44,19 @@ func (ic *ContainerEngine) podsToStatsReport(pods []*libpod.Pod) ([]*entities.Po
 		}
 		podID := pods[i].ID()[:12]
 		for j := range podStats {
+			var podNetInput uint64
+			var podNetOutput uint64
+			for _, stats := range podStats[j].Network {
+				podNetInput += stats.RxBytes
+				podNetOutput += stats.TxBytes
+			}
+
 			r := entities.PodStatsReport{
 				CPU:           floatToPercentString(podStats[j].CPU),
 				MemUsage:      combineHumanValues(podStats[j].MemUsage, podStats[j].MemLimit),
 				MemUsageBytes: combineBytesValues(podStats[j].MemUsage, podStats[j].MemLimit),
 				Mem:           floatToPercentString(podStats[j].MemPerc),
-				NetIO:         combineHumanValues(podStats[j].NetInput, podStats[j].NetOutput),
+				NetIO:         combineHumanValues(podNetInput, podNetOutput),
 				BlockIO:       combineHumanValues(podStats[j].BlockInput, podStats[j].BlockOutput),
 				PIDS:          pidsToString(podStats[j].PIDs),
 				CID:           podStats[j].ContainerID[:12],

test/apiv2/19-stats.at

@@ -9,3 +9,21 @@ if root; then
     # regression for https://github.com/containers/podman/issues/15754
     t GET libpod/containers/container1/stats?stream=false 200 .cpu_stats.online_cpus=1
 fi
+
+podman run -dt --name testctr1 $IMAGE top &>/dev/null
+
+t GET libpod/containers/testctr1/stats?stream=false 200 '.network | length'=1
+
+podman rm -f testctr1
+
+podman network create testnet1
+podman network create testnet2
+
+podman run -dt --name testctr2 --net testnet1,testnet2 $IMAGE top &>/dev/null
+
+t GET libpod/containers/testctr2/stats?stream=false 200 '.network | length'=2
+
+podman rm -f testctr2
+
+podman network rm testnet1
+podman network rm testnet2