Adopt cgroup based CPU monitoring from protocol (#144)

livekit · Oct 18, 2022 · 3304f9a · 3304f9a
1 parent 9ead047
commit 3304f9a
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 31 deletions.
diff --git a/go.mod b/go.mod
@@ -18,7 +18,7 @@ require (
 	github.com/grafov/m3u8 v0.11.1
 	github.com/livekit/livekit-server v1.2.1
 	github.com/livekit/mageutil v0.0.0-20220927214055-ff37ecf1f093
-	github.com/livekit/protocol v1.1.3-0.20221014231313-85bf30570f0f
+	github.com/livekit/protocol v1.1.3-0.20221017221212-9b09be77efec
 	github.com/livekit/server-sdk-go v1.0.0
 	github.com/mackerelio/go-osstat v0.2.3
 	github.com/pion/rtp v1.7.13

diff --git a/go.sum b/go.sum
@@ -312,6 +312,8 @@ github.com/livekit/mageutil v0.0.0-20220927214055-ff37ecf1f093 h1:a5Pzvr9EC+qLVD
 github.com/livekit/mageutil v0.0.0-20220927214055-ff37ecf1f093/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ=
 github.com/livekit/protocol v1.1.3-0.20221014231313-85bf30570f0f h1:pGJ6SFZzL77Fj70RmnMdTn7bSmCsJiBb4S1gEsClHo4=
 github.com/livekit/protocol v1.1.3-0.20221014231313-85bf30570f0f/go.mod h1:jshI3nWbZkF1y1TUr2WIqzhN9HnyMqM9v/e/31L78z0=
+github.com/livekit/protocol v1.1.3-0.20221017221212-9b09be77efec h1:hlFRD++CoQkBsuxqMWkgG/uy+W1/ScQAGiFkr97LxQ0=
+github.com/livekit/protocol v1.1.3-0.20221017221212-9b09be77efec/go.mod h1:2xNj9tUehjGYkptvCF7r0vgx3+D7kawK+a8rTsaLelY=
 github.com/livekit/rtcscore-go v0.0.0-20220815072451-20ee10ae1995 h1:vOaY2qvfLihDyeZtnGGN1Law9wRrw8BMGCr1TygTvMw=
 github.com/livekit/rtcscore-go v0.0.0-20220815072451-20ee10ae1995/go.mod h1:116ych8UaEs9vfIE8n6iZCZ30iagUFTls0vRmC+Ix5U=
 github.com/livekit/server-sdk-go v1.0.0 h1:bcWTcvbAc4QgA1d8a9iyJVFeN8uIh5y23IeJIK+nvI0=

diff --git a/pkg/stats/monitor.go b/pkg/stats/monitor.go
@@ -6,14 +6,14 @@ import (
 	"time"
 
 	"github.com/frostbyte73/go-throttle"
-	"github.com/mackerelio/go-osstat/cpu"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/atomic"
 
 	"github.com/livekit/egress/pkg/config"
 	"github.com/livekit/egress/pkg/errors"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
 )
 
 type Monitor struct {
@@ -22,7 +22,8 @@ type Monitor struct {
 	promCPULoad  prometheus.Gauge
 	requestGauge *prometheus.GaugeVec
 
-	idleCPUs        atomic.Float64
+	cpuStats *utils.CPUStats
+
 	pendingCPUs     atomic.Float64
 	numCPUs         float64
 	warningThrottle func(func())
@@ -63,7 +64,15 @@ func (m *Monitor) Start(conf *config.Config, close chan struct{}, isAvailable fu
 
 	prometheus.MustRegister(promNodeAvailable, m.promCPULoad, m.requestGauge)
 
-	go m.monitorCPULoad(close)
+	cpuStats, err := utils.NewCPUStats(func(idle float64) {
+		m.promCPULoad.Set(1 - idle/m.numCPUs)
+	})
+	if err != nil {
+		return err
+	}
+
+	m.cpuStats = cpuStats
+
 	return nil
 }
 
@@ -122,38 +131,13 @@ func (m *Monitor) checkCPUConfig(costConfig config.CPUCostConfig) error {
 	return nil
 }
 
-func (m *Monitor) monitorCPULoad(close chan struct{}) {
-	prev, _ := cpu.Get()
-
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-close:
-			return
-		case <-ticker.C:
-			next, _ := cpu.Get()
-			idlePercent := float64(next.Idle-prev.Idle) / float64(next.Total-prev.Total)
-			m.idleCPUs.Store(m.numCPUs * idlePercent)
-			m.promCPULoad.Set(1 - idlePercent)
-
-			if idlePercent < 0.1 {
-				m.warningThrottle(func() { logger.Infow("high cpu load", "load", 100-idlePercent) })
-			}
-
-			prev = next
-		}
-	}
-}
-
 func (m *Monitor) GetCPULoad() float64 {
-	return (m.numCPUs - m.idleCPUs.Load()) / m.numCPUs * 100
+	return (m.numCPUs - m.cpuStats.GetCPUIdle()) / m.numCPUs * 100
 }
 
 func (m *Monitor) CanAcceptRequest(req *livekit.StartEgressRequest) bool {
 	accept := false
-	available := m.idleCPUs.Load() - m.pendingCPUs.Load()
+	available := m.cpuStats.GetCPUIdle() - m.pendingCPUs.Load()
 
 	switch req.Request.(type) {
 	case *livekit.StartEgressRequest_RoomComposite: