Skip to content

Commit

Permalink
Update Traffic Stats to use TO APIv5 (#7765)
Browse files Browse the repository at this point in the history 
* Update Traffic Stats to use TO APIv5

* update changelog
  • Loading branch information
@ocket8888
ocket8888 committed Aug 30, 2023
1 parent 2cf3335 commit e835435
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 39 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- [#7716](https://github.com/apache/trafficcontrol/pull/7716) *Apache Traffic Server* Use GCC 11 for building.
- [#7742](https://github.com/apache/trafficcontrol/pull/7742) *Traffic Ops* Changed api tests to supply the absolute path of certs.
- [#7718](https://github.com/apache/trafficcontrol/pull/7718) *Traffic Ops* `/servers` endpoint now responds with RFC3339 timestamps for all timestamp fields. Cleaned up naming conventions and superfluous data.
- [#7765](https://github.com/apache/trafficcontrol/pull/7765) *Traffic Stats* now uses Traffic Ops APIv5

### Fixed
- [RFC3339](https://github.com/apache/trafficcontrol/issues/5911)
Expand Down
43 changes: 25 additions & 18 deletions traffic_stats/traffic_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/apache/trafficcontrol/lib/go-tc"
"github.com/apache/trafficcontrol/lib/go-util"
client "github.com/apache/trafficcontrol/traffic_ops/v3-client"
client "github.com/apache/trafficcontrol/traffic_ops/v5-client"

"github.com/Shopify/sarama"
"github.com/cihub/seelog"
Expand Down Expand Up @@ -163,7 +163,7 @@ var useSeelog bool = true
// about caches, cachegroups, and health urls
type RunningConfig struct {
HealthUrls map[string]map[string][]string // the 1st map key is CDN_name, the second is DsStats or CacheStats
CacheMap map[string]tc.Server // map hostName to cache
CacheMap map[string]tc.ServerV5 // map hostName to cache
LastSummaryTime time.Time
}

Expand Down Expand Up @@ -622,7 +622,7 @@ func calcDailyMaxGbps(client influx.Client, bp influx.BatchPoints, startTime tim
value = value / kilobitsToGigabits
statTime, _ := time.Parse(time.RFC3339, t)
infof("max gbps for cdn %v = %v", cdn, value)
var statsSummary tc.StatsSummary
var statsSummary tc.StatsSummaryV5
statsSummary.CDNName = util.StrPtr(cdn)
statsSummary.DeliveryService = util.StrPtr("all")
statsSummary.StatName = util.StrPtr("daily_maxgbps")
Expand Down Expand Up @@ -682,7 +682,7 @@ func calcDailyBytesServed(client influx.Client, bp influx.BatchPoints, startTime
bytesServedTB := bytesServed / bytesToTerabytes
infof("TBytes served for cdn %v = %v", cdn, bytesServedTB)
//write to Traffic Ops
var statsSummary tc.StatsSummary
var statsSummary tc.StatsSummaryV5
statsSummary.CDNName = util.StrPtr(cdn)
statsSummary.DeliveryService = util.StrPtr("all")
statsSummary.StatName = util.StrPtr("daily_bytesserved")
Expand Down Expand Up @@ -725,14 +725,14 @@ func queryDB(con influx.Client, cmd string, database string) (res []influx.Resul
return
}

func writeSummaryStats(config StartupConfig, statsSummary tc.StatsSummary) {
func writeSummaryStats(config StartupConfig, statsSummary tc.StatsSummaryV5) {
to, _, err := client.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false, time.Duration(config.ToRequestTimeoutSeconds)*time.Second)
if err != nil {
newErr := fmt.Errorf("Could not store summary stats! Error logging in to %v: %v", config.ToURL, err)
errorln(newErr)
return
}
_, _, err = to.CreateSummaryStats(statsSummary)
_, _, err = to.CreateSummaryStats(statsSummary, client.RequestOptions{})
if err != nil {
errorf("could not create summary stats: %v", err)
}
Expand All @@ -750,7 +750,7 @@ func getToData(config StartupConfig, init bool, configChan chan RunningConfig) {
return
}

servers, _, err := to.GetServers(nil)
servers, _, err := to.GetServers(client.RequestOptions{})
if err != nil {
msg := fmt.Sprintf("Error getting server list from %v: %v ", config.ToURL, err)
if init {
Expand All @@ -760,14 +760,14 @@ func getToData(config StartupConfig, init bool, configChan chan RunningConfig) {
return
}

runningConfig.CacheMap = make(map[string]tc.Server)
for _, server := range servers {
runningConfig.CacheMap = make(map[string]tc.ServerV5)
for _, server := range servers.Response {
runningConfig.CacheMap[server.HostName] = server
}

cacheStatPath := "/publish/CacheStats?hc=1&wildcard=1&stats="
dsStatPath := "/publish/DsStats?hc=1&wildcard=1&stats="
parameters, _, err := to.GetParametersByProfileName("TRAFFIC_STATS")
parameters, _, err := to.GetParametersByProfileName("TRAFFIC_STATS", client.RequestOptions{})
if err != nil {
msg := fmt.Sprintf("Error getting parameter list from %v: %v", config.ToURL, err)
if init {
Expand All @@ -777,7 +777,7 @@ func getToData(config StartupConfig, init bool, configChan chan RunningConfig) {
return
}

for _, param := range parameters {
for _, param := range parameters.Response {
if param.Name == "DsStats" {
statName := param.Value
dsStatPath += "," + statName
Expand All @@ -790,7 +790,13 @@ func getToData(config StartupConfig, init bool, configChan chan RunningConfig) {

setHealthURLs(config, &runningConfig, cacheStatPath, dsStatPath)

lastSummaryTimeResponse, _, err := to.GetSummaryStatsLastUpdated(util.StrPtr("daily_maxgbps"))
opts := client.RequestOptions{
QueryParameters: url.Values{
"lastSummaryDate": {"true"},
"statName": {"daily_maxgbps"},
},
}
lastSummaryTimeResponse, _, err := to.GetSummaryStatsLastUpdated(opts)
if err != nil {
errorf("unable to get summary stats last updated: %v", err)
} else if lastSummaryTimeResponse.Response.SummaryTime == nil {
Expand All @@ -811,7 +817,7 @@ func setHealthURLs(config StartupConfig, runningConfig *RunningConfig, cacheStat
}

if server.Type == tc.MonitorTypeName && server.Status == config.StatusToMon {
cdnName := server.CDNName
cdnName := server.CDN
if cdnName == "" {
errorln("Unable to find CDN name for " + server.HostName + ".. skipping")
continue
Expand All @@ -820,15 +826,16 @@ func setHealthURLs(config StartupConfig, runningConfig *RunningConfig, cacheStat
if runningConfig.HealthUrls[cdnName] == nil {
runningConfig.HealthUrls[cdnName] = make(map[string][]string)
}
healthURL := "http://" + server.HostName + "." + server.DomainName + ":" + strconv.Itoa(server.TCPPort) + cacheStatPath
tcpPort := util.CoalesceToDefault(server.TCPPort)
healthURL := "http://" + server.HostName + "." + server.DomainName + ":" + strconv.Itoa(tcpPort) + cacheStatPath
runningConfig.HealthUrls[cdnName]["CacheStats"] = append(runningConfig.HealthUrls[cdnName]["CacheStats"], healthURL)
healthURL = "http://" + server.HostName + "." + server.DomainName + ":" + strconv.Itoa(server.TCPPort) + dsStatPath
healthURL = "http://" + server.HostName + "." + server.DomainName + ":" + strconv.Itoa(tcpPort) + dsStatPath
runningConfig.HealthUrls[cdnName]["DsStats"] = append(runningConfig.HealthUrls[cdnName]["DsStats"], healthURL)
}
}
}

func calcMetrics(cdnName string, urls []string, cacheMap map[string]tc.Server, config StartupConfig) {
func calcMetrics(cdnName string, urls []string, cacheMap map[string]tc.ServerV5, config StartupConfig) {
sampleTime := time.Now().Unix()
// get the data from trafficMonitor
var trafMonData []byte
Expand Down Expand Up @@ -952,7 +959,7 @@ func calcDsValues(tmData []byte, cdnName string, sampleTime int64, config Startu
return nil
}

func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cacheMap map[string]tc.Server, config StartupConfig) error {
func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cacheMap map[string]tc.ServerV5, config StartupConfig) error {
var jData tc.LegacyStats
err := json.Unmarshal(trafmonData, &jData)
if err != nil {
Expand Down Expand Up @@ -998,7 +1005,7 @@ func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cache
}

tags := map[string]string{
"cachegroup": cache.Cachegroup,
"cachegroup": cache.CacheGroup,
"hostname": string(cacheName),
"cdn": cdnName,
"type": cache.Type,
Expand Down
43 changes: 22 additions & 21 deletions traffic_stats/traffic_stats_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,14 @@ import (
"time"

"github.com/apache/trafficcontrol/lib/go-tc"
"github.com/apache/trafficcontrol/lib/go-util"
influx "github.com/influxdata/influxdb/client/v2"
)

func TestCalcCacheValuesWithInvalidValue(t *testing.T) {
stats := make(map[string][]tc.ResultStatVal)
caches := make(map[tc.CacheName]map[string][]tc.ResultStatVal)
cacheMap := make(map[string]tc.Server)
cacheMap := make(map[string]tc.ServerV5)
resultSatsVal := []tc.ResultStatVal{
{
Span: 0,
Expand All @@ -43,7 +44,7 @@ func TestCalcCacheValuesWithInvalidValue(t *testing.T) {
}
stats["maxKbps"] = resultSatsVal
caches["cache1"] = stats
cacheMap["cache1"] = tc.Server{}
cacheMap["cache1"] = tc.ServerV5{}
config := StartupConfig{
BpsChan: make(chan influx.BatchPoints),
}
Expand Down Expand Up @@ -76,7 +77,7 @@ func TestCalcCacheValuesWithInvalidValue(t *testing.T) {
func TestCalcCacheValuesWithEmptyInterface(t *testing.T) {
stats := make(map[string][]tc.ResultStatVal)
caches := make(map[tc.CacheName]map[string][]tc.ResultStatVal)
cacheMap := make(map[string]tc.Server)
cacheMap := make(map[string]tc.ServerV5)
resultSatsVal := []tc.ResultStatVal{
{
Span: 0,
Expand All @@ -85,7 +86,7 @@ func TestCalcCacheValuesWithEmptyInterface(t *testing.T) {
}
stats["maxKbps"] = resultSatsVal
caches["cache1"] = stats
cacheMap["cache1"] = tc.Server{}
cacheMap["cache1"] = tc.ServerV5{}
config := StartupConfig{
BpsChan: make(chan influx.BatchPoints),
}
Expand Down Expand Up @@ -118,7 +119,7 @@ func TestCalcCacheValuesWithEmptyInterface(t *testing.T) {
func TestCalcCacheValues(t *testing.T) {
stats := make(map[string][]tc.ResultStatVal)
caches := make(map[tc.CacheName]map[string][]tc.ResultStatVal)
cacheMap := make(map[string]tc.Server)
cacheMap := make(map[string]tc.ServerV5)
resultSatsVal := []tc.ResultStatVal{
{
Span: 0,
Expand All @@ -128,7 +129,7 @@ func TestCalcCacheValues(t *testing.T) {
}
stats["maxKbps"] = resultSatsVal
caches["cache1"] = stats
cacheMap["cache1"] = tc.Server{}
cacheMap["cache1"] = tc.ServerV5{}
config := StartupConfig{
BpsChan: make(chan influx.BatchPoints),
}
Expand Down Expand Up @@ -163,61 +164,61 @@ func TestSetHealthURLs(t *testing.T) {
StatusToMon: tc.CacheStatusOffline.String(),
}
runningCfg := RunningConfig{}
runningCfg.CacheMap = map[string]tc.Server{
runningCfg.CacheMap = map[string]tc.ServerV5{
"tm1": {
CDNName: "foo",
CDN: "foo",
DomainName: "example.org",
HostName: "tm1",
Status: tc.CacheStatusOffline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.MonitorTypeName,
},
"tm2": {
CDNName: "bar",
CDN: "bar",
DomainName: "example.org",
HostName: "tm2",
Status: tc.CacheStatusOffline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.MonitorTypeName,
},
"tm3": {
CDNName: "foo",
CDN: "foo",
DomainName: "example.org",
HostName: "tm3",
Status: tc.CacheStatusOffline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.MonitorTypeName,
},
"tm4": {
CDNName: "bar",
CDN: "bar",
DomainName: "example.org",
HostName: "tm4",
Status: tc.CacheStatusOffline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.MonitorTypeName,
},
"tm5": {
CDNName: "foo",
CDN: "foo",
DomainName: "example.org",
HostName: "tm5",
Status: tc.CacheStatusOnline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.MonitorTypeName,
},
"tm6": {
CDNName: "bar",
CDN: "bar",
DomainName: "example.org",
HostName: "tm6",
Status: tc.CacheStatusOnline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.MonitorTypeName,
},
"cache1": {
CDNName: "foo",
CDN: "foo",
DomainName: "example.org",
HostName: "cache1",
Status: tc.CacheStatusOffline.String(),
TCPPort: 8080,
TCPPort: util.Ptr(8080),
Type: tc.CacheTypeEdge.String(),
},
}
Expand Down

0 comments on commit e835435

Please sign in to comment.