From 6f1a8feea94478c2545c66aebc0e5a4610c4cc3c Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Tue, 14 Feb 2017 10:20:54 -0700 Subject: [PATCH 01/15] Fix TO client Login, add stats UserAgent --- traffic_ops/client/traffic_ops.go | 2 +- traffic_stats/traffic_stats.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/traffic_ops/client/traffic_ops.go b/traffic_ops/client/traffic_ops.go index 40b43e557d..624346f875 100644 --- a/traffic_ops/client/traffic_ops.go +++ b/traffic_ops/client/traffic_ops.go @@ -127,7 +127,7 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { } // Deprecated: Login is deprecated, use LoginWithAgent instead. The `Login` function with its present signature will be removed in the next version and replaced with `Login(toURL string, toUser string, toPasswd string, insecure bool, userAgent string)`. The `LoginWithAgent` function will be removed the version after that. -func Login(toURL string, toUser string, toPasswd string, insecure bool, userAgent string) (*Session, error) { +func Login(toURL string, toUser string, toPasswd string, insecure bool) (*Session, error) { return LoginWithAgent(toURL, toUser, toPasswd, insecure, "traffic-ops-client") // TODO add version } diff --git a/traffic_stats/traffic_stats.go b/traffic_stats/traffic_stats.go index 18419cac68..3d474c0a58 100644 --- a/traffic_stats/traffic_stats.go +++ b/traffic_stats/traffic_stats.go @@ -40,6 +40,8 @@ import ( influx "github.com/influxdata/influxdb/client/v2" ) +const UserAgent = "traffic-stats" + const ( // FATAL will exit after printing error FATAL = iota @@ -412,7 +414,7 @@ func queryDB(con influx.Client, cmd string, database string) (res []influx.Resul } func writeSummaryStats(config StartupConfig, statsSummary traffic_ops.StatsSummary) { - to, err := traffic_ops.Login(config.ToURL, config.ToUser, config.ToPasswd, true) + to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent) if err != nil { newErr := fmt.Errorf("Could not store summary stats! Error logging in to %v: %v", config.ToURL, err) log.Error(newErr) @@ -426,7 +428,7 @@ func writeSummaryStats(config StartupConfig, statsSummary traffic_ops.StatsSumma func getToData(config StartupConfig, init bool, configChan chan RunningConfig) { var runningConfig RunningConfig - to, err := traffic_ops.Login(config.ToURL, config.ToUser, config.ToPasswd, true) + to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent) if err != nil { msg := fmt.Sprintf("Error logging in to %v: %v", config.ToURL, err) if init { From fb33bb9097eb1128e9e9eb3a4a7c70ed121d8fbc Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Tue, 14 Feb 2017 17:20:42 -0700 Subject: [PATCH 02/15] Add TM2 crconfig to unmarshal full CRConfig json --- .../traffic_monitor/crconfig/data.go | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 traffic_monitor_golang/traffic_monitor/crconfig/data.go diff --git a/traffic_monitor_golang/traffic_monitor/crconfig/data.go b/traffic_monitor_golang/traffic_monitor/crconfig/data.go new file mode 100644 index 0000000000..6c9136f316 --- /dev/null +++ b/traffic_monitor_golang/traffic_monitor/crconfig/data.go @@ -0,0 +1,239 @@ +package crconfig + +import ( + "time" +) + +// CRConfig is JSON-serializable as the CRConfig used by Traffic Control. However, it also contains diff timestamps, for the last update time of each field. These can be used to return only fields which have changed since a given time. +type CRConfig struct { + Config Config `json:"config,omitempty"` + ConfigTime time.Time `json:"-"` + ContentServers map[string]Server `json:"contentServers,omitempty"` + ContentServersTime map[string]time.Time `json:"-"` + ContentRouters map[string]Router `json:"contentRouters,omitempty"` + ContentRoutersTime map[string]time.Time `json:"-"` + DeliveryServices map[string]DeliveryService `json:"deliveryServices,omitempty"` + DeliveryServicesTime map[string]time.Time `json:"-"` + EdgeLocations map[string]LatitudeLongitude `json:"edgeLocations,omitempty"` + EdgeLocationsTime map[string]time.Time `json:"-"` + Monitors map[string]Monitor `json:"monitors,omitempty"` + MonitorsTime map[string]time.Time `json:"-"` + Stats Stats `json:"stats,omitempty"` + StatsTime time.Time `json:"-"` +} + +type MatchSet struct { + Protocol string `json:"protocol"` + MatchList []MatchType `json:"matchlist"` +} + +type MatchType struct { + MatchType string `json:"match-type"` + Regex string `json:"regex"` +} + +type Config struct { + APICacheControlMaxAge *int `json:"api.cache-control.max_age,string,omitempty"` + APICacheControlMaxAgeLastTime time.Time `json:"-"` + ConsistentDNSRouting *bool `json:"consistent.dns.routing,string,omitempty"` + ConsistentDNSRoutingTime time.Time `json:"-"` + CoverageZonePollingIntervalSeconds *int `json:"coveragezone.polling.interval,string,omitempty"` + CoverageZonePollingIntervalSecondsTime time.Time `json:"-"` + CoverageZonePollingURL *string `json:"coveragezone.polling.url,omitempty"` + CoverageZonePollingURLTime time.Time `json:"-"` + DNSSecDynamicResponseExpiration *string `json:"dnssec.dynamic.response.expiration,omitempty"` + DNSSecDynamicResponseExpirationTime time.Time `json:"-"` + DNSSecEnabled *bool `json:"dnssec.enabled,string,omitempty"` + DNSSecEnabledTime time.Time `json:"-"` + DomainName *string `json:"domain_name,omitempty"` + DomainNameTime time.Time `json:"-"` + FederationMappingPollingIntervalSeconds *int `json:"federationmapping.polling.interval,string,omitempty"` + FederationMappingPollingIntervalSecondsTime time.Time `json:"-"` + FederationMappingPollingURL *string `json:"federationmapping.polling.url"` + FederationMappingPollingURLTime time.Time `json:"-"` + GeoLocationPollingInterval *int `json:"geolocation.polling.interval,string,omitempty"` + GeoLocationPollingIntervalTime time.Time `json:"-"` + GeoLocationPollingURL *string `json:"geolocation.polling.url,omitempty"` + GeoLocationPollingURLTime time.Time `json:"-"` + KeyStoreMaintenanceIntervalSeconds *int `json:"keystore.maintenance.interval,string,omitempty"` + KeyStoreMaintenanceIntervalSecondsTime time.Time `json:"-"` + NeustarPollingIntervalSeconds *int `json:"neustar.polling.interval,string,omitempty"` + NeustarPollingIntervalSecondsTime time.Time `json:"-"` + NeustarPollingURL *string `json:"neustar.polling.url,omitempty"` + NeustarPollingURLTime time.Time `json:"-"` + SOA *SOA `json:"soa,omitempty"` + SOATime time.Time `json:"-"` + DNSSecInceptionSeconds *int `json:"dnssec.inception,string,omitempty"` + DNSSecInceptionSecondsTime time.Time `json:"-"` + Ttls *TTL `json:"ttls,omitempty"` + TtlsTime time.Time `json:"-"` + Weight *float64 `json:"weight,string,omitempty"` + WeightTime time.Time `json:"-"` + ZoneManagerCacheMaintenanceIntervalSeconds *int `json:"zonemanager.cache.maintenance.interval,string,omitempty"` + ZoneManagerCacheMaintenanceIntervalSecondsTime time.Time `json:"-"` + ZoneManagerThreadpoolScale *float64 `json:"zonemanager.threadpool.scale,string,omitempty"` + ZoneManagerThreadpoolScaleTime time.Time `json:"-"` +} + +type SOA struct { + Admin *string `json:"admin,omitempty"` + AdminTime time.Time + ExpireSeconds *int `json:"expire,string,omitempty"` + ExpireSecondsTime time.Time + MinimumSeconds *int `json:"minimum,string,omitempty"` + MinimumSecondsTime time.Time + RefreshSeconds *int `json:"refresh,string,omitempty"` + RefreshSecondsTime time.Time + RetrySeconds *int `json:"retry,string,omitempty"` + RetrySecondsTime time.Time +} + +type TTL struct { + ASeconds *int `json:"A,string,omitempty"` + ASecondsTime time.Time + AAAASeconds *int `json:"AAAA,string,omitempty"` + AAAASecondsTime time.Time + DNSkeySeconds *int `json:"DNSKEY,string,omitempty"` + DNSkeySecondsTime time.Time + DSSeconds *int `json:"DS,string,omitempty"` + DSSecondsTime time.Time + NSSeconds *int `json:"NS,string,omitempty"` + NSSecondsTime time.Time + SOASeconds *int `json:"SOA,string,omitempty"` + SOASecondsTime time.Time +} + +type Router struct { + APIPort *int `json:"apiPort,string,omitempty"` + APIPortTime time.Time + FQDN *string `json:"fqdn,omitempty"` + FQDNTime time.Time + HTTPSPort *int `json:"httpsPort,string,omitempty"` + HTTPSPortTime time.Time + IP *string `json:"ip,omitempty"` + IPTime time.Time + IP6 *string `json:"ip6,omitempty"` + IP6Time time.Time + Location *string `json:"location,omitempty"` + LocationTime time.Time + Port *int `json:"port,string,omitempty"` + PortTime time.Time + Profile *string `json:"profile,omitempty"` + ProfileTime time.Time + Status *Status `json:"status,omitempty"` + StatusTime time.Time +} + +type Status string + +type Server struct { + CacheGroup *string `json:"cacheGroup,omitempty"` + CacheGroupTime time.Time `json:"-"` + DeliveryServices map[string][]string `json:"deliveryServices,omitempty"` + DeliveryServicesTime time.Time `json:"-"` + Fqdn *string `json:"fqdn,omitempty"` + FqdnTime time.Time `json:"-"` + HashCount *int `json:"hashCount,omitempty"` + HashCountTime time.Time `json:"-"` + HashId *string `json:"hashId,omitempty"` + HashIdTime time.Time `json:"-"` + HttpsPort *int `json:"httpsPort,string,omitempty"` + HttpsPortTime time.Time `json:"-"` + InterfaceName *string `json:"interfaceName,omitempty"` + InterfaceNameTime time.Time `json:"-"` + Ip *string `json:"ip,omitempty"` + IpTime time.Time `json:"-"` + Ip6 *string `json:"ip6,omitempty"` + Ip6Time time.Time `json:"-"` + LocationId *string `json:"locationId,omitempty"` + LocationIdTime time.Time `json:"-"` + Port *int `json:"port,string,omitempty"` + PortTime time.Time `json:"-"` + Profile *string `json:"profile,omitempty"` + ProfileTime time.Time `json:"-"` + Status *Status `json:"status,omitempty"` + StatusTime time.Time `json:"-"` + ServerType *string `json:"type,omitempty"` + ServerTypeTime time.Time `json:"-"` +} + +type DeliveryService struct { + CoverageZoneOnly *bool `json:"coverageZoneOnly,string,omitempty"` + CoverageZoneOnlyTime time.Time `json:"-"` + Dispersion *Dispersion `json:"dispersion,omitempty"` + DispersionTime time.Time `json:"-"` + Domains []string `json:"domains,omitempty"` + DomainsTime time.Time `json:"-"` + GeoLocationProvider *string `json:"geoLocationProvider,omitempty"` + GeoLocationProviderTime time.Time `json:"-"` + MatchSets []MatchSet `json:"matchSets,omitempty"` + MatchSetsTime time.Time `json:"-"` + MissLocation *LatLon `json:"missLocation,omitempty"` + MissLocationTime time.Time `json:"-"` + Protocol *DeliveryServiceProtocol `json:"protocol,omitempty"` + ProtocolTime time.Time `json:"-"` + RegionalGeoBlocking *bool `json:"regionalGeoBlocking,string,omitempty"` + RegionalGeoBlockingTime time.Time `json:"-"` + ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` + ResponseHeadersTime time.Time `json:"-"` + Soa *SOA `json:"soa,omitempty"` + SoaTime time.Time `json:"-"` + SSLEnabled *bool `json:"sslEnabled,string,omitempty"` + SSLEnabledTime time.Time `json:"-"` + TTL *int `json:"ttl,string,omitempty"` + TTLTime time.Time `json:"-"` + TTLs *TTL `json:"ttls,omitempty"` + TTLsTime time.Time `json:"-"` +} +type Dispersion struct { + Limit int `json:"limit,omitempty"` + Shuffled bool `json:"shuffled,string,omitempty"` +} +type LatLon struct { + Lat float64 `json:"lat,string"` + Lon float64 `json:"lon,string"` +} + +type LatitudeLongitude struct { + Lat float64 `json:"latitude"` + Lon float64 `json:"longitude"` +} + +type DeliveryServiceProtocol struct { + AcceptHTTPS bool `json:"acceptHttps,string,omitempty"` + RedirectOnHTTPS bool `json:"redirectOnHttps,string,omitempty"` +} + +type Monitor struct { + FQDN *string `json:"fqdn,omitempty"` + FQDNTime time.Time `json:"-"` + HTTPSPort *int `json:"httpsPort,string,omitempty"` + HTTPSPortTime time.Time `json:"-"` + IP *string `json:"ip,omitempty"` + IPTime time.Time `json:"-"` + IP6 *string `json:"ip6,omitempty"` + IP6Time time.Time `json:"-"` + Location *string `json:"location,omitempty"` + LocationTime time.Time `json:"-"` + Port *int `json:"port,string,omitempty"` + PortTime time.Time `json:"-"` + Profile *string `json:"profile,omitempty"` + ProfileTime time.Time `json:"-"` + Status *Status `json:"status,omitempty"` + StatusTime time.Time `json:"-"` +} + +type Stats struct { + CDNName *string `json:"CDN_name,omitempty"` + CDNNameTime time.Time `json:"-"` + DateUnixSeconds *int64 `json:"date,omitempty"` + DateUnixSecondsTime time.Time `json:"-"` + TMHost *string `json:"tm_host,omitempty"` + TMHostTime time.Time `json:"-"` + TMPath *string `json:"tm_path,omitempty"` + TMPathTime time.Time `json:"-"` + TMUser *string `json:"tm_user,omitempty"` + TMUserTime time.Time `json:"-"` + TMVersion *string `json:"tm_version,omitempty"` + TMVersionTime time.Time `json:"-"` +} From d298d0ff0a29f431babd90166f3625d7fd22cb10 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Tue, 14 Feb 2017 17:17:13 -0700 Subject: [PATCH 03/15] Fix TM2 to overwrite monitoring.json with CRConfig --- .../trafficopswrapper/trafficopswrapper.go | 195 +++++++++++++++++- 1 file changed, 193 insertions(+), 2 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index 2815507a9f..e30c39ece9 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -20,9 +20,11 @@ package trafficopswrapper */ import ( + "encoding/json" "fmt" "sync" + "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/crconfig" to "github.com/apache/incubator-trafficcontrol/traffic_ops/client" ) @@ -34,6 +36,7 @@ type ITrafficOpsSession interface { URL() (string, error) User() (string, error) Servers() ([]to.Server, error) + Profiles() ([]to.Profile, error) Parameters(profileName string) ([]to.Parameter, error) DeliveryServices() ([]to.DeliveryService, error) CacheGroups() ([]to.CacheGroup, error) @@ -83,8 +86,8 @@ func (s TrafficOpsSessionThreadsafe) CRConfigRaw(cdn string) ([]byte, error) { return b, e } -// TrafficMonitorConfigMap returns the Traffic Monitor config map from the Traffic Ops. This is safe for multiple goroutines. -func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMap(cdn string) (*to.TrafficMonitorConfigMap, error) { +// TrafficMonitorConfigMapRaw returns the Traffic Monitor config map from the Traffic Ops, directly from the monitoring.json endpoint. This is not usually what is needed, rather monitoring needs the snapshotted CRConfig data, which is filled in by `TrafficMonitorConfigMap`. This is safe for multiple goroutines. +func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMapRaw(cdn string) (*to.TrafficMonitorConfigMap, error) { s.m.Lock() defer s.m.Unlock() if s.session == nil || *s.session == nil { @@ -94,6 +97,185 @@ func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMap(cdn string) (*to.Tr return d, e } +// TrafficMonitorConfigMap returns the Traffic Monitor config map from the Traffic Ops. This is safe for multiple goroutines. +func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMap(cdn string) (*to.TrafficMonitorConfigMap, error) { + mc, err := s.TrafficMonitorConfigMapRaw(cdn) + if err != nil { + fmt.Printf("DEBUG4 TrafficMonitorConfigMap err: %v\n", err) + return nil, fmt.Errorf("getting monitor config map: %v", err) + } + + crcData, err := s.CRConfigRaw(cdn) + if err != nil { + fmt.Printf("DEBUG4 CRConfigRaw err: %v\n", err) + return nil, fmt.Errorf("getting CRConfig: %v", err) + } + + crConfig := crconfig.CRConfig{} + if err := json.Unmarshal(crcData, &crConfig); err != nil { + fmt.Printf("DEBUG4 CRConfig Unmarshal err: %v\n", err) + return nil, fmt.Errorf("Error unmarshalling CRConfig JSON: %v", err) + return nil, err + } + + mc, err = CreateMonitorConfig(crConfig, mc) + if err != nil { + fmt.Printf("DEBUG4 CreateMonitorConfig err: %v\n", err) + return nil, fmt.Errorf("Error creating Traffic Monitor Config: %v", err) + } + + // mcMap, err := to.TrafficMonitorTransformToMap(mc) + // if err != nil { + // fmt.Printf("DEBUG4 TrafficMonitorTransformToMap err: %v\n", err) + // return nil, fmt.Errorf("Error transforming Traffic Monitor Config to Map: %v", err) + // } + + // debug + + // if bytes, err := json.Marshal(mcMap); err != nil { + // fmt.Printf("DEBUG4 error marshalling map: %v\n", err) + // } else { + // fmt.Printf("DEBUG4 New Map: %v\n\n", string(bytes)) + // } + + return mc, nil +} + +func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfigMap) (*to.TrafficMonitorConfigMap, error) { + // mc := to.TrafficMonitorConfig{} + + // cgs, err := s.CacheGroups() + // if err != nil { + // return nil, fmt.Errorf("Error getting CacheGroups: %v", err) + // } + + // allProfiles, err := s.Profiles() + // if err != nil { + // return nil, fmt.Errorf("Error getting Profiles: %v", err) + // } + + // Dump the "live" monitoring.json servers, and populate with the "snapshotted" CRConfig + mc.TrafficServer = map[string]to.TrafficServer{} + for name, srv := range crConfig.ContentServers { + mc.TrafficServer[name] = to.TrafficServer{ + Profile: *srv.Profile, + IP: *srv.Ip, + Status: string(*srv.Status), + CacheGroup: *srv.CacheGroup, + IP6: *srv.Ip6, + Port: *srv.Port, + HostName: name, + FQDN: *srv.Fqdn, + InterfaceName: *srv.InterfaceName, + Type: *srv.ServerType, + HashID: *srv.HashId, + } + } + + // for _, cg := range cgs { + // mc.CacheGroups = append(mc.CacheGroups, to.TMCacheGroup{ + // Name: cg.Name, + // Coordinates: to.Coordinates{ + // Latitude: cg.Latitude, + // Longitude: cg.Longitude, + // }, + // }) + // } + + // monitorProfile := "" + + // Dump the "live" monitoring.json monitors, and populate with the "snapshotted" CRConfig + mc.TrafficMonitor = map[string]to.TrafficMonitor{} + for name, mon := range crConfig.Monitors { + // monitorProfile = *mon.Profile + mc.TrafficMonitor[name] = to.TrafficMonitor{ + Port: *mon.Port, + IP6: *mon.IP6, + IP: *mon.IP, + HostName: name, + FQDN: *mon.FQDN, + Profile: *mon.Profile, + Location: *mon.Location, + Status: string(*mon.Status), + } + } + + // monitorParams, err := s.Parameters(monitorProfile) + // if err != nil { + // return nil, fmt.Errorf("Error getting profile %v parameters: %v", monitorProfile, err) + // } + // mc.Config = map[string]interface{}{} + // for _, param := range monitorParams { + // if numParam, err := strconv.ParseFloat(param.Value, 64); err == nil { + // mc.Config[param.Name] = numParam + // } else { + // mc.Config[param.Name] = param.Value + // } + // } + + // Dump the "live" monitoring.json DeliveryServices, and populate with the "snapshotted" CRConfig + mc.DeliveryService = map[string]to.TMDeliveryService{} + for name, _ := range crConfig.DeliveryServices { + mc.DeliveryService[name] = to.TMDeliveryService{ + XMLID: name, + TotalTPSThreshold: 0, // TODO verify + Status: "Reported", // TODO verify + TotalKbpsThreshold: 0, // TODO verify + } + } + + // mc.Profiles = []to.TMProfile{} + // for _, prof := range allProfiles { + // if strings.HasPrefix(prof.Name, "EDGE") || strings.HasPrefix(prof.Name, "TEAK") { + // mc.Profiles = append(mc.Profiles, to.TMProfile{ + // Name: prof.Name, + // Type: "EDGE", + // }) + // } else if strings.HasPrefix(prof.Name, "MID") { + // mc.Profiles = append(mc.Profiles, to.TMProfile{ + // Name: prof.Name, + // Type: "MID", + // }) + // } + // } + // for profI, prof := range mc.Profiles { + // // MID2_TOP_v5.3.2-757 + // prof.Parameters.Thresholds = map[string]to.HealthThreshold{} + // // TODO lock + // params, err := s.Parameters(prof.Name) + // if err != nil { + // return nil, fmt.Errorf("Error getting profile %v parameters: %v", prof, err) + // } + // fmt.Printf("DEBUG5 profile %v len(params) %v\n", prof.Name, len(params)) + // for _, param := range params { + // if param.Name == "health.connection.timeout" { + // i, err := strconv.Atoi(param.Value) + // if err != nil { + // return nil, fmt.Errorf("Error getting profile %v parameter %v: %v", prof, param.Name, err) + // } + // prof.Parameters.HealthConnectionTimeout = i + // } else if param.Name == "health.polling.url" { + // prof.Parameters.HealthPollingURL = param.Value + // } else if param.Name == "history.count" { + // i, err := strconv.Atoi(param.Value) + // if err != nil { + // return nil, fmt.Errorf("Error getting profile %v parameter %v: %v", prof, param.Name, err) + // } + // prof.Parameters.HistoryCount = i + // } else if strings.HasPrefix(param.Name, "health.threshold.") { + // stat := param.Name[len("health.threshold."):] + // thresh, err := to.StrToThreshold(param.Value) + // if err != nil { + // return nil, fmt.Errorf("Error getting profile %v parameter %v: %v", prof, param.Name, err) + // } + // prof.Parameters.Thresholds[stat] = thresh + // } + // } + // mc.Profiles[profI] = prof + // } + return mc, nil +} + // Set sets the internal Traffic Ops session. This is safe for multiple goroutines, being aware they will race. func (s TrafficOpsSessionThreadsafe) Set(session *to.Session) { s.m.Lock() @@ -110,6 +292,15 @@ func (s TrafficOpsSessionThreadsafe) Servers() ([]to.Server, error) { return (*s.session).Servers() } +func (s TrafficOpsSessionThreadsafe) Profiles() ([]to.Profile, error) { + s.m.Lock() + defer s.m.Unlock() + if s.session == nil || *s.session == nil { + return nil, ErrNilSession + } + return (*s.session).Profiles() +} + func (s TrafficOpsSessionThreadsafe) Parameters(profileName string) ([]to.Parameter, error) { s.m.Lock() defer s.m.Unlock() From 6d2edae7a3345614b93081387346e41462c2a750 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Tue, 14 Feb 2017 17:18:53 -0700 Subject: [PATCH 04/15] Remove TM2 commented code to build monitoring.json --- .../trafficopswrapper/trafficopswrapper.go | 109 +----------------- 1 file changed, 2 insertions(+), 107 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index e30c39ece9..741bfd809d 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -101,59 +101,28 @@ func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMapRaw(cdn string) (*to func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMap(cdn string) (*to.TrafficMonitorConfigMap, error) { mc, err := s.TrafficMonitorConfigMapRaw(cdn) if err != nil { - fmt.Printf("DEBUG4 TrafficMonitorConfigMap err: %v\n", err) return nil, fmt.Errorf("getting monitor config map: %v", err) } crcData, err := s.CRConfigRaw(cdn) if err != nil { - fmt.Printf("DEBUG4 CRConfigRaw err: %v\n", err) return nil, fmt.Errorf("getting CRConfig: %v", err) } crConfig := crconfig.CRConfig{} if err := json.Unmarshal(crcData, &crConfig); err != nil { - fmt.Printf("DEBUG4 CRConfig Unmarshal err: %v\n", err) - return nil, fmt.Errorf("Error unmarshalling CRConfig JSON: %v", err) - return nil, err + return nil, fmt.Errorf("unmarshalling CRConfig JSON: %v", err) } mc, err = CreateMonitorConfig(crConfig, mc) if err != nil { - fmt.Printf("DEBUG4 CreateMonitorConfig err: %v\n", err) - return nil, fmt.Errorf("Error creating Traffic Monitor Config: %v", err) + return nil, fmt.Errorf("creating Traffic Monitor Config: %v", err) } - // mcMap, err := to.TrafficMonitorTransformToMap(mc) - // if err != nil { - // fmt.Printf("DEBUG4 TrafficMonitorTransformToMap err: %v\n", err) - // return nil, fmt.Errorf("Error transforming Traffic Monitor Config to Map: %v", err) - // } - - // debug - - // if bytes, err := json.Marshal(mcMap); err != nil { - // fmt.Printf("DEBUG4 error marshalling map: %v\n", err) - // } else { - // fmt.Printf("DEBUG4 New Map: %v\n\n", string(bytes)) - // } - return mc, nil } func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfigMap) (*to.TrafficMonitorConfigMap, error) { - // mc := to.TrafficMonitorConfig{} - - // cgs, err := s.CacheGroups() - // if err != nil { - // return nil, fmt.Errorf("Error getting CacheGroups: %v", err) - // } - - // allProfiles, err := s.Profiles() - // if err != nil { - // return nil, fmt.Errorf("Error getting Profiles: %v", err) - // } - // Dump the "live" monitoring.json servers, and populate with the "snapshotted" CRConfig mc.TrafficServer = map[string]to.TrafficServer{} for name, srv := range crConfig.ContentServers { @@ -172,18 +141,6 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig } } - // for _, cg := range cgs { - // mc.CacheGroups = append(mc.CacheGroups, to.TMCacheGroup{ - // Name: cg.Name, - // Coordinates: to.Coordinates{ - // Latitude: cg.Latitude, - // Longitude: cg.Longitude, - // }, - // }) - // } - - // monitorProfile := "" - // Dump the "live" monitoring.json monitors, and populate with the "snapshotted" CRConfig mc.TrafficMonitor = map[string]to.TrafficMonitor{} for name, mon := range crConfig.Monitors { @@ -200,19 +157,6 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig } } - // monitorParams, err := s.Parameters(monitorProfile) - // if err != nil { - // return nil, fmt.Errorf("Error getting profile %v parameters: %v", monitorProfile, err) - // } - // mc.Config = map[string]interface{}{} - // for _, param := range monitorParams { - // if numParam, err := strconv.ParseFloat(param.Value, 64); err == nil { - // mc.Config[param.Name] = numParam - // } else { - // mc.Config[param.Name] = param.Value - // } - // } - // Dump the "live" monitoring.json DeliveryServices, and populate with the "snapshotted" CRConfig mc.DeliveryService = map[string]to.TMDeliveryService{} for name, _ := range crConfig.DeliveryServices { @@ -224,55 +168,6 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig } } - // mc.Profiles = []to.TMProfile{} - // for _, prof := range allProfiles { - // if strings.HasPrefix(prof.Name, "EDGE") || strings.HasPrefix(prof.Name, "TEAK") { - // mc.Profiles = append(mc.Profiles, to.TMProfile{ - // Name: prof.Name, - // Type: "EDGE", - // }) - // } else if strings.HasPrefix(prof.Name, "MID") { - // mc.Profiles = append(mc.Profiles, to.TMProfile{ - // Name: prof.Name, - // Type: "MID", - // }) - // } - // } - // for profI, prof := range mc.Profiles { - // // MID2_TOP_v5.3.2-757 - // prof.Parameters.Thresholds = map[string]to.HealthThreshold{} - // // TODO lock - // params, err := s.Parameters(prof.Name) - // if err != nil { - // return nil, fmt.Errorf("Error getting profile %v parameters: %v", prof, err) - // } - // fmt.Printf("DEBUG5 profile %v len(params) %v\n", prof.Name, len(params)) - // for _, param := range params { - // if param.Name == "health.connection.timeout" { - // i, err := strconv.Atoi(param.Value) - // if err != nil { - // return nil, fmt.Errorf("Error getting profile %v parameter %v: %v", prof, param.Name, err) - // } - // prof.Parameters.HealthConnectionTimeout = i - // } else if param.Name == "health.polling.url" { - // prof.Parameters.HealthPollingURL = param.Value - // } else if param.Name == "history.count" { - // i, err := strconv.Atoi(param.Value) - // if err != nil { - // return nil, fmt.Errorf("Error getting profile %v parameter %v: %v", prof, param.Name, err) - // } - // prof.Parameters.HistoryCount = i - // } else if strings.HasPrefix(param.Name, "health.threshold.") { - // stat := param.Name[len("health.threshold."):] - // thresh, err := to.StrToThreshold(param.Value) - // if err != nil { - // return nil, fmt.Errorf("Error getting profile %v parameter %v: %v", prof, param.Name, err) - // } - // prof.Parameters.Thresholds[stat] = thresh - // } - // } - // mc.Profiles[profI] = prof - // } return mc, nil } From 76f7f25e1f27b2f237646bf4aa3c965506eadf27 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Tue, 14 Feb 2017 18:06:16 -0700 Subject: [PATCH 05/15] Change TM2 monitoring to use raw DS for thresholds --- .../trafficopswrapper/trafficopswrapper.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index 741bfd809d..0bbc483146 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -158,16 +158,22 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig } // Dump the "live" monitoring.json DeliveryServices, and populate with the "snapshotted" CRConfig + // But keep using the monitoring.json thresholds, because they're not in the CRConfig. + rawDeliveryServices := mc.DeliveryService mc.DeliveryService = map[string]to.TMDeliveryService{} for name, _ := range crConfig.DeliveryServices { - mc.DeliveryService[name] = to.TMDeliveryService{ - XMLID: name, - TotalTPSThreshold: 0, // TODO verify - Status: "Reported", // TODO verify - TotalKbpsThreshold: 0, // TODO verify + if rawDS, ok := rawDeliveryServices[name]; ok { + // use the raw DS if it exists, because the CRConfig doesn't have thresholds or statuses + mc.DeliveryService[name] = rawDS + } else { + mc.DeliveryService[name] = to.TMDeliveryService{ + XMLID: name, + TotalTPSThreshold: 0, + Status: "REPORTED", + TotalKbpsThreshold: 0, + } } } - return mc, nil } From f9e213f1476e86560ed5072132615beea06d0549 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Wed, 15 Feb 2017 11:30:10 -0700 Subject: [PATCH 06/15] Fix TO client cache, make requests threadsafe Fixes the client cache, which was previously recreating the map on every request, and thus only ever caching one request. Changes the TO client to make HTTP requests threadsafe. This should make all `Session` funcs threadsafe, but I haven't tested thoroughly, it's possible some functions are storing or mutating state and need additional locks. --- traffic_ops/client/traffic_ops.go | 73 +++++++++++++++++++------------ 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/traffic_ops/client/traffic_ops.go b/traffic_ops/client/traffic_ops.go index 624346f875..bfb53097c3 100644 --- a/traffic_ops/client/traffic_ops.go +++ b/traffic_ops/client/traffic_ops.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "net/http" "strings" + "sync" "time" "github.com/juju/persistent-cookiejar" @@ -35,10 +36,23 @@ type Session struct { Password string URL string UserAgent *http.Client - Cache map[string]CacheEntry + cache map[string]CacheEntry + cacheMutex *sync.RWMutex UserAgentStr string } +func NewSession(user, password, url, userAgent string, client *http.Client) *Session { + return &Session{ + UserName: user, + Password: password, + URL: url, + UserAgent: client, + cache: map[string]CacheEntry{}, + cacheMutex: &sync.RWMutex{}, + UserAgentStr: userAgent, + } +} + // HTTPError is returned on Update Session failure. type HTTPError struct { HTTPStatusCode int @@ -103,16 +117,12 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { return nil, err } - to := Session{ - UserAgent: &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, - }, - Jar: jar, + to := NewSession("", "", toURL, "", &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, }, - URL: toURL, - Cache: make(map[string]CacheEntry), - } + Jar: jar, + }) resp, err := to.request("GET", "/api/1.2/user/current.json", nil) @@ -123,7 +133,7 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { jar.Save() fmt.Printf("Traffic Ops Session Resumed (%s)\n", resp.Status) - return &to, nil + return to, nil } // Deprecated: Login is deprecated, use LoginWithAgent instead. The `Login` function with its present signature will be removed in the next version and replaced with `Login(toURL string, toUser string, toPasswd string, insecure bool, userAgent string)`. The `LoginWithAgent` function will be removed the version after that. @@ -150,19 +160,12 @@ func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, return nil, err } - to := Session{ - UserAgent: &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, - }, - Jar: jar, + to := NewSession(toUser, toPasswd, toURL, userAgent, &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, }, - UserAgentStr: userAgent, - URL: toURL, - UserName: toUser, - Password: toPasswd, - Cache: make(map[string]CacheEntry), - } + Jar: jar, + }) path := "/api/1.2/user/login" resp, err := to.request("POST", path, credentials) @@ -191,7 +194,7 @@ func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, jar.Save() - return &to, nil + return to, nil } // request performs the actual HTTP request to Traffic Ops @@ -257,6 +260,23 @@ func StringToCacheHitStatus(s string) CacheHitStatus { } } +// setCache Sets the given cache key and value. This is threadsafe for multiple goroutines. +func (to *Session) setCache(path string, entry CacheEntry) { + to.cacheMutex.Lock() + defer to.cacheMutex.Unlock() + to.cache[path] = entry +} + +// getCache gets the cache value at the given key, or false if it doesn't exist. This is threadsafe for multiple goroutines. +func (to *Session) getCache(path string) (CacheEntry, bool) { + to.cacheMutex.RLock() + defer to.cacheMutex.RUnlock() + cacheEntry, ok := to.cache[path] + return cacheEntry, ok +} + +//if cacheEntry, ok := to.Cache[path]; ok { + // getBytesWithTTL - get the path, and cache in the session // return from cache is found and the ttl isn't expired, otherwise get it and // store it in cache @@ -265,7 +285,7 @@ func (to *Session) getBytesWithTTL(path string, ttl int64) ([]byte, CacheHitStat var err error var cacheHitStatus CacheHitStatus getFresh := false - if cacheEntry, ok := to.Cache[path]; ok { + if cacheEntry, ok := to.getCache(path); ok { if cacheEntry.Entered > time.Now().Unix()-ttl { cacheHitStatus = CacheHitStatusHit body = cacheEntry.Bytes @@ -274,7 +294,6 @@ func (to *Session) getBytesWithTTL(path string, ttl int64) ([]byte, CacheHitStat getFresh = true } } else { - to.Cache = make(map[string]CacheEntry) cacheHitStatus = CacheHitStatusMiss getFresh = true } @@ -289,7 +308,7 @@ func (to *Session) getBytesWithTTL(path string, ttl int64) ([]byte, CacheHitStat Entered: time.Now().Unix(), Bytes: body, } - to.Cache[path] = newEntry + to.setCache(path, newEntry) } return body, cacheHitStatus, nil From 4ca9c830cfe3fc3e8bff993028034d625d2cb762 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Wed, 15 Feb 2017 11:33:32 -0700 Subject: [PATCH 07/15] Change TM2 TO wrapper to assume thread safety Changes trafficopswrapper to assume the Traffic Ops Session requests are threadsafe, and only lock the Session pointer-pointer. This should drastically reduce mutex contention for TO client requests. --- .../trafficopswrapper/trafficopswrapper.go | 108 +++++++++--------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index 0bbc483146..8acf61dcda 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -44,62 +44,72 @@ type ITrafficOpsSession interface { var ErrNilSession = fmt.Errorf("nil session") -func (s TrafficOpsSessionThreadsafe) URL() (string, error) { +// TrafficOpsSessionThreadsafe provides access to the Traffic Ops client safe for multiple goroutines. This fulfills the ITrafficOpsSession interface. +type TrafficOpsSessionThreadsafe struct { + session **to.Session // pointer-to-pointer, because we're given a pointer from the Traffic Ops package, and we don't want to copy it. + m *sync.Mutex +} + +// NewTrafficOpsSessionThreadsafe returns a new threadsafe TrafficOpsSessionThreadsafe wrapping the given `Session`. +func NewTrafficOpsSessionThreadsafe(s *to.Session) TrafficOpsSessionThreadsafe { + return TrafficOpsSessionThreadsafe{&s, &sync.Mutex{}} +} + +// Set sets the internal Traffic Ops session. This is safe for multiple goroutines, being aware they will race. +func (s TrafficOpsSessionThreadsafe) Set(session *to.Session) { s.m.Lock() defer s.m.Unlock() - if s.session == nil || *s.session == nil { - return "", ErrNilSession - } - url := (*s.session).URL - return url, nil + *s.session = session } -func (s TrafficOpsSessionThreadsafe) User() (string, error) { +// getThreadsafeSession is used internally to get a copy of the session pointer, or nil if it doesn't exist. This should not be used outside TrafficOpsSessionThreadsafe, and never stored, because part of the purpose of TrafficOpsSessionThreadsafe is to store a pointer to the Session pointer, so it can be updated by one goroutine and immediately used by another. This should only be called immediately before using the session, since someone else may update it concurrently. +func (s TrafficOpsSessionThreadsafe) get() *to.Session { s.m.Lock() defer s.m.Unlock() if s.session == nil || *s.session == nil { - return "", ErrNilSession + return nil } - user := (*s.session).UserName - return user, nil + return *s.session } -// TrafficOpsSessionThreadsafe provides access to the Traffic Ops client safe for multiple goroutines. This fulfills the ITrafficOpsSession interface. -type TrafficOpsSessionThreadsafe struct { - session **to.Session // pointer-to-pointer, because we're given a pointer from the Traffic Ops package, and we don't want to copy it. - m *sync.Mutex +func (s TrafficOpsSessionThreadsafe) URL() (string, error) { + ss := s.get() + if ss == nil { + return "", ErrNilSession + } + return ss.URL, nil } -// NewTrafficOpsSessionThreadsafe returns a new threadsafe TrafficOpsSessionThreadsafe wrapping the given `Session`. -func NewTrafficOpsSessionThreadsafe(s *to.Session) TrafficOpsSessionThreadsafe { - return TrafficOpsSessionThreadsafe{&s, &sync.Mutex{}} +func (s TrafficOpsSessionThreadsafe) User() (string, error) { + ss := s.get() + if ss == nil { + return "", ErrNilSession + } + return ss.UserName, nil } // CRConfigRaw returns the CRConfig from the Traffic Ops. This is safe for multiple goroutines. func (s TrafficOpsSessionThreadsafe) CRConfigRaw(cdn string) ([]byte, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - b, _, e := (*s.session).GetCRConfig(cdn) + b, _, e := ss.GetCRConfig(cdn) return b, e } // TrafficMonitorConfigMapRaw returns the Traffic Monitor config map from the Traffic Ops, directly from the monitoring.json endpoint. This is not usually what is needed, rather monitoring needs the snapshotted CRConfig data, which is filled in by `TrafficMonitorConfigMap`. This is safe for multiple goroutines. -func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMapRaw(cdn string) (*to.TrafficMonitorConfigMap, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { +func (s TrafficOpsSessionThreadsafe) trafficMonitorConfigMapRaw(cdn string) (*to.TrafficMonitorConfigMap, error) { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - d, e := (*s.session).TrafficMonitorConfigMap(cdn) - return d, e + return ss.TrafficMonitorConfigMap(cdn) } // TrafficMonitorConfigMap returns the Traffic Monitor config map from the Traffic Ops. This is safe for multiple goroutines. func (s TrafficOpsSessionThreadsafe) TrafficMonitorConfigMap(cdn string) (*to.TrafficMonitorConfigMap, error) { - mc, err := s.TrafficMonitorConfigMapRaw(cdn) + mc, err := s.trafficMonitorConfigMapRaw(cdn) if err != nil { return nil, fmt.Errorf("getting monitor config map: %v", err) } @@ -177,54 +187,42 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig return mc, nil } -// Set sets the internal Traffic Ops session. This is safe for multiple goroutines, being aware they will race. -func (s TrafficOpsSessionThreadsafe) Set(session *to.Session) { - s.m.Lock() - defer s.m.Unlock() - *s.session = session -} - func (s TrafficOpsSessionThreadsafe) Servers() ([]to.Server, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - return (*s.session).Servers() + return ss.Servers() } func (s TrafficOpsSessionThreadsafe) Profiles() ([]to.Profile, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - return (*s.session).Profiles() + return ss.Profiles() } func (s TrafficOpsSessionThreadsafe) Parameters(profileName string) ([]to.Parameter, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - return (*s.session).Parameters(profileName) + return ss.Parameters(profileName) } func (s TrafficOpsSessionThreadsafe) DeliveryServices() ([]to.DeliveryService, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - return (*s.session).DeliveryServices() + return ss.DeliveryServices() } func (s TrafficOpsSessionThreadsafe) CacheGroups() ([]to.CacheGroup, error) { - s.m.Lock() - defer s.m.Unlock() - if s.session == nil || *s.session == nil { + ss := s.get() + if ss == nil { return nil, ErrNilSession } - return (*s.session).CacheGroups() + return ss.CacheGroups() } From 2503a7103f892fccd40fe735c2c2f997bba58ccc Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Wed, 15 Feb 2017 13:58:04 -0700 Subject: [PATCH 08/15] Fix TM2 monitoring for missing values --- .../trafficopswrapper/trafficopswrapper.go | 76 ++++++++++++++----- .../traffic_monitor/version.go | 2 +- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index 8acf61dcda..cd80e2bfb4 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -136,35 +136,69 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig // Dump the "live" monitoring.json servers, and populate with the "snapshotted" CRConfig mc.TrafficServer = map[string]to.TrafficServer{} for name, srv := range crConfig.ContentServers { - mc.TrafficServer[name] = to.TrafficServer{ - Profile: *srv.Profile, - IP: *srv.Ip, - Status: string(*srv.Status), - CacheGroup: *srv.CacheGroup, - IP6: *srv.Ip6, - Port: *srv.Port, - HostName: name, - FQDN: *srv.Fqdn, - InterfaceName: *srv.InterfaceName, - Type: *srv.ServerType, - HashID: *srv.HashId, + s := to.TrafficServer{} + if srv.Profile != nil { + s.Profile = *srv.Profile } + if srv.Ip != nil { + s.IP = *srv.Ip + } + if srv.Status != nil { + s.Status = string(*srv.Status) + } + if srv.CacheGroup != nil { + s.CacheGroup = *srv.CacheGroup + } + if srv.Ip6 != nil { + s.IP6 = *srv.Ip6 + } + if srv.Port != nil { + s.Port = *srv.Port + } + s.HostName = name + if srv.Fqdn != nil { + s.FQDN = *srv.Fqdn + } + if srv.InterfaceName != nil { + s.InterfaceName = *srv.InterfaceName + } + if srv.ServerType != nil { + s.Type = *srv.ServerType + } + if srv.HashId != nil { + s.HashID = *srv.HashId + } + mc.TrafficServer[name] = s } // Dump the "live" monitoring.json monitors, and populate with the "snapshotted" CRConfig mc.TrafficMonitor = map[string]to.TrafficMonitor{} for name, mon := range crConfig.Monitors { // monitorProfile = *mon.Profile - mc.TrafficMonitor[name] = to.TrafficMonitor{ - Port: *mon.Port, - IP6: *mon.IP6, - IP: *mon.IP, - HostName: name, - FQDN: *mon.FQDN, - Profile: *mon.Profile, - Location: *mon.Location, - Status: string(*mon.Status), + m := to.TrafficMonitor{} + if mon.Port != nil { + m.Port = *mon.Port + } + if mon.IP6 != nil { + m.IP6 = *mon.IP6 + } + if mon.IP != nil { + m.IP = *mon.IP + } + m.HostName = name + if mon.FQDN != nil { + m.FQDN = *mon.FQDN + } + if mon.Profile != nil { + m.Profile = *mon.Profile + } + if mon.Location != nil { + m.Location = *mon.Location + } + if mon.Status != nil { + m.Status = string(*mon.Status) } + mc.TrafficMonitor[name] = m } // Dump the "live" monitoring.json DeliveryServices, and populate with the "snapshotted" CRConfig diff --git a/traffic_monitor_golang/traffic_monitor/version.go b/traffic_monitor_golang/traffic_monitor/version.go index d33edc55ec..f75bdac6f0 100644 --- a/traffic_monitor_golang/traffic_monitor/version.go +++ b/traffic_monitor_golang/traffic_monitor/version.go @@ -20,4 +20,4 @@ package main */ // Version is the current version of the app, in string form. -var Version = "2.0.2" +var Version = "2.0.3" From 3390821157f4880136d0816a8d22923df3563968 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Wed, 15 Feb 2017 15:52:02 -0700 Subject: [PATCH 09/15] Change TM2 DisabledLocations to cachegroups --- .../traffic_monitor/health/cache.go | 47 ++++++++++++++++--- .../traffic_monitor/manager/monitorconfig.go | 2 +- .../traffic_monitor/manager/statecombiner.go | 18 +++---- .../traffic_monitor/peer/crstates.go | 4 +- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/health/cache.go b/traffic_monitor_golang/traffic_monitor/health/cache.go index 94be59fe48..806cad7cad 100644 --- a/traffic_monitor_golang/traffic_monitor/health/cache.go +++ b/traffic_monitor_golang/traffic_monitor/health/cache.go @@ -203,7 +203,7 @@ func CalcAvailability(results []cache.Result, pollerName string, statResultHisto localStates.SetCache(result.ID, peer.IsAvailable{IsAvailable: isAvailable}) } - calculateDeliveryServiceState(toData.DeliveryServiceServers, localStates) + calculateDeliveryServiceState(toData.DeliveryServiceServers, localStates, toData) localCacheStatusThreadsafe.Set(localCacheStatuses) } @@ -253,17 +253,52 @@ func eventDesc(status enum.CacheStatus, message string) string { } //calculateDeliveryServiceState calculates the state of delivery services from the new cache state data `cacheState` and the CRConfig data `deliveryServiceServers` and puts the calculated state in the outparam `deliveryServiceStates` -func calculateDeliveryServiceState(deliveryServiceServers map[enum.DeliveryServiceName][]enum.CacheName, states peer.CRStatesThreadsafe) { +func calculateDeliveryServiceState(deliveryServiceServers map[enum.DeliveryServiceName][]enum.CacheName, states peer.CRStatesThreadsafe, toData todata.TOData) { + cacheStates := states.GetCaches() // map[enum.CacheName]IsAvailable + deliveryServices := states.GetDeliveryServices() for deliveryServiceName, deliveryServiceState := range deliveryServices { if _, ok := deliveryServiceServers[deliveryServiceName]; !ok { log.Infof("CRConfig does not have delivery service %s, but traffic monitor poller does; skipping\n", deliveryServiceName) continue } - deliveryServiceState.DisabledLocations = []enum.CacheName{} // it's important this isn't nil, so it serialises to the JSON `[]` instead of `null` - for _, server := range deliveryServiceServers[deliveryServiceName] { - deliveryServiceState.DisabledLocations = append(deliveryServiceState.DisabledLocations, server) - } + deliveryServiceState.DisabledLocations = getDisabledLocations(deliveryServiceName, toData.DeliveryServiceServers[deliveryServiceName], cacheStates, toData.ServerCachegroups) states.SetDeliveryService(deliveryServiceName, deliveryServiceState) } } + +func getDisabledLocations(deliveryService enum.DeliveryServiceName, deliveryServiceServers []enum.CacheName, cacheStates map[enum.CacheName]peer.IsAvailable, serverCacheGroups map[enum.CacheName]enum.CacheGroupName) []enum.CacheGroupName { + disabledLocations := []enum.CacheGroupName{} // it's important this isn't nil, so it serialises to the JSON `[]` instead of `null` + dsCacheStates := getDeliveryServiceCacheAvailability(cacheStates, deliveryServiceServers) + dsCachegroupsAvailable := getDeliveryServiceCachegroupAvailability(dsCacheStates, serverCacheGroups) + for cg, avail := range dsCachegroupsAvailable { + if avail { + continue + } + disabledLocations = append(disabledLocations, cg) + } + return disabledLocations +} + +func getDeliveryServiceCacheAvailability(cacheStates map[enum.CacheName]peer.IsAvailable, deliveryServiceServers []enum.CacheName) map[enum.CacheName]peer.IsAvailable { + dsCacheStates := map[enum.CacheName]peer.IsAvailable{} + for _, server := range deliveryServiceServers { + dsCacheStates[server] = cacheStates[server] + } + return dsCacheStates +} + +func getDeliveryServiceCachegroupAvailability(dsCacheStates map[enum.CacheName]peer.IsAvailable, serverCachegroups map[enum.CacheName]enum.CacheGroupName) map[enum.CacheGroupName]bool { + cgAvail := map[enum.CacheGroupName]bool{} + for cache, available := range dsCacheStates { + cg, ok := serverCachegroups[cache] + if !ok { + log.Errorf("cache %v not found in cachegroups!") + continue + } + if _, ok := cgAvail[cg]; !ok || available.IsAvailable { + cgAvail[cg] = available.IsAvailable + } + } + return cgAvail +} diff --git a/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go b/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go index d7df9813fd..7edc4e004a 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go +++ b/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go @@ -275,7 +275,7 @@ func monitorConfigListen( for _, ds := range monitorConfig.DeliveryService { // since caches default to unavailable, also default DS false if _, exists := localStates.GetDeliveryService(enum.DeliveryServiceName(ds.XMLID)); !exists { - localStates.SetDeliveryService(enum.DeliveryServiceName(ds.XMLID), peer.Deliveryservice{IsAvailable: false, DisabledLocations: []enum.CacheName{}}) // important to initialize DisabledLocations, so JSON is `[]` not `null` + localStates.SetDeliveryService(enum.DeliveryServiceName(ds.XMLID), peer.Deliveryservice{IsAvailable: false, DisabledLocations: []enum.CacheGroupName{}}) // important to initialize DisabledLocations, so JSON is `[]` not `null` } } for ds := range localStates.GetDeliveryServices() { diff --git a/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go b/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go index 72b2e0a92d..2aa1dcc9b6 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go +++ b/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go @@ -118,7 +118,7 @@ func combineCacheState(cacheName enum.CacheName, localCacheState peer.IsAvailabl } func combineDSState(deliveryServiceName enum.DeliveryServiceName, localDeliveryService peer.Deliveryservice, events health.ThreadsafeEvents, peerOptimistic bool, peerStates peer.CRStatesPeersThreadsafe, localStates peer.Crstates, combinedStates peer.CRStatesThreadsafe, overrideMap map[enum.CacheName]bool, toData todata.TOData) { - deliveryService := peer.Deliveryservice{IsAvailable: false, DisabledLocations: []enum.CacheName{}} // important to initialize DisabledLocations, so JSON is `[]` not `null` + deliveryService := peer.Deliveryservice{IsAvailable: false, DisabledLocations: []enum.CacheGroupName{}} // important to initialize DisabledLocations, so JSON is `[]` not `null` if localDeliveryService.IsAvailable { deliveryService.IsAvailable = true } @@ -148,18 +148,18 @@ func combineCrStates(events health.ThreadsafeEvents, peerOptimistic bool, peerSt } // CacheNameSlice is a slice of cache names, which fulfills the `sort.Interface` interface. -type CacheNameSlice []enum.CacheName +type CacheGroupNameSlice []enum.CacheGroupName -func (p CacheNameSlice) Len() int { return len(p) } -func (p CacheNameSlice) Less(i, j int) bool { return p[i] < p[j] } -func (p CacheNameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p CacheGroupNameSlice) Len() int { return len(p) } +func (p CacheGroupNameSlice) Less(i, j int) bool { return p[i] < p[j] } +func (p CacheGroupNameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // intersection returns strings in both a and b. // Note this modifies a and b. Specifically, it sorts them. If that isn't acceptable, pass copies of your real data. -func intersection(a []enum.CacheName, b []enum.CacheName) []enum.CacheName { - sort.Sort(CacheNameSlice(a)) - sort.Sort(CacheNameSlice(b)) - c := []enum.CacheName{} // important to initialize, so JSON is `[]` not `null` +func intersection(a []enum.CacheGroupName, b []enum.CacheGroupName) []enum.CacheGroupName { + sort.Sort(CacheGroupNameSlice(a)) + sort.Sort(CacheGroupNameSlice(b)) + c := []enum.CacheGroupName{} // important to initialize, so JSON is `[]` not `null` for _, s := range a { i := sort.Search(len(b), func(i int) bool { return b[i] >= s }) if i < len(b) && b[i] == s { diff --git a/traffic_monitor_golang/traffic_monitor/peer/crstates.go b/traffic_monitor_golang/traffic_monitor/peer/crstates.go index ba42d7e149..d6867e9c9e 100644 --- a/traffic_monitor_golang/traffic_monitor/peer/crstates.go +++ b/traffic_monitor_golang/traffic_monitor/peer/crstates.go @@ -78,8 +78,8 @@ type IsAvailable struct { // Deliveryservice contains data about the availability of a particular delivery service, and which caches in that delivery service have been marked as unavailable. type Deliveryservice struct { - DisabledLocations []enum.CacheName `json:"disabledLocations"` - IsAvailable bool `json:"isAvailable"` + DisabledLocations []enum.CacheGroupName `json:"disabledLocations"` + IsAvailable bool `json:"isAvailable"` } // CrstatesUnMarshall takes bytes of a JSON string, and unmarshals them into a Crstates object. From d74600d2ef26cca504e961331befe0edb52176cb Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Wed, 15 Feb 2017 19:28:23 -0700 Subject: [PATCH 10/15] Fix TM2 to delete removed caches from CrStates Fixes Traffic Monitor 2.0 to remove caches which have either been deleted or marked OFFLINE in the monitor config (monitoring.json plus CRConfig) from the localStates and combinedStates, which in turn removes them from `CrStates` and other endpoints. --- .../traffic_monitor/manager/monitorconfig.go | 4 ++-- .../traffic_monitor/manager/statecombiner.go | 15 ++++++++++++++- .../traffic_monitor/peer/crstates.go | 11 ++++++++++- traffic_monitor_golang/traffic_monitor/version.go | 2 +- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go b/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go index 7edc4e004a..82dd15c7f3 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go +++ b/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go @@ -211,7 +211,7 @@ func monitorConfigListen( srvStatus := enum.CacheStatusFromString(srv.Status) if srvStatus == enum.CacheStatusOnline { - localStates.SetCache(cacheName, peer.IsAvailable{IsAvailable: true}) + localStates.AddCache(cacheName, peer.IsAvailable{IsAvailable: true}) continue } if srvStatus == enum.CacheStatusOffline { @@ -219,7 +219,7 @@ func monitorConfigListen( } // seed states with available = false until our polling cycle picks up a result if _, exists := localStates.GetCache(cacheName); !exists { - localStates.SetCache(cacheName, peer.IsAvailable{IsAvailable: false}) + localStates.AddCache(cacheName, peer.IsAvailable{IsAvailable: false}) } url := monitorConfig.Profile[srv.Profile].Parameters.HealthPollingURL diff --git a/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go b/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go index 2aa1dcc9b6..97f21e7f2d 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go +++ b/traffic_monitor_golang/traffic_monitor/manager/statecombiner.go @@ -114,7 +114,7 @@ func combineCacheState(cacheName enum.CacheName, localCacheState peer.IsAvailabl events.Add(health.Event{Time: health.Time(time.Now()), Description: fmt.Sprintf("Health protocol override condition %s", overrideCondition), Name: cacheName.String(), Hostname: cacheName.String(), Type: toData.ServerTypes[cacheName].String(), Available: available}) } - combinedStates.SetCache(cacheName, peer.IsAvailable{IsAvailable: available}) + combinedStates.AddCache(cacheName, peer.IsAvailable{IsAvailable: available}) } func combineDSState(deliveryServiceName enum.DeliveryServiceName, localDeliveryService peer.Deliveryservice, events health.ThreadsafeEvents, peerOptimistic bool, peerStates peer.CRStatesPeersThreadsafe, localStates peer.Crstates, combinedStates peer.CRStatesThreadsafe, overrideMap map[enum.CacheName]bool, toData todata.TOData) { @@ -138,13 +138,26 @@ func combineDSState(deliveryServiceName enum.DeliveryServiceName, localDeliveryS combinedStates.SetDeliveryService(deliveryServiceName, deliveryService) } +// pruneCombinedCaches deletes caches in combined states which have been removed from localStates. +func pruneCombinedCaches(combinedStates peer.CRStatesThreadsafe, localStates peer.Crstates) { + combinedCaches := combinedStates.GetCaches() + for cacheName, _ := range combinedCaches { + if _, ok := localStates.Caches[cacheName]; !ok { + combinedStates.DeleteCache(cacheName) + } + } +} + func combineCrStates(events health.ThreadsafeEvents, peerOptimistic bool, peerStates peer.CRStatesPeersThreadsafe, localStates peer.Crstates, combinedStates peer.CRStatesThreadsafe, overrideMap map[enum.CacheName]bool, toData todata.TOData) { for cacheName, localCacheState := range localStates.Caches { // localStates gets pruned when servers are disabled, it's the source of truth combineCacheState(cacheName, localCacheState, events, peerOptimistic, peerStates, localStates, combinedStates, overrideMap, toData) } + for deliveryServiceName, localDeliveryService := range localStates.Deliveryservice { combineDSState(deliveryServiceName, localDeliveryService, events, peerOptimistic, peerStates, localStates, combinedStates, overrideMap, toData) } + + pruneCombinedCaches(combinedStates, localStates) } // CacheNameSlice is a slice of cache names, which fulfills the `sort.Interface` interface. diff --git a/traffic_monitor_golang/traffic_monitor/peer/crstates.go b/traffic_monitor_golang/traffic_monitor/peer/crstates.go index d6867e9c9e..ef0133eac3 100644 --- a/traffic_monitor_golang/traffic_monitor/peer/crstates.go +++ b/traffic_monitor_golang/traffic_monitor/peer/crstates.go @@ -145,8 +145,17 @@ func (t *CRStatesThreadsafe) GetDeliveryService(name enum.DeliveryServiceName) ( return } -// SetCache sets the internal availability data for a particular cache. +// SetCache sets the internal availability data for a particular cache. It does NOT set data if the cache doesn't already exist. By adding newly received caches with `AddCache`, this allows easily avoiding a race condition when an in-flight poller tries to set a cache which has been removed. func (t *CRStatesThreadsafe) SetCache(cacheName enum.CacheName, available IsAvailable) { + t.m.Lock() + if _, ok := t.crStates.Caches[cacheName]; ok { + t.crStates.Caches[cacheName] = available + } + t.m.Unlock() +} + +// AddCache adds the internal availability data for a particular cache. +func (t *CRStatesThreadsafe) AddCache(cacheName enum.CacheName, available IsAvailable) { t.m.Lock() t.crStates.Caches[cacheName] = available t.m.Unlock() diff --git a/traffic_monitor_golang/traffic_monitor/version.go b/traffic_monitor_golang/traffic_monitor/version.go index f75bdac6f0..31b0e57697 100644 --- a/traffic_monitor_golang/traffic_monitor/version.go +++ b/traffic_monitor_golang/traffic_monitor/version.go @@ -20,4 +20,4 @@ package main */ // Version is the current version of the app, in string form. -var Version = "2.0.3" +var Version = "2.0.4" From baf3e40bbec71d019951bd5968e4cb40307d504e Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Thu, 16 Feb 2017 10:20:01 -0700 Subject: [PATCH 11/15] Add TM2 MonitorConfig err logs, API endpoint --- .../traffic_monitor/manager/datarequest.go | 7 ++++ .../traffic_monitor/static/index.html | 1 + .../trafficopswrapper/trafficopswrapper.go | 35 +++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/traffic_monitor_golang/traffic_monitor/manager/datarequest.go b/traffic_monitor_golang/traffic_monitor/manager/datarequest.go index da552dd30c..fd6ad9e014 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/datarequest.go +++ b/traffic_monitor_golang/traffic_monitor/manager/datarequest.go @@ -777,10 +777,17 @@ func MakeDispatchMap( "/api/bandwidth-capacity-kbps": wrap(WrapBytes(func() []byte { return srvAPIBandwidthCapacityKbps(statMaxKbpses) }, ContentTypeJSON)), + "/api/monitor-config": wrap(WrapErr(errorCount, func() ([]byte, error) { + return srvMonitorConfig(monitorConfig) + }, ContentTypeJSON)), } return addTrailingSlashEndpoints(dispatchMap) } +func srvMonitorConfig(mcThs TrafficMonitorConfigMapThreadsafe) ([]byte, error) { + return json.Marshal(mcThs.Get()) +} + // latestResultInfoTimeMS returns the length of time in milliseconds that it took to request the most recent non-errored result info. func latestResultInfoTimeMS(cacheName enum.CacheName, history cache.ResultInfoHistory) (int64, error) { results, ok := history[cacheName] diff --git a/traffic_monitor_golang/traffic_monitor/static/index.html b/traffic_monitor_golang/traffic_monitor/static/index.html index 66cd03b99d..f0a9651851 100644 --- a/traffic_monitor_golang/traffic_monitor/static/index.html +++ b/traffic_monitor_golang/traffic_monitor/static/index.html @@ -498,6 +498,7 @@
  • /api/cache-statuses
  • /api/bandwidth-kbps
  • /api/bandwidth-capacity-kbps
    • +
  • /api/monitor-config
    • diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index cd80e2bfb4..b86d844e13 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -24,6 +24,7 @@ import ( "fmt" "sync" + "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/common/log" "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/crconfig" to "github.com/apache/incubator-trafficcontrol/traffic_ops/client" ) @@ -139,34 +140,54 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig s := to.TrafficServer{} if srv.Profile != nil { s.Profile = *srv.Profile + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing Profile field\n", name) } if srv.Ip != nil { s.IP = *srv.Ip + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing IP field\n", name) } if srv.Status != nil { s.Status = string(*srv.Status) + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing Status field\n", name) } if srv.CacheGroup != nil { s.CacheGroup = *srv.CacheGroup + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing CacheGroup field\n", name) } if srv.Ip6 != nil { s.IP6 = *srv.Ip6 + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing IP6 field\n", name) } if srv.Port != nil { s.Port = *srv.Port + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing Port field\n", name) } s.HostName = name if srv.Fqdn != nil { s.FQDN = *srv.Fqdn + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing FQDN field\n", name) } if srv.InterfaceName != nil { s.InterfaceName = *srv.InterfaceName + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing InterfaceName field\n", name) } if srv.ServerType != nil { s.Type = *srv.ServerType + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing Type field\n", name) } if srv.HashId != nil { s.HashID = *srv.HashId + } else { + log.Warnf("Creating monitor config: CRConfig server %s missing HashId field\n", name) } mc.TrafficServer[name] = s } @@ -178,25 +199,39 @@ func CreateMonitorConfig(crConfig crconfig.CRConfig, mc *to.TrafficMonitorConfig m := to.TrafficMonitor{} if mon.Port != nil { m.Port = *mon.Port + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing Port field\n", name) } if mon.IP6 != nil { m.IP6 = *mon.IP6 + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing IP6 field\n", name) } if mon.IP != nil { m.IP = *mon.IP + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing IP field\n", name) } m.HostName = name if mon.FQDN != nil { m.FQDN = *mon.FQDN + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing FQDN field\n", name) } if mon.Profile != nil { m.Profile = *mon.Profile + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing Profile field\n", name) } if mon.Location != nil { m.Location = *mon.Location + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing Location field\n", name) } if mon.Status != nil { m.Status = string(*mon.Status) + } else { + log.Warnf("Creating monitor config: CRConfig monitor %s missing Status field\n", name) } mc.TrafficMonitor[name] = m } From 3014895f59bd9961b36c83b18f4ad6a434b3c6f3 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Thu, 16 Feb 2017 13:06:50 -0700 Subject: [PATCH 12/15] Add TO client noCache option, change TM2 to use --- .../traffic_monitor/manager/opsconfig.go | 3 ++- traffic_ops/client/traffic_ops.go | 15 ++++++++++----- traffic_stats/traffic_stats.go | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go b/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go index 111452cc79..dba64ca31a 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go +++ b/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go @@ -159,7 +159,8 @@ func StartOpsConfigManager( continue } - realToSession, err := to.LoginWithAgent(newOpsConfig.Url, newOpsConfig.Username, newOpsConfig.Password, newOpsConfig.Insecure, staticAppData.UserAgent) + useCache := false // TODO add config + realToSession, err := to.LoginWithAgent(newOpsConfig.Url, newOpsConfig.Username, newOpsConfig.Password, newOpsConfig.Insecure, staticAppData.UserAgent, useCache) if err != nil { handleErr(fmt.Errorf("MonitorConfigPoller: error instantiating Session with traffic_ops: %s\n", err)) continue diff --git a/traffic_ops/client/traffic_ops.go b/traffic_ops/client/traffic_ops.go index bfb53097c3..d02ffefd5b 100644 --- a/traffic_ops/client/traffic_ops.go +++ b/traffic_ops/client/traffic_ops.go @@ -38,10 +38,11 @@ type Session struct { UserAgent *http.Client cache map[string]CacheEntry cacheMutex *sync.RWMutex + useCache bool UserAgentStr string } -func NewSession(user, password, url, userAgent string, client *http.Client) *Session { +func NewSession(user, password, url, userAgent string, client *http.Client, useCache bool) *Session { return &Session{ UserName: user, Password: password, @@ -49,6 +50,7 @@ func NewSession(user, password, url, userAgent string, client *http.Client) *Ses UserAgent: client, cache: map[string]CacheEntry{}, cacheMutex: &sync.RWMutex{}, + useCache: useCache, UserAgentStr: userAgent, } } @@ -122,7 +124,7 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, }, Jar: jar, - }) + }, false) resp, err := to.request("GET", "/api/1.2/user/current.json", nil) @@ -138,14 +140,14 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { // Deprecated: Login is deprecated, use LoginWithAgent instead. The `Login` function with its present signature will be removed in the next version and replaced with `Login(toURL string, toUser string, toPasswd string, insecure bool, userAgent string)`. The `LoginWithAgent` function will be removed the version after that. func Login(toURL string, toUser string, toPasswd string, insecure bool) (*Session, error) { - return LoginWithAgent(toURL, toUser, toPasswd, insecure, "traffic-ops-client") // TODO add version + return LoginWithAgent(toURL, toUser, toPasswd, insecure, "traffic-ops-client", false) // TODO add version } // Login to traffic_ops, the response should set the cookie for this session // automatically. Start with // to := traffic_ops.Login("user", "passwd", true) // subsequent calls like to.GetData("datadeliveryservice") will be authenticated. -func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, userAgent string) (*Session, error) { +func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, userAgent string, useCache bool) (*Session, error) { credentials, err := loginCreds(toUser, toPasswd) if err != nil { return nil, err @@ -165,7 +167,7 @@ func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, }, Jar: jar, - }) + }, useCache) path := "/api/1.2/user/login" resp, err := to.request("POST", path, credentials) @@ -262,6 +264,9 @@ func StringToCacheHitStatus(s string) CacheHitStatus { // setCache Sets the given cache key and value. This is threadsafe for multiple goroutines. func (to *Session) setCache(path string, entry CacheEntry) { + if !to.useCache { + return + } to.cacheMutex.Lock() defer to.cacheMutex.Unlock() to.cache[path] = entry diff --git a/traffic_stats/traffic_stats.go b/traffic_stats/traffic_stats.go index 3d474c0a58..5bf9c44ad0 100644 --- a/traffic_stats/traffic_stats.go +++ b/traffic_stats/traffic_stats.go @@ -414,7 +414,7 @@ func queryDB(con influx.Client, cmd string, database string) (res []influx.Resul } func writeSummaryStats(config StartupConfig, statsSummary traffic_ops.StatsSummary) { - to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent) + to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false) if err != nil { newErr := fmt.Errorf("Could not store summary stats! Error logging in to %v: %v", config.ToURL, err) log.Error(newErr) @@ -428,7 +428,7 @@ func writeSummaryStats(config StartupConfig, statsSummary traffic_ops.StatsSumma func getToData(config StartupConfig, init bool, configChan chan RunningConfig) { var runningConfig RunningConfig - to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent) + to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false) if err != nil { msg := fmt.Sprintf("Error logging in to %v: %v", config.ToURL, err) if init { From a44bc00c05a7d3103dbf3431be07a7eea11d808b Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Thu, 16 Feb 2017 14:57:29 -0700 Subject: [PATCH 13/15] Add TM2 CRConfig API caching --- .../traffic_monitor/manager/datarequest.go | 21 +++++-- .../trafficopswrapper/trafficopswrapper.go | 56 +++++++++++++++++-- .../traffic_monitor/version.go | 2 +- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/manager/datarequest.go b/traffic_monitor_golang/traffic_monitor/manager/datarequest.go index fd6ad9e014..86ebe01704 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/datarequest.go +++ b/traffic_monitor_golang/traffic_monitor/manager/datarequest.go @@ -547,15 +547,26 @@ func WrapParams(f SrvFunc, contentType string) http.HandlerFunc { } } -func srvTRConfig(opsConfig OpsConfigThreadsafe, toSession towrap.ITrafficOpsSession) ([]byte, error) { +func WrapAgeErr(errorCount threadsafe.Uint, f func() ([]byte, time.Time, error), contentType string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + bytes, contentTime, err := f() + _, code := WrapErrCode(errorCount, r.URL.EscapedPath(), bytes, err) + w.Header().Set("Content-Type", contentType) + w.Header().Set("Age", fmt.Sprintf("%.0f", time.Since(contentTime).Seconds())) + w.WriteHeader(code) + log.Write(w, bytes, r.URL.EscapedPath()) + } +} + +func srvTRConfig(opsConfig OpsConfigThreadsafe, toSession towrap.ITrafficOpsSession) ([]byte, time.Time, error) { cdnName := opsConfig.Get().CdnName if toSession == nil { - return nil, fmt.Errorf("Unable to connect to Traffic Ops") + return nil, time.Time{}, fmt.Errorf("Unable to connect to Traffic Ops") } if cdnName == "" { - return nil, fmt.Errorf("No CDN Configured") + return nil, time.Time{}, fmt.Errorf("No CDN Configured") } - return toSession.CRConfigRaw(cdnName) + return toSession.LastCRConfig(cdnName) } func srvTRState(params url.Values, localStates peer.CRStatesThreadsafe, combinedStates peer.CRStatesThreadsafe) ([]byte, error) { @@ -725,7 +736,7 @@ func MakeDispatchMap( } dispatchMap := map[string]http.HandlerFunc{ - "/publish/CrConfig": wrap(WrapErr(errorCount, func() ([]byte, error) { + "/publish/CrConfig": wrap(WrapAgeErr(errorCount, func() ([]byte, time.Time, error) { return srvTRConfig(opsConfig, toSession) }, ContentTypeJSON)), "/publish/CrStates": wrap(WrapParams(func(params url.Values, path string) ([]byte, int) { diff --git a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go index b86d844e13..a9d9e07e6c 100644 --- a/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go +++ b/traffic_monitor_golang/traffic_monitor/trafficopswrapper/trafficopswrapper.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "sync" + "time" "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/common/log" "github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/traffic_monitor/crconfig" @@ -32,6 +33,7 @@ import ( // ITrafficOpsSession provides an interface to the Traffic Ops client, so it may be wrapped or mocked. type ITrafficOpsSession interface { CRConfigRaw(cdn string) ([]byte, error) + LastCRConfig(cdn string) ([]byte, time.Time, error) TrafficMonitorConfigMap(cdn string) (*to.TrafficMonitorConfigMap, error) Set(session *to.Session) URL() (string, error) @@ -45,15 +47,46 @@ type ITrafficOpsSession interface { var ErrNilSession = fmt.Errorf("nil session") +type ByteTime struct { + bytes []byte + time time.Time +} + +type ByteMapCache struct { + cache *map[string]ByteTime + m *sync.RWMutex +} + +func NewByteMapCache() ByteMapCache { + return ByteMapCache{m: &sync.RWMutex{}, cache: &map[string]ByteTime{}} +} + +func (c ByteMapCache) Set(key string, newBytes []byte) { + c.m.Lock() + defer c.m.Unlock() + (*c.cache)[key] = ByteTime{bytes: newBytes, time: time.Now()} +} + +func (c ByteMapCache) Get(key string) ([]byte, time.Time) { + c.m.RLock() + defer c.m.RUnlock() + if byteTime, ok := (*c.cache)[key]; !ok { + return nil, time.Time{} + } else { + return byteTime.bytes, byteTime.time + } +} + // TrafficOpsSessionThreadsafe provides access to the Traffic Ops client safe for multiple goroutines. This fulfills the ITrafficOpsSession interface. type TrafficOpsSessionThreadsafe struct { - session **to.Session // pointer-to-pointer, because we're given a pointer from the Traffic Ops package, and we don't want to copy it. - m *sync.Mutex + session **to.Session // pointer-to-pointer, because we're given a pointer from the Traffic Ops package, and we don't want to copy it. + m *sync.Mutex + lastCRConfig ByteMapCache } // NewTrafficOpsSessionThreadsafe returns a new threadsafe TrafficOpsSessionThreadsafe wrapping the given `Session`. func NewTrafficOpsSessionThreadsafe(s *to.Session) TrafficOpsSessionThreadsafe { - return TrafficOpsSessionThreadsafe{&s, &sync.Mutex{}} + return TrafficOpsSessionThreadsafe{session: &s, m: &sync.Mutex{}, lastCRConfig: NewByteMapCache()} } // Set sets the internal Traffic Ops session. This is safe for multiple goroutines, being aware they will race. @@ -95,8 +128,21 @@ func (s TrafficOpsSessionThreadsafe) CRConfigRaw(cdn string) ([]byte, error) { if ss == nil { return nil, ErrNilSession } - b, _, e := ss.GetCRConfig(cdn) - return b, e + b, _, err := ss.GetCRConfig(cdn) + if err == nil { + s.lastCRConfig.Set(cdn, b) + } + return b, err +} + +// LastCRConfig returns the last CRConfig requested from CRConfigRaw, and the time it was returned. This is designed to be used in conjunction with a poller which regularly calls CRConfigRaw. If no last CRConfig exists, because CRConfigRaw has never been called successfully, this calls CRConfigRaw once to try to get the CRConfig from Traffic Ops. +func (s TrafficOpsSessionThreadsafe) LastCRConfig(cdn string) ([]byte, time.Time, error) { + crConfig, crConfigTime := s.lastCRConfig.Get(cdn) + if crConfig == nil { + b, err := s.CRConfigRaw(cdn) + return b, time.Now(), err + } + return crConfig, crConfigTime, nil } // TrafficMonitorConfigMapRaw returns the Traffic Monitor config map from the Traffic Ops, directly from the monitoring.json endpoint. This is not usually what is needed, rather monitoring needs the snapshotted CRConfig data, which is filled in by `TrafficMonitorConfigMap`. This is safe for multiple goroutines. diff --git a/traffic_monitor_golang/traffic_monitor/version.go b/traffic_monitor_golang/traffic_monitor/version.go index 31b0e57697..31569de859 100644 --- a/traffic_monitor_golang/traffic_monitor/version.go +++ b/traffic_monitor_golang/traffic_monitor/version.go @@ -20,4 +20,4 @@ package main */ // Version is the current version of the app, in string form. -var Version = "2.0.4" +var Version = "2.0.5" From a129bca00985387af3ae75057c7b0159e3d6e6ff Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Fri, 17 Feb 2017 09:27:19 -0700 Subject: [PATCH 14/15] Add MonitorConfig poller, manager failure logs --- traffic_monitor_golang/common/poller/poller.go | 8 ++++++++ .../traffic_monitor/manager/monitorconfig.go | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/traffic_monitor_golang/common/poller/poller.go b/traffic_monitor_golang/common/poller/poller.go index 81eeb3ccbc..018e95cc14 100644 --- a/traffic_monitor_golang/common/poller/poller.go +++ b/traffic_monitor_golang/common/poller/poller.go @@ -120,6 +120,14 @@ func NewMonitorConfig(interval time.Duration) MonitorConfigPoller { func (p MonitorConfigPoller) Poll() { tick := time.NewTicker(p.Interval) defer tick.Stop() + defer func() { + if err := recover(); err != nil { + log.Errorf("MonitorConfigPoller panic: %v\n", err) + } else { + log.Errorf("MonitorConfigPoller failed without panic\n") + } + os.Exit(1) // The Monitor can't run without a MonitorConfigPoller + }() for { select { case opsConfig := <-p.OpsConfigChannel: diff --git a/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go b/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go index 82dd15c7f3..bb1c049864 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go +++ b/traffic_monitor_golang/traffic_monitor/manager/monitorconfig.go @@ -21,6 +21,7 @@ package manager import ( "fmt" + "os" "strings" "sync" "time" @@ -191,6 +192,14 @@ func monitorConfigListen( cfg config.Config, staticAppData StaticAppData, ) { + defer func() { + if err := recover(); err != nil { + log.Errorf("MonitorConfigManager panic: %v\n", err) + } else { + log.Errorf("MonitorConfigManager failed without panic\n") + } + os.Exit(1) // The Monitor can't run without a MonitorConfigManager + }() for monitorConfig := range monitorConfigPollChan { monitorConfigTS.Set(monitorConfig) healthURLs := map[string]poller.PollConfig{} From 5eea4959429779474b647a47b354fdc1776a819e Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Sat, 18 Feb 2017 14:57:50 -0700 Subject: [PATCH 15/15] Add TO client timeout parameter --- .../traffic_monitor/manager/opsconfig.go | 7 +++++-- traffic_ops/client/traffic_ops.go | 14 +++++++++----- traffic_stats/traffic_stats.go | 5 +++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go b/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go index dba64ca31a..6cbe709ac4 100644 --- a/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go +++ b/traffic_monitor_golang/traffic_monitor/manager/opsconfig.go @@ -159,8 +159,11 @@ func StartOpsConfigManager( continue } - useCache := false // TODO add config - realToSession, err := to.LoginWithAgent(newOpsConfig.Url, newOpsConfig.Username, newOpsConfig.Password, newOpsConfig.Insecure, staticAppData.UserAgent, useCache) + // TODO config? parameter? + useCache := false + trafficOpsRequestTimeout := time.Second * time.Duration(10) + + realToSession, err := to.LoginWithAgent(newOpsConfig.Url, newOpsConfig.Username, newOpsConfig.Password, newOpsConfig.Insecure, staticAppData.UserAgent, useCache, trafficOpsRequestTimeout) if err != nil { handleErr(fmt.Errorf("MonitorConfigPoller: error instantiating Session with traffic_ops: %s\n", err)) continue diff --git a/traffic_ops/client/traffic_ops.go b/traffic_ops/client/traffic_ops.go index d02ffefd5b..24508089b9 100644 --- a/traffic_ops/client/traffic_ops.go +++ b/traffic_ops/client/traffic_ops.go @@ -35,7 +35,7 @@ type Session struct { UserName string Password string URL string - UserAgent *http.Client + Client *http.Client cache map[string]CacheEntry cacheMutex *sync.RWMutex useCache bool @@ -47,7 +47,7 @@ func NewSession(user, password, url, userAgent string, client *http.Client, useC UserName: user, Password: password, URL: url, - UserAgent: client, + Client: client, cache: map[string]CacheEntry{}, cacheMutex: &sync.RWMutex{}, useCache: useCache, @@ -55,6 +55,8 @@ func NewSession(user, password, url, userAgent string, client *http.Client, useC } } +const DefaultTimeout = time.Second * time.Duration(30) + // HTTPError is returned on Update Session failure. type HTTPError struct { HTTPStatusCode int @@ -120,6 +122,7 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { } to := NewSession("", "", toURL, "", &http.Client{ + Timeout: DefaultTimeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, }, @@ -140,14 +143,14 @@ func ResumeSession(toURL string, insecure bool) (*Session, error) { // Deprecated: Login is deprecated, use LoginWithAgent instead. The `Login` function with its present signature will be removed in the next version and replaced with `Login(toURL string, toUser string, toPasswd string, insecure bool, userAgent string)`. The `LoginWithAgent` function will be removed the version after that. func Login(toURL string, toUser string, toPasswd string, insecure bool) (*Session, error) { - return LoginWithAgent(toURL, toUser, toPasswd, insecure, "traffic-ops-client", false) // TODO add version + return LoginWithAgent(toURL, toUser, toPasswd, insecure, "traffic-ops-client", false, DefaultTimeout) // TODO add UserAgent version } // Login to traffic_ops, the response should set the cookie for this session // automatically. Start with // to := traffic_ops.Login("user", "passwd", true) // subsequent calls like to.GetData("datadeliveryservice") will be authenticated. -func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, userAgent string, useCache bool) (*Session, error) { +func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, userAgent string, useCache bool, requestTimeout time.Duration) (*Session, error) { credentials, err := loginCreds(toUser, toPasswd) if err != nil { return nil, err @@ -163,6 +166,7 @@ func LoginWithAgent(toURL string, toUser string, toPasswd string, insecure bool, } to := NewSession(toUser, toPasswd, toURL, userAgent, &http.Client{ + Timeout: requestTimeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, }, @@ -220,7 +224,7 @@ func (to *Session) request(method, path string, body []byte) (*http.Response, er } req.Header.Set("User-Agent", to.UserAgentStr) - resp, err := to.UserAgent.Do(req) + resp, err := to.Client.Do(req) if err != nil { return nil, err } diff --git a/traffic_stats/traffic_stats.go b/traffic_stats/traffic_stats.go index 5bf9c44ad0..b802a179cb 100644 --- a/traffic_stats/traffic_stats.go +++ b/traffic_stats/traffic_stats.go @@ -41,6 +41,7 @@ import ( ) const UserAgent = "traffic-stats" +const TrafficOpsRequestTimeout = time.Second * time.Duration(10) const ( // FATAL will exit after printing error @@ -414,7 +415,7 @@ func queryDB(con influx.Client, cmd string, database string) (res []influx.Resul } func writeSummaryStats(config StartupConfig, statsSummary traffic_ops.StatsSummary) { - to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false) + to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false, TrafficOpsRequestTimeout) if err != nil { newErr := fmt.Errorf("Could not store summary stats! Error logging in to %v: %v", config.ToURL, err) log.Error(newErr) @@ -428,7 +429,7 @@ func writeSummaryStats(config StartupConfig, statsSummary traffic_ops.StatsSumma func getToData(config StartupConfig, init bool, configChan chan RunningConfig) { var runningConfig RunningConfig - to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false) + to, err := traffic_ops.LoginWithAgent(config.ToURL, config.ToUser, config.ToPasswd, true, UserAgent, false, TrafficOpsRequestTimeout) if err != nil { msg := fmt.Sprintf("Error logging in to %v: %v", config.ToURL, err) if init {