Update Traffic Monitor to use TO API v5 (#7761)

* Update Traffic Monitor to use TO API v5

Also use TO API v4 for the TO legacy API version

* Update date format in test data

* Do not ignore libraries necessary for the newer version of the TO client
in the TM integration test dockerfiles

* Update fields in test data

* Use API version 5 for POSTs to the test TO

* changing hostname to hostName for server v5 version

* Bugfix: Make fields omitempty if POSTing it them would result in an
unmarshalling error

* Update CrConfig, TMConfig, and servers response based on the Go structs'
serialization of them

* Re-add cachegroup name

* re-add domain name

---------

Co-authored-by: Srijeet Chatterjee <srijeet0406@gmail.com>

.github/actions/tm-integration-tests/entrypoint.sh

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -o errexit -o nounset -o pipefail
 
 function wait_for_endpoint() {
   try=0
@@ -97,11 +97,11 @@ touch to.log
 ./testto $TESTTO_PORT >to.log 2>&1 &
 tail -f to.log | color_and_prefix "${gray_bg}" 'Test TO' &
 
-wait_for_endpoint "${TESTTO_URI}:${TESTTO_PORT}/api/4.0/ping"
+wait_for_endpoint "${TESTTO_URI}:${TESTTO_PORT}/api/5.0/ping"
 
-curl -Lvsk ${TESTTO_URI}:${TESTTO_PORT}/api/4.0/cdns/$CDN/snapshot -X POST -d "@${test_dir}/snapshot.json"
-curl -Lvsk ${TESTTO_URI}:${TESTTO_PORT}/api/4.0/cdns/$CDN/configs/monitoring -X POST -d "@${test_dir}/monitoring.json"
-curl -Lvsk ${TESTTO_URI}:${TESTTO_PORT}/api/4.0/servers -X POST -d "@${test_dir}/servers.json"
+curl -Lvsk ${TESTTO_URI}:${TESTTO_PORT}/api/5.0/cdns/$CDN/snapshot -X POST -d "@${test_dir}/snapshot.json"
+curl -Lvsk ${TESTTO_URI}:${TESTTO_PORT}/api/5.0/cdns/$CDN/configs/monitoring -X POST -d "@${test_dir}/monitoring.json"
+curl -Lvsk ${TESTTO_URI}:${TESTTO_PORT}/api/5.0/servers -X POST -d "@${test_dir}/servers.json"
 
 cd "${repo_dir}/traffic_monitor/tools/testcaches"
 go mod vendor

CHANGELOG.md

@@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 ### Changed
 - [#7757](https://github.com/apache/trafficcontrol/pull/7757) *Traffic Router* Changed Traffic Router to point to api version 5.0 of Traffic Ops.
 - [#7732](https://github.com/apache/trafficcontrol/pull/7732) *Traffic Router* Increased negative TTL value to 900 seconds.
+- [#7761](https://github.com/apache/trafficcontrol/pull/7761) *Traffic Monitor* Use API v5 for the TM's Traffic Ops client, and use TO API v4 for TM's Traffic Ops legacy client
 - [#7584](https://github.com/apache/trafficcontrol/pull/7584) *Documentation* Upgrade Traffic Control Sphinx documentation Makefile OS intelligent.
 - [#7521](https://github.com/apache/trafficcontrol/pull/7521) *Traffic Ops* Returns empty array instead of null when no permissions are given for roles endpoint using POST or PUT request.
 - [#7369](https://github.com/apache/trafficcontrol/pull/7369) *Traffic Portal* Adds better labels to routing methods widget on the TP dashboard.

lib/go-tc/crconfig.go

@@ -120,7 +120,7 @@ type CRConfigTrafficOpsServer struct {
 	Fqdn             *string               `json:"fqdn,omitempty"`
 	HashCount        *int                  `json:"hashCount,omitempty"`
 	HashId           *string               `json:"hashId,omitempty"`
-	HttpsPort        *int                  `json:"httpsPort"`
+	HttpsPort        *int                  `json:"httpsPort,omitempty"`
 	InterfaceName    *string               `json:"interfaceName"`
 	Ip               *string               `json:"ip,omitempty"`
 	Ip6              *string               `json:"ip6,omitempty"`
@@ -242,7 +242,7 @@ type CRConfigLatitudeLongitude struct {
 	Lat                 float64                 `json:"latitude"`
 	Lon                 float64                 `json:"longitude"`
 	BackupLocations     CRConfigBackupLocations `json:"backupLocations,omitempty"`
-	LocalizationMethods []LocalizationMethod    `json:"localizationMethods"`
+	LocalizationMethods []LocalizationMethod    `json:"localizationMethods,omitempty"`
 }
 
 // CRConfigLatitudeLongitudeShort is a geographical location where clients will
@@ -271,7 +271,7 @@ type CRConfigDeliveryServiceProtocol struct {
 // Monitors, named with "CRConfig" for legacy reasons.
 type CRConfigMonitor struct {
 	FQDN         *string               `json:"fqdn,omitempty"`
-	HTTPSPort    *int                  `json:"httpsPort"`
+	HTTPSPort    *int                  `json:"httpsPort,omitempty"`
 	IP           *string               `json:"ip,omitempty"`
 	IP6          *string               `json:"ip6,omitempty"`
 	Location     *string               `json:"location,omitempty"`

lib/go-tc/servers.go

@@ -1317,53 +1317,53 @@ type ServerV50 struct {
 	CDN          string `json:"cdn" db:"cdn_name"`
 	// The time at which configuration updates were last applied for this server
 	// by t3c.
-	ConfigApplyTime *time.Time `json:"configApplyTime" db:"config_apply_time"`
+	ConfigApplyTime *time.Time `json:"configApplyTime,omitempty" db:"config_apply_time"`
 	// The time at which configuration updates were last queued for this server.
-	ConfigUpdateTime *time.Time `json:"configUpdateTime" db:"config_update_time"`
+	ConfigUpdateTime *time.Time `json:"configUpdateTime,omitempty" db:"config_update_time"`
 	// If the last config apply failed for this server
 	ConfigUpdateFailed bool   `json:"configUpdateFailed" db:"config_update_failed"`
 	DomainName         string `json:"domainName" db:"domain_name"`
 	// Deprecated: This property has unknown purpose and should not be used so
 	// that we can get rid of it.
 	GUID         *string                  `json:"guid" db:"guid"`
 	HostName     string                   `json:"hostName" db:"host_name"`
-	HTTPSPort    *int                     `json:"httpsPort" db:"https_port"`
+	HTTPSPort    *int                     `json:"httpsPort,omitempty" db:"https_port"`
 	ID           int                      `json:"id" db:"id"`
-	ILOIPAddress *string                  `json:"iloIpAddress" db:"ilo_ip_address"`
-	ILOIPGateway *string                  `json:"iloIpGateway" db:"ilo_ip_gateway"`
-	ILOIPNetmask *string                  `json:"iloIpNetmask" db:"ilo_ip_netmask"`
-	ILOPassword  *string                  `json:"iloPassword" db:"ilo_password"`
-	ILOUsername  *string                  `json:"iloUsername" db:"ilo_username"`
+	ILOIPAddress *string                  `json:"iloIpAddress,omitempty" db:"ilo_ip_address"`
+	ILOIPGateway *string                  `json:"iloIpGateway,omitempty" db:"ilo_ip_gateway"`
+	ILOIPNetmask *string                  `json:"iloIpNetmask,omitempty" db:"ilo_ip_netmask"`
+	ILOPassword  *string                  `json:"iloPassword,omitempty" db:"ilo_password"`
+	ILOUsername  *string                  `json:"iloUsername,omitempty" db:"ilo_username"`
 	Interfaces   []ServerInterfaceInfoV40 `json:"interfaces" db:"interfaces"`
 	LastUpdated  time.Time                `json:"lastUpdated" db:"last_updated"`
 	// Deprecated: In the future, management interfaces must be configured as
 	// interfaces within the Interfaces of the server, not separately on these
 	// properties.
-	MgmtIPAddress *string `json:"mgmtIpAddress" db:"mgmt_ip_address"`
+	MgmtIPAddress *string `json:"mgmtIpAddress,omitempty" db:"mgmt_ip_address"`
 	// Deprecated: In the future, management interfaces must be configured as
 	// interfaces within the Interfaces of the server, not separately on these
 	// properties.
-	MgmtIPGateway *string `json:"mgmtIpGateway" db:"mgmt_ip_gateway"`
+	MgmtIPGateway *string `json:"mgmtIpGateway,omitempty" db:"mgmt_ip_gateway"`
 	// Deprecated: In the future, management interfaces must be configured as
 	// interfaces within the Interfaces of the server, not separately on these
 	// properties.
-	MgmtIPNetmask      *string  `json:"mgmtIpNetmask" db:"mgmt_ip_netmask"`
+	MgmtIPNetmask      *string  `json:"mgmtIpNetmask,omitempty" db:"mgmt_ip_netmask"`
 	OfflineReason      *string  `json:"offlineReason" db:"offline_reason"`
 	PhysicalLocation   string   `json:"physicalLocation" db:"phys_location"`
 	PhysicalLocationID int      `json:"physicalLocationID" db:"phys_location_id"`
-	Profiles           []string `json:"profiles" db:"profile_name"`
+	Profiles           []string `json:"profiles,omitempty" db:"profile_name"`
 	// Deprecated: This property has unknown purpose and should not be used so
 	// that we can get rid of it.
 	Rack *string `json:"rack" db:"rack"`
 	// The time at which revalidations for this server were last updated by t3c.
-	RevalApplyTime *time.Time `json:"revalApplyTime" db:"revalidate_apply_time"`
+	RevalApplyTime *time.Time `json:"revalApplyTime,omitempty" db:"revalidate_apply_time"`
 	// The time at which revalidations were last queued for this server.
-	RevalUpdateTime *time.Time `json:"revalUpdateTime" db:"revalidate_update_time"`
+	RevalUpdateTime *time.Time `json:"revalUpdateTime,omitempty" db:"revalidate_update_time"`
 	// If the last reval apply failed for this server
 	RevalUpdateFailed bool       `json:"revalUpdateFailed" db:"revalidate_update_failed"`
 	Status            string     `json:"status" db:"status"`
 	StatusID          int        `json:"statusID" db:"status_id"`
-	StatusLastUpdated *time.Time `json:"statusLastUpdated" db:"status_last_updated"`
+	StatusLastUpdated *time.Time `json:"statusLastUpdated,omitempty" db:"status_last_updated"`
 	TCPPort           *int       `json:"tcpPort" db:"tcp_port"`
 	Type              string     `json:"type" db:"server_type"`
 	TypeID            int        `json:"typeID" db:"server_type_id"`

lib/go-tc/traffic_monitor.go

@@ -86,7 +86,7 @@ type TrafficMonitorConfig struct {
 	Profiles []TMProfile `json:"profiles,omitempty"`
 	// Topologies is the set of topologies defined in Traffic Ops, consisting
 	// of just the EDGE_LOC-type cachegroup nodes.
-	Topologies map[string]CRConfigTopology `json:"topologies"`
+	Topologies map[string]CRConfigTopology `json:"topologies,omitempty"`
 }
 
 const healthThresholdAvailableBandwidthInKbps = "availableBandwidthInKbps"
@@ -475,7 +475,7 @@ type TMDeliveryService struct {
 	TotalKbpsThreshold int64    `json:"TotalKbpsThreshold"`
 	Topology           string   `json:"topology"`
 	Type               string   `json:"type"`
-	HostRegexes        []string `json:"hostRegexes"`
+	HostRegexes        []string `json:"hostRegexes,omitempty"`
 }
 
 // TMProfile is primarily a collection of the Parameters with special meaning

traffic_monitor/tests/_integration/Dockerfile

@@ -35,18 +35,7 @@ RUN dnf install -y initscripts epel-release jq git
 COPY traffic_monitor/tests/_integration/ /tm/
 
 RUN mkdir -p ${GOPATH}/src/github.com/apache/trafficcontrol
-COPY go.mod \
-     go.sum \
-     ${GOPATH}/src/github.com/apache/trafficcontrol/
-# config.go is included so that `go mod vendor` includes github.com/kelseyhightower/envconfig
-# since dependencies within _integration are not considered
-COPY cache-config/testing/ort-tests/config/config.go ${GOPATH}/src/github.com/apache/trafficcontrol/cache-config/testing/ort-tests/config/config.go
-COPY lib ${GOPATH}/src/github.com/apache/trafficcontrol/lib
-COPY vendor ${GOPATH}/src/github.com/apache/trafficcontrol/vendor
-COPY traffic_monitor/ ${GOPATH}/src/github.com/apache/trafficcontrol/traffic_monitor/
-COPY traffic_ops/toclientlib/ ${GOPATH}/src/github.com/apache/trafficcontrol/traffic_ops/toclientlib/
-COPY traffic_ops/v3-client/ ${GOPATH}/src/github.com/apache/trafficcontrol/traffic_ops/v3-client/
-COPY traffic_ops/v4-client/ ${GOPATH}/src/github.com/apache/trafficcontrol/traffic_ops/v4-client/
+COPY . ${GOPATH}/src/github.com/apache/trafficcontrol
 
 WORKDIR ${GOPATH}/src/github.com/apache/trafficcontrol/traffic_monitor/tests/_integration/
 RUN go mod vendor && \

traffic_monitor/tests/_integration/Dockerfile.dockerignore

@@ -15,12 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 *
+!/cache-config/t3cutil/
+# config.go is included so that `go mod vendor` includes github.com/kelseyhightower/envconfig
+# since dependencies within _integration are not considered
 !/cache-config/testing/ort-tests/config/config.go
 !/go.mod
 !/go.sum
 !/GO_VERSION
 !/lib/
 !/traffic_monitor
 !/traffic_ops/toclientlib
+!/traffic_ops/traffic_ops_golang/test
 !/traffic_ops/v*-client
 !/vendor/

traffic_monitor/tests/_integration/Dockerfile_run.sh

@@ -30,7 +30,7 @@ for v in $envvars; do
   fi
 done
 
-TO_API_VERSION=4.0
+TO_API_VERSION=5.0
 CFG_FILE=/tm/traffic-monitor-integration-test.cfg
 
 start() {