Add stats to HealthCheck, and export its cache.

go/vt/discovery/fake_health_check_test.go

@@ -31,3 +31,8 @@ func (fhc *fakeHealthCheck) GetEndPointStatsFromKeyspaceShard(keyspace, shard st
 func (fhc *fakeHealthCheck) GetEndPointStatsFromTarget(keyspace, shard string, tabletType pbt.TabletType) []*EndPointStats {
 	return nil
 }
+
+// CacheStatus returns a displayable version of the cache.
+func (fhc *fakeHealthCheck) CacheStatus() EndPointsCacheStatusList {
+	return nil
+}

go/vt/discovery/healthcheck.go

@@ -2,19 +2,31 @@ package discovery
 
 import (
 	"fmt"
+	"html/template"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
 	log "github.com/golang/glog"
+	"github.com/youtube/vitess/go/stats"
 	pbq "github.com/youtube/vitess/go/vt/proto/query"
 	pbt "github.com/youtube/vitess/go/vt/proto/topodata"
 	"github.com/youtube/vitess/go/vt/tabletserver/tabletconn"
 	"github.com/youtube/vitess/go/vt/topo"
 	"golang.org/x/net/context"
 )
 
+var (
+	hcConnCounters  *stats.MultiCounters
+	hcErrorCounters *stats.MultiCounters
+)
+
+func init() {
+	hcConnCounters = stats.NewMultiCounters("HealthcheckConnections", []string{"keyspace", "shard", "tablettype"})
+	hcErrorCounters = stats.NewMultiCounters("HealthcheckErrors", []string{"keyspace", "shard", "tablettype"})
+}
+
 // HealthCheckStatsListener is the listener to receive health check stats update.
 type HealthCheckStatsListener interface {
 	StatsUpdate(endPoint *pbt.EndPoint, cell string, target *pbq.Target, tabletExternallyReparentedTimestamp int64, stats *pbq.RealtimeStats)
@@ -39,6 +51,8 @@ type HealthCheck interface {
 	GetEndPointStatsFromKeyspaceShard(keyspace, shard string) []*EndPointStats
 	// GetEndPointStatsFromTarget returns all EndPointStats for the given target.
 	GetEndPointStatsFromTarget(keyspace, shard string, tabletType pbt.TabletType) []*EndPointStats
+	// CacheStatus returns a displayable version of the cache.
+	CacheStatus() EndPointsCacheStatusList
 }
 
 // NewHealthCheck creates a new HealthCheck object.
@@ -98,6 +112,9 @@ func (hc *HealthCheckImpl) checkConn(cell string, endPoint *pbt.EndPoint) {
 				return
 			default:
 			}
+			if hcc.target != nil {
+				hcErrorCounters.Add([]string{hcc.target.Keyspace, hcc.target.Shard, strings.ToLower(hcc.target.TabletType.String())}, 1)
+			}
 			log.Errorf("cannot connect to %+v: %v", endPoint, err)
 			time.Sleep(hc.retryDelay)
 			continue
@@ -114,6 +131,9 @@ func (hc *HealthCheckImpl) checkConn(cell string, endPoint *pbt.EndPoint) {
 					return
 				default:
 				}
+				if hcc.target != nil {
+					hcErrorCounters.Add([]string{hcc.target.Keyspace, hcc.target.Shard, strings.ToLower(hcc.target.TabletType.String())}, 1)
+				}
 				log.Errorf("error when streaming tablet health from %+v: %v", endPoint, err)
 				time.Sleep(hc.retryDelay)
 				break
@@ -313,6 +333,7 @@ func (hc *HealthCheckImpl) addEndPointToTargetProtected(target *pbq.Target, endP
 		}
 	}
 	ttMap[target.TabletType] = append(epList, endPoint)
+	hcConnCounters.Add([]string{target.Keyspace, target.Shard, strings.ToLower(target.TabletType.String())}, 1)
 }
 
 // deleteEndPointFromTargetProtected deletes the endpoint for the given target.
@@ -334,11 +355,89 @@ func (hc *HealthCheckImpl) deleteEndPointFromTargetProtected(target *pbq.Target,
 		if topo.EndPointEquality(ep, endPoint) {
 			epList = append(epList[:i], epList[i+1:]...)
 			ttMap[target.TabletType] = epList
+			hcConnCounters.Add([]string{target.Keyspace, target.Shard, strings.ToLower(target.TabletType.String())}, -1)
 			return
 		}
 	}
 }
 
+// EndPointsCacheStatus is the current endpoints for a cell/target.
+type EndPointsCacheStatus struct {
+	Cell           string
+	Target         *pbq.Target
+	EndPointsStats []*EndPointStats
+}
+
+// StatusAsHTML returns an HTML version of the status.
+func (epcs *EndPointsCacheStatus) StatusAsHTML() template.HTML {
+	epLinks := make([]string, 0, 1)
+	for _, eps := range epcs.EndPointsStats {
+		vtPort := eps.EndPoint.PortMap["vt"]
+		epLinks = append(epLinks, fmt.Sprintf(`<a href="http://%v:%d">%v:%d</a>`, eps.EndPoint.Host, vtPort, eps.EndPoint.Host, vtPort))
+	}
+	return template.HTML(strings.Join(epLinks, " "))
+}
+
+// EndPointsCacheStatusList is used for sorting.
+type EndPointsCacheStatusList []*EndPointsCacheStatus
+
+// Len is part of sort.Interface.
+func (epcsl EndPointsCacheStatusList) Len() int {
+	return len(epcsl)
+}
+
+// Less is part of sort.Interface
+func (epcsl EndPointsCacheStatusList) Less(i, j int) bool {
+	return epcsl[i].Cell+"."+epcsl[i].Target.Keyspace+"."+epcsl[i].Target.Shard+"."+string(epcsl[i].Target.TabletType) <
+		epcsl[j].Cell+"."+epcsl[j].Target.Keyspace+"."+epcsl[j].Target.Shard+"."+string(epcsl[j].Target.TabletType)
+}
+
+// Swap is part of sort.Interface
+func (epcsl EndPointsCacheStatusList) Swap(i, j int) {
+	epcsl[i], epcsl[j] = epcsl[j], epcsl[i]
+}
+
+// CacheStatus returns a displayable version of the cache.
+func (hc *HealthCheckImpl) CacheStatus() EndPointsCacheStatusList {
+	epcsl := make(EndPointsCacheStatusList, 0, 1)
+	hc.mu.RLock()
+	for _, shardMap := range hc.targetToEPs {
+		for _, ttMap := range shardMap {
+			for _, epList := range ttMap {
+				var epcs *EndPointsCacheStatus
+				for _, ep := range epList {
+					key := endPointToMapKey(ep)
+					hcc, ok := hc.addrToConns[key]
+					if !ok {
+						continue
+					}
+					hcc.mu.RLock()
+					if epcs == nil {
+						epcs = &EndPointsCacheStatus{
+							Cell:           hcc.cell,
+							Target:         hcc.target,
+							EndPointsStats: make([]*EndPointStats, 0, 1),
+						}
+						epcsl = append(epcsl, epcs)
+					}
+					stats := &EndPointStats{
+						Cell:     hcc.cell,
+						Target:   hcc.target,
+						EndPoint: ep,
+						Stats:    hcc.stats,
+						TabletExternallyReparentedTimestamp: hcc.tabletExternallyReparentedTimestamp,
+					}
+					hcc.mu.RUnlock()
+					epcs.EndPointsStats = append(epcs.EndPointsStats, stats)
+				}
+			}
+		}
+	}
+	hc.mu.RUnlock()
+	sort.Sort(epcsl)
+	return epcsl
+}
+
 // endPointToMapKey creates a key to the map from endpoint's host and ports.
 func endPointToMapKey(endPoint *pbt.EndPoint) string {
 	parts := make([]string, 0, 1)