Merge pull request #9222 from tomponline/tp-cluster-heartbeat-handler

Cluster: Fix slow heartbeat response due to multiple remote queries when populating raft node names

lxd/cluster/gateway.go

@@ -229,25 +229,13 @@ func (g *Gateway) HandlerFuncs(nodeRefreshTask func(*APIHeartbeat), trustedCerts
 			raftNodes := make([]db.RaftNode, 0)
 			for _, node := range heartbeatData.Members {
 				if node.RaftID > 0 {
-					nodeInfo := db.NodeInfo{}
-					if g.Cluster != nil {
-						err = g.Cluster.Transaction(func(tx *db.ClusterTx) error {
-							var err error
-							nodeInfo, err = tx.GetNodeByAddress(node.Address)
-							return err
-						})
-						if err != nil {
-							logger.Warn("Failed to retrieve cluster member", log.Ctx{"err": err})
-						}
-					}
-
 					raftNodes = append(raftNodes, db.RaftNode{
 						NodeInfo: client.NodeInfo{
 							ID:      node.RaftID,
 							Address: node.Address,
 							Role:    db.RaftRole(node.RaftRole),
 						},
-						Name: nodeInfo.Name,
+						Name: node.Name,
 					})
 				}
 			}
@@ -943,7 +931,7 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
 		return nil, err
 	}
 
-	raftNodes := []db.RaftNode{}
+	raftNodes := make([]db.RaftNode, 0, len(servers))
 	for i, server := range servers {
 		address, err := g.nodeAddress(server.Address)
 		if err != nil {
@@ -958,19 +946,29 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
 	// Get the names of the raft nodes from the global database.
 	if g.Cluster != nil {
 		err = g.Cluster.Transaction(func(tx *db.ClusterTx) error {
+			nodes, err := tx.GetNodes()
+			if err != nil {
+				return fmt.Errorf("Failed loading cluster members: %w", err)
+			}
+
+			nodesByAddress := make(map[string]db.NodeInfo, len(nodes))
+			for _, node := range nodes {
+				nodesByAddress[node.Address] = node
+			}
+
 			for i, server := range servers {
-				node, err := tx.GetNodeByAddress(server.Address)
-				if err != nil {
-					return err
+				node, found := nodesByAddress[server.Address]
+				if !found {
+					return fmt.Errorf("Cluster member info not found for %q", server.Address)
 				}
 
 				raftNodes[i].Name = node.Name
-
 			}
+
 			return nil
 		})
 		if err != nil {
-			logger.Warn("Failed to retrieve cluster member", log.Ctx{"err": err})
+			logger.Warn("Failed getting raft nodes", log.Ctx{"err": err})
 		}
 	}
 

lxd/cluster/heartbeat.go

@@ -34,6 +34,7 @@ const (
 type APIHeartbeatMember struct {
 	ID            int64     // ID field value in nodes table.
 	Address       string    // Host and Port of node.
+	Name          string    // Name of cluster member.
 	RaftID        uint64    // ID field value in raft_nodes table, zero if non-raft node.
 	RaftRole      int       // Node role in the raft cluster, from the raft_nodes table
 	Raft          bool      // Deprecated, use non-zero RaftID instead to indicate raft node.
@@ -74,9 +75,8 @@ func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode,
 	// If we've been supplied a fresh set of node states, this is a full state list.
 	hbState.FullStateList = fullStateList
 
-	raftNodeMap := make(map[string]db.RaftNode)
-
 	// Convert raftNodes to a map keyed on address for lookups later.
+	raftNodeMap := make(map[string]db.RaftNode, len(raftNodes))
 	for _, raftNode := range raftNodes {
 		raftNodeMap[raftNode.Address] = raftNode
 	}
@@ -86,6 +86,7 @@ func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode,
 		member := APIHeartbeatMember{
 			ID:            node.ID,
 			Address:       node.Address,
+			Name:          node.Name,
 			LastHeartbeat: node.Heartbeat,
 			Online:        !node.Heartbeat.Before(time.Now().Add(-offlineThreshold)),
 		}

lxd/cluster/recover.go

@@ -139,8 +139,8 @@ func Reconfigure(database *db.Node, raftNodes []db.RaftNode) error {
 	}
 
 	localAddress := info.Address
-	nodes := []client.NodeInfo{}
 
+	nodes := make([]client.NodeInfo, 0, len(raftNodes))
 	for _, raftNode := range raftNodes {
 		nodes = append(nodes, raftNode.NodeInfo)