Skip to content

Commit

Permalink
les, les/lespay/server: refactor client pool (#21236)
Browse files Browse the repository at this point in the history 
* les, les/lespay/server: refactor client pool

* les: use ns.Operation and sub calls where needed

* les: fixed tests

* les: removed active/inactive logic from peerSet

* les: removed active/inactive peer logic

* les: fixed linter warnings

* les: fixed more linter errors and added missing metrics

* les: addressed comments

* cmd/geth: fixed TestPriorityClient

* les: simplified clientPool state machine

* les/lespay/server: do not use goroutine for balance callbacks

* internal/web3ext: fix addBalance required parameters

* les: removed freeCapacity, always connect at minCapacity initially

* les: only allow capacity change with priority status

Co-authored-by: rjl493456442 <garyrong0905@gmail.com>
  • Loading branch information
@zsfelfoldi @rjl493456442
zsfelfoldi and rjl493456442 committed Sep 14, 2020
1 parent f7112cc commit 4996fce
Show file tree
Hide file tree
Showing 24 changed files with 3,100 additions and 1,692 deletions.
2 changes: 1 addition & 1 deletion cmd/geth/les_test.go
View file
Expand Up @@ -152,7 +152,7 @@ func TestPriorityClient(t *testing.T) {
defer prioCli.killAndWait()
// 3_000_000_000 once we move to Go 1.13
tokens := 3000000000
lightServer.callRPC(nil, "les_addBalance", prioCli.getNodeInfo().ID, tokens, "foobar")
lightServer.callRPC(nil, "les_addBalance", prioCli.getNodeInfo().ID, tokens)
prioCli.addPeer(lightServer)

// Check if priority client is actually syncing and the regular client got kicked out
Expand Down
49 changes: 32 additions & 17 deletions common/prque/lazyqueue.go
View file
Expand Up @@ -36,14 +36,15 @@ type LazyQueue struct {
// Items are stored in one of two internal queues ordered by estimated max
// priority until the next and the next-after-next refresh. Update and Refresh
// always places items in queue[1].
queue [2]*sstack
popQueue *sstack
period time.Duration
maxUntil mclock.AbsTime
indexOffset int
setIndex SetIndexCallback
priority PriorityCallback
maxPriority MaxPriorityCallback
queue [2]*sstack
popQueue *sstack
period time.Duration
maxUntil mclock.AbsTime
indexOffset int
setIndex SetIndexCallback
priority PriorityCallback
maxPriority MaxPriorityCallback
lastRefresh1, lastRefresh2 mclock.AbsTime
}

type (
Expand All @@ -54,14 +55,17 @@ type (
// NewLazyQueue creates a new lazy queue
func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPriority MaxPriorityCallback, clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue {
q := &LazyQueue{
popQueue: newSstack(nil),
setIndex: setIndex,
priority: priority,
maxPriority: maxPriority,
clock: clock,
period: refreshPeriod}
popQueue: newSstack(nil),
setIndex: setIndex,
priority: priority,
maxPriority: maxPriority,
clock: clock,
period: refreshPeriod,
lastRefresh1: clock.Now(),
lastRefresh2: clock.Now(),
}
q.Reset()
q.Refresh()
q.refresh(clock.Now())
return q
}

Expand All @@ -71,9 +75,19 @@ func (q *LazyQueue) Reset() {
q.queue[1] = newSstack(q.setIndex1)
}

// Refresh should be called at least with the frequency specified by the refreshPeriod parameter
// Refresh performs queue re-evaluation if necessary
func (q *LazyQueue) Refresh() {
q.maxUntil = q.clock.Now() + mclock.AbsTime(q.period)
now := q.clock.Now()
for time.Duration(now-q.lastRefresh2) >= q.period*2 {
q.refresh(now)
q.lastRefresh2 = q.lastRefresh1
q.lastRefresh1 = now
}
}

// refresh re-evaluates items in the older queue and swaps the two queues
func (q *LazyQueue) refresh(now mclock.AbsTime) {
q.maxUntil = now + mclock.AbsTime(q.period)
for q.queue[0].Len() != 0 {
q.Push(heap.Pop(q.queue[0]).(*item).value)
}
Expand Down Expand Up @@ -139,6 +153,7 @@ func (q *LazyQueue) MultiPop(callback func(data interface{}, priority int64) boo
}
return
}
nextIndex = q.peekIndex() // re-check because callback is allowed to push items back
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion internal/web3ext/web3ext.go
View file
Expand Up @@ -844,7 +844,7 @@ web3._extend({
new web3._extend.Method({
name: 'addBalance',
call: 'les_addBalance',
params: 3
params: 2
}),
],
properties:
Expand Down
110 changes: 53 additions & 57 deletions les/api.go
View file
Expand Up @@ -19,28 +19,25 @@ package les
import (
"errors"
"fmt"
"math"
"time"

"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/mclock"
lps "github.com/ethereum/go-ethereum/les/lespay/server"
"github.com/ethereum/go-ethereum/p2p/enode"
)

var (
errNoCheckpoint = errors.New("no local checkpoint provided")
errNotActivated = errors.New("checkpoint registrar is not activated")
errUnknownBenchmarkType = errors.New("unknown benchmark type")
errBalanceOverflow = errors.New("balance overflow")
errNoPriority = errors.New("priority too low to raise capacity")
)

const maxBalance = math.MaxInt64

// PrivateLightServerAPI provides an API to access the LES light server.
type PrivateLightServerAPI struct {
server *LesServer
defaultPosFactors, defaultNegFactors priceFactors
defaultPosFactors, defaultNegFactors lps.PriceFactors
}

// NewPrivateLightServerAPI creates a new LES light server API.
Expand All @@ -57,17 +54,15 @@ func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} {
res := make(map[string]interface{})
res["minimumCapacity"] = api.server.minCapacity
res["maximumCapacity"] = api.server.maxCapacity
res["freeClientCapacity"] = api.server.freeCapacity
res["totalCapacity"], res["totalConnectedCapacity"], res["priorityConnectedCapacity"] = api.server.clientPool.capacityInfo()
return res
}

// ClientInfo returns information about clients listed in the ids list or matching the given tags
func (api *PrivateLightServerAPI) ClientInfo(ids []enode.ID) map[enode.ID]map[string]interface{} {
res := make(map[enode.ID]map[string]interface{})
api.server.clientPool.forClients(ids, func(client *clientInfo, id enode.ID) error {
res[id] = api.clientInfo(client, id)
return nil
api.server.clientPool.forClients(ids, func(client *clientInfo) {
res[client.node.ID()] = api.clientInfo(client)
})
return res
}
Expand All @@ -80,48 +75,40 @@ func (api *PrivateLightServerAPI) ClientInfo(ids []enode.ID) map[enode.ID]map[st
// assigned to it.
func (api *PrivateLightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCount int) map[enode.ID]map[string]interface{} {
res := make(map[enode.ID]map[string]interface{})
ids := api.server.clientPool.ndb.getPosBalanceIDs(start, stop, maxCount+1)
ids := api.server.clientPool.bt.GetPosBalanceIDs(start, stop, maxCount+1)
if len(ids) > maxCount {
res[ids[maxCount]] = make(map[string]interface{})
ids = ids[:maxCount]
}
if len(ids) != 0 {
api.server.clientPool.forClients(ids, func(client *clientInfo, id enode.ID) error {
res[id] = api.clientInfo(client, id)
return nil
api.server.clientPool.forClients(ids, func(client *clientInfo) {
res[client.node.ID()] = api.clientInfo(client)
})
}
return res
}

// clientInfo creates a client info data structure
func (api *PrivateLightServerAPI) clientInfo(c *clientInfo, id enode.ID) map[string]interface{} {
func (api *PrivateLightServerAPI) clientInfo(c *clientInfo) map[string]interface{} {
info := make(map[string]interface{})
if c != nil {
now := mclock.Now()
info["isConnected"] = true
info["connectionTime"] = float64(now-c.connectedAt) / float64(time.Second)
info["capacity"] = c.capacity
pb, nb := c.balanceTracker.getBalance(now)
info["pricing/balance"], info["pricing/negBalance"] = pb, nb
info["pricing/balanceMeta"] = c.balanceMetaInfo
info["priority"] = pb != 0
} else {
info["isConnected"] = false
pb := api.server.clientPool.ndb.getOrNewPB(id)
info["pricing/balance"], info["pricing/balanceMeta"] = pb.value, pb.meta
info["priority"] = pb.value != 0
pb, nb := c.balance.GetBalance()
info["isConnected"] = c.connected
info["pricing/balance"] = pb
info["priority"] = pb != 0
// cb := api.server.clientPool.ndb.getCurrencyBalance(id)
// info["pricing/currency"] = cb.amount
if c.connected {
info["connectionTime"] = float64(mclock.Now()-c.connectedAt) / float64(time.Second)
info["capacity"], _ = api.server.clientPool.ns.GetField(c.node, priorityPoolSetup.CapacityField).(uint64)
info["pricing/negBalance"] = nb
}
return info
}

// setParams either sets the given parameters for a single connected client (if specified)
// or the default parameters applicable to clients connected in the future
func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientInfo, posFactors, negFactors *priceFactors) (updateFactors bool, err error) {
func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientInfo, posFactors, negFactors *lps.PriceFactors) (updateFactors bool, err error) {
defParams := client == nil
if !defParams {
posFactors, negFactors = &client.posFactors, &client.negFactors
}
for name, value := range params {
errValue := func() error {
return fmt.Errorf("invalid value for parameter '%s'", name)
Expand All @@ -137,20 +124,20 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien

switch {
case name == "pricing/timeFactor":
setFactor(&posFactors.timeFactor)
setFactor(&posFactors.TimeFactor)
case name == "pricing/capacityFactor":
setFactor(&posFactors.capacityFactor)
setFactor(&posFactors.CapacityFactor)
case name == "pricing/requestCostFactor":
setFactor(&posFactors.requestFactor)
setFactor(&posFactors.RequestFactor)
case name == "pricing/negative/timeFactor":
setFactor(&negFactors.timeFactor)
setFactor(&negFactors.TimeFactor)
case name == "pricing/negative/capacityFactor":
setFactor(&negFactors.capacityFactor)
setFactor(&negFactors.CapacityFactor)
case name == "pricing/negative/requestCostFactor":
setFactor(&negFactors.requestFactor)
setFactor(&negFactors.RequestFactor)
case !defParams && name == "capacity":
if capacity, ok := value.(float64); ok && uint64(capacity) >= api.server.minCapacity {
err = api.server.clientPool.setCapacity(client, uint64(capacity))
_, err = api.server.clientPool.setCapacity(client.node, client.address, uint64(capacity), 0, true)
// Don't have to call factor update explicitly. It's already done
// in setCapacity function.
} else {
Expand All @@ -170,27 +157,25 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien
return
}

// AddBalance updates the balance of a client (either overwrites it or adds to it).
// It also updates the balance meta info string.
func (api *PrivateLightServerAPI) AddBalance(id enode.ID, value int64, meta string) ([2]uint64, error) {
oldBalance, newBalance, err := api.server.clientPool.addBalance(id, value, meta)
return [2]uint64{oldBalance, newBalance}, err
}

// SetClientParams sets client parameters for all clients listed in the ids list
// or all connected clients if the list is empty
func (api *PrivateLightServerAPI) SetClientParams(ids []enode.ID, params map[string]interface{}) error {
return api.server.clientPool.forClients(ids, func(client *clientInfo, id enode.ID) error {
if client != nil {
update, err := api.setParams(params, client, nil, nil)
var err error
api.server.clientPool.forClients(ids, func(client *clientInfo) {
if client.connected {
posFactors, negFactors := client.balance.GetPriceFactors()
update, e := api.setParams(params, client, &posFactors, &negFactors)
if update {
client.updatePriceFactors()
client.balance.SetPriceFactors(posFactors, negFactors)
}
if e != nil {
err = e
}
return err
} else {
return fmt.Errorf("client %064x is not connected", id[:])
err = fmt.Errorf("client %064x is not connected", client.node.ID())
}
})
return err
}

// SetDefaultParams sets the default parameters applicable to clients connected in the future
Expand All @@ -214,6 +199,15 @@ func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error {
return nil
}

// AddBalance adds the given amount to the balance of a client if possible and returns
// the balance before and after the operation
func (api *PrivateLightServerAPI) AddBalance(id enode.ID, amount int64) (balance [2]uint64, err error) {
api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo) {
balance[0], balance[1], err = c.balance.AddBalance(amount)
})
return
}

// Benchmark runs a request performance benchmark with a given set of measurement setups
// in multiple passes specified by passCount. The measurement time for each setup in each
// pass is specified in milliseconds by length.
Expand Down Expand Up @@ -304,13 +298,15 @@ func NewPrivateDebugAPI(server *LesServer) *PrivateDebugAPI {

// FreezeClient forces a temporary client freeze which normally happens when the server is overloaded
func (api *PrivateDebugAPI) FreezeClient(id enode.ID) error {
return api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo, id enode.ID) error {
if c == nil {
return fmt.Errorf("client %064x is not connected", id[:])
var err error
api.server.clientPool.forClients([]enode.ID{id}, func(c *clientInfo) {
if c.connected {
c.peer.freeze()
} else {
err = fmt.Errorf("client %064x is not connected", id[:])
}
c.peer.freezeClient()
return nil
})
return err
}

// PrivateLightAPI provides an API to access the LES light server or light client.
Expand Down
13 changes: 6 additions & 7 deletions les/api_test.go
View file
Expand Up @@ -107,7 +107,7 @@ func testCapacityAPI(t *testing.T, clientCount int) {
t.Fatalf("Failed to obtain rpc client: %v", err)
}
headNum, headHash := getHead(ctx, t, serverRpcClient)
minCap, freeCap, totalCap := getCapacityInfo(ctx, t, serverRpcClient)
minCap, totalCap := getCapacityInfo(ctx, t, serverRpcClient)
testCap := totalCap * 3 / 4
t.Logf("Server testCap: %d minCap: %d head number: %d head hash: %064x\n", testCap, minCap, headNum, headHash)
reqMinCap := uint64(float64(testCap) * minRelCap / (minRelCap + float64(len(clients)-1)))
Expand Down Expand Up @@ -202,7 +202,7 @@ func testCapacityAPI(t *testing.T, clientCount int) {

weights := make([]float64, len(clients))
for c := 0; c < 5; c++ {
setCapacity(ctx, t, serverRpcClient, clients[freeIdx].ID(), freeCap)
setCapacity(ctx, t, serverRpcClient, clients[freeIdx].ID(), minCap)
freeIdx = rand.Intn(len(clients))
var sum float64
for i := range clients {
Expand All @@ -214,7 +214,7 @@ func testCapacityAPI(t *testing.T, clientCount int) {
sum += weights[i]
}
for i, client := range clients {
weights[i] *= float64(testCap-freeCap-100) / sum
weights[i] *= float64(testCap-minCap-100) / sum
capacity := uint64(weights[i])
if i != freeIdx && capacity < getCapacity(ctx, t, serverRpcClient, client.ID()) {
setCapacity(ctx, t, serverRpcClient, client.ID(), capacity)
Expand All @@ -227,7 +227,7 @@ func testCapacityAPI(t *testing.T, clientCount int) {
setCapacity(ctx, t, serverRpcClient, client.ID(), capacity)
}
}
weights[freeIdx] = float64(freeCap)
weights[freeIdx] = float64(minCap)
for i := range clients {
weights[i] /= float64(testCap)
}
Expand All @@ -247,7 +247,7 @@ func testCapacityAPI(t *testing.T, clientCount int) {
default:
}

_, _, totalCap = getCapacityInfo(ctx, t, serverRpcClient)
_, totalCap = getCapacityInfo(ctx, t, serverRpcClient)
if totalCap < testCap {
t.Log("Total capacity underrun")
close(stop)
Expand Down Expand Up @@ -370,7 +370,7 @@ func getCapacity(ctx context.Context, t *testing.T, server *rpc.Client, clientID
return uint64(vv)
}

func getCapacityInfo(ctx context.Context, t *testing.T, server *rpc.Client) (minCap, freeCap, totalCap uint64) {
func getCapacityInfo(ctx context.Context, t *testing.T, server *rpc.Client) (minCap, totalCap uint64) {
var res map[string]interface{}
if err := server.CallContext(ctx, &res, "les_serverInfo"); err != nil {
t.Fatalf("Failed to query server info: %v", err)
Expand All @@ -387,7 +387,6 @@ func getCapacityInfo(ctx context.Context, t *testing.T, server *rpc.Client) (min
return uint64(vv)
}
minCap = decode("minimumCapacity")
freeCap = decode("freeClientCapacity")
totalCap = decode("totalCapacity")
return
}
Expand Down

0 comments on commit 4996fce

Please sign in to comment.