Skip to content

Commit

Permalink
feat: Double buffer for sequencer catch (#231)
Browse files Browse the repository at this point in the history
  • Loading branch information
ZLBer committed Dec 2, 2021
1 parent bc0cb27 commit e0e5eba
Show file tree
Hide file tree
Showing 9 changed files with 429 additions and 71 deletions.
5 changes: 1 addition & 4 deletions go.mod
Expand Up @@ -3,19 +3,16 @@ module mosn.io/layotto
go 1.14

require (
github.com/alicebob/miniredis/v2 v2.13.3
github.com/dapr/components-contrib v1.4.0-rc2
github.com/dapr/kit v0.0.2-0.20210614175626-b9074b64d233
github.com/fsnotify/fsnotify v1.4.9
github.com/gammazero/workerpool v1.1.2
github.com/go-ini/ini v1.63.2 // indirect
github.com/golang/mock v1.6.0
github.com/golang/protobuf v1.5.2
github.com/google/uuid v1.2.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/json-iterator/go v1.1.11
github.com/minio/minio-go v6.0.14+incompatible // indirect
github.com/minio/minio-go/v6 v6.0.57 // indirect
github.com/minio/minio-go/v7 v7.0.15 // indirect
github.com/pkg/errors v0.9.1
github.com/shirou/gopsutil v3.21.3+incompatible
github.com/stretchr/testify v1.7.0
Expand Down
20 changes: 0 additions & 20 deletions go.sum
Expand Up @@ -531,8 +531,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ini/ini v1.63.2 h1:kwN3umicd2HF3Tgvap4um1ZG52/WyKT9GGdPx0CJk6Y=
github.com/go-ini/ini v1.63.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
Expand Down Expand Up @@ -1153,11 +1151,6 @@ github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz
github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go v1.0.0 h1:ooSujki+Z1PRGZsYffJw5jnF5eMBvzMVV86TLAlM0UM=
github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw=
github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM=
github.com/minio/minio-go/v7 v7.0.15 h1:r9/NhjJ+nXYrIYvbObhvc1wPj3YH1iDpJzz61uRKLyY=
github.com/minio/minio-go/v7 v7.0.15/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
Expand Down Expand Up @@ -1452,7 +1445,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
Expand Down Expand Up @@ -1614,8 +1606,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0 h1:OtISOGfH6sOWa1/qXqqAiOIAO6Z5J3AEAE18WAq6BiQ=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e h1:oIpIX9VKxSCFrfjsKpluGbNPBGq9iNnT9crH781j9wY=
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
Expand Down Expand Up @@ -1734,7 +1724,6 @@ golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaE
golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down Expand Up @@ -1797,8 +1786,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -1855,7 +1842,6 @@ golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5RKzgYBMMxLxt6n5XW50=
golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -1968,10 +1954,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef h1:fPxZ3Umkct3LZ8gK9nbk+DWDJ9fstZa2grBn+lWVKPs=
golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down Expand Up @@ -2073,8 +2055,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
2 changes: 1 addition & 1 deletion pkg/grpc/api.go
Expand Up @@ -1031,7 +1031,7 @@ func (a *api) GetNextId(ctx context.Context, req *runtimev1pb.GetNextIdRequest)
return &runtimev1pb.GetNextIdResponse{}, err
}
// modify key
compReq.Key, err = runtime_sequencer.GetModifiedKey(compReq.Key, req.StoreName, a.appId)
compReq.Key, err = runtime_sequencer.GetModifiedSeqKey(compReq.Key, req.StoreName, a.appId)
if err != nil {
log.DefaultLogger.Errorf("[runtime] [grpc.GetNextId] error: %v", err)
return &runtimev1pb.GetNextIdResponse{}, err
Expand Down
6 changes: 6 additions & 0 deletions pkg/runtime/runtime.go
Expand Up @@ -429,6 +429,12 @@ func (m *MosnRuntime) initSequencers(factorys ...*runtime_sequencer.Factory) err
m.errInt(err, "init sequencer component %s failed", name)
return err
}
// 2.3. save runtime related configs
err = runtime_sequencer.SaveSeqConfiguration(name, config.Metadata)
if err != nil {
m.errInt(err, "save sequencer configuration %s failed", name)
return err
}
m.sequencers[name] = comp
}
return nil
Expand Down
212 changes: 205 additions & 7 deletions pkg/runtime/sequencer/cache.go
Expand Up @@ -15,15 +15,213 @@ package sequencer

import (
"context"
"errors"
"mosn.io/layotto/components/sequencer"
"mosn.io/pkg/log"
"mosn.io/pkg/utils"
"sync"
"time"
)

const defaultSize = 10000
const defaultLimit = 1000
const defaultRetry = 5
const waitTime = time.Second * 2

// DoubleBuffer is double segment id buffer.
// There are two buffers in DoubleBuffer: inUseBuffer is in use, BackUpBuffer is a backup buffer.
// Their default capacity is 1000. When the inUseBuffer usage exceeds 30%, the BackUpBuffer will be initialized.
// When inUseBuffer is used up, swap them.
type DoubleBuffer struct {
Key string
size int
inUseBuffer *Buffer
backUpBufferChan chan *Buffer
lock sync.Mutex
Store sequencer.Store
}

type Buffer struct {
from int64
to int64
}

func NewDoubleBuffer(key string, store sequencer.Store) *DoubleBuffer {

d := &DoubleBuffer{
Key: key,
size: defaultSize,
Store: store,
backUpBufferChan: make(chan *Buffer, 1),
}

return d
}

//init double buffer
func (d *DoubleBuffer) init() error {

buffer, err := d.getNewBuffer()
if err != nil {
return err
}

d.inUseBuffer = buffer

return nil
}

//getId next id
func (d *DoubleBuffer) getId() (int64, error) {

d.lock.Lock()
defer d.lock.Unlock()

if d.inUseBuffer == nil {
return 0, errors.New("[DoubleBuffer] Get error: inUseBuffer nil ")
}
//check swap
if d.inUseBuffer.from > d.inUseBuffer.to {
err := d.swap()
if err != nil {
return 0, err
}
}
next := d.inUseBuffer.from
d.inUseBuffer.from++

//when inUseBuffer id more than limit used, initialize BackUpBuffer.
//equal make sure only one thread enter
if d.inUseBuffer.to-d.inUseBuffer.from == defaultLimit {
utils.GoWithRecover(func() {
//quick retry
for i := 0; i < defaultRetry; i++ {
buffer, err := d.getNewBuffer()
if err != nil {
log.DefaultLogger.Errorf("[DoubleBuffer] [getNewBuffer] error: %v", err)
continue
}
d.backUpBufferChan <- buffer
return
}
//slow retry
for true {
buffer, err := d.getNewBuffer()
if err != nil {
log.DefaultLogger.Errorf("[DoubleBuffer] [getNewBuffer] error: %v", err)
time.Sleep(waitTime)
continue
}
d.backUpBufferChan <- buffer
return
}
}, nil)
}

return next, nil
}

//swap inUseBuffer and BackUpBuffer, must be locked
func (d *DoubleBuffer) swap() error {

select {
case buffer := <-d.backUpBufferChan:
{
d.inUseBuffer = buffer
return nil
}
//timeout, return error
case <-time.After(waitTime):
{
return errors.New("[DoubleBuffer] swap error")
}
}
}

//getNewBuffer return a new segment
func (d *DoubleBuffer) getNewBuffer() (*Buffer, error) {
support, result, err := d.Store.GetSegment(&sequencer.GetSegmentRequest{
Key: d.Key,
Size: d.size,
})
if err != nil {
return nil, err
}
if !support {
return nil, errors.New("[DoubleBuffer] unSupport Segment id")
}
return &Buffer{
from: result.From,
to: result.To,
}, nil
}

// BufferCatch catch key and buffer
var BufferCatch = map[string]*DoubleBuffer{}

//read/write lock for BufferCatch
var rwLock sync.RWMutex

func GetNextIdFromCache(ctx context.Context, store sequencer.Store, req *sequencer.GetNextIdRequest) (bool, int64, error) {
// TODO
// 1. check cache
// 2. load cache with lock
// 2.1. return if not support
// 2.2. add segment into cache
// 2.3. return result
return false, 0, nil

// 1. check support
support, _, _ := store.GetSegment(&sequencer.GetSegmentRequest{
Key: req.Key,
Size: 0,
})

// return if not support
if !support {
return false, 0, nil
}

// 2. find the DoubleBuffer for this store and key
var d *DoubleBuffer
var err error

d = getDoubleBufferInRL(req.Key)
if d == nil {
d, err = getDoubleBufferInWL(req.Key, store)
}

if err != nil {
return true, 0, err
}

// 3. get the next id.
// The buffer should automatically load segment into cache if the cache is (nearly) empty
id, err := d.getId()

if err != nil {
return true, 0, err
}

return true, id, nil
}

// get DoubleBuffer using write lock
func getDoubleBufferInWL(key string, store sequencer.Store) (*DoubleBuffer, error) {
d := NewDoubleBuffer(key, store)
rwLock.Lock()
defer rwLock.Unlock()
//double check
if _, ok := BufferCatch[key]; ok {
return BufferCatch[key], nil
}
err := d.init()
if err != nil {
return nil, err
}
BufferCatch[key] = d
return d, nil
}

// get DoubleBuffer using read lock
func getDoubleBufferInRL(key string) *DoubleBuffer {
rwLock.RLock()
defer rwLock.RUnlock()
if buffer, ok := BufferCatch[key]; ok {
return buffer
}
return nil
}
52 changes: 52 additions & 0 deletions pkg/runtime/sequencer/catch_test.go
@@ -0,0 +1,52 @@
//
// Copyright 2021 Layotto Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sequencer

import (
"context"
"github.com/alicebob/miniredis/v2"
"github.com/stretchr/testify/assert"
"mosn.io/layotto/components/sequencer"
"mosn.io/layotto/components/sequencer/redis"
"mosn.io/pkg/log"
"testing"
)

const keyXx = "resource_xxx"
const idLimit = 12000

func TestGetNextIdFromCache(t *testing.T) {
s, err := miniredis.Run()
assert.NoError(t, err)
defer s.Close()
// construct componen
comp := redis.NewStandaloneRedisSequencer(log.DefaultLogger)
cfg := sequencer.Configuration{
Properties: make(map[string]string),
}
cfg.Properties["redisHost"] = s.Addr()
cfg.Properties["redisPassword"] = ""
// init
err = comp.Init(cfg)
assert.NoError(t, err)

for i := 1; i < idLimit; i++ {
support, id, err := GetNextIdFromCache(context.Background(), comp, &sequencer.GetNextIdRequest{
Key: keyXx,
})
assert.NoError(t, err)
assert.Equal(t, true, support)
assert.Equal(t, id, int64(i))
}
}

0 comments on commit e0e5eba

Please sign in to comment.