Skip to content

Commit

Permalink
Merge branch 'lni:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@tylerwilliams
tylerwilliams committed Apr 12, 2022
2 parents 240ba62 + 68619d0 commit fa0dbd6
Show file tree
Hide file tree
Showing 23 changed files with 352 additions and 107 deletions.
38 changes: 17 additions & 21 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
Expand All @@ -22,7 +22,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
Expand All @@ -33,7 +33,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
Expand All @@ -44,31 +44,31 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
run: |
sudo apt-get install librocksdb-dev
DRAGONBOAT_LOGDB=rocksdb make test
go1-14-unit-test:
go1-16-unit-test:
runs-on: ubuntu-18.04
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
go-version: 1.16.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
run: make test
go1-16-unit-test:
go1-17-unit-test:
runs-on: ubuntu-18.04
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.16.x
go-version: 1.17.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
Expand All @@ -79,7 +79,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
Expand All @@ -90,7 +90,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Test
Expand All @@ -101,7 +101,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Build
Expand All @@ -112,7 +112,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
go-version: 1.18.x
- name: Build
run: |
sudo apt-get install librocksdb-dev libsnappy-dev
Expand All @@ -126,15 +126,11 @@ jobs:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.15.x
- name: Install golangci-lint
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $GOROOT/bin v1.23.8
go get -u github.com/kisielk/errcheck
go get -u golang.org/x/lint/golint
go get -u github.com/gordonklaus/ineffassign
sudo apt-get install librocksdb-dev
go-version: 1.18.x
- name: Checkout code
uses: actions/checkout@v2
- name: Static check
run: make static-check
run:
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $GOROOT/bin v1.45.2
sudo apt-get install librocksdb-dev
make static-check
22 changes: 7 additions & 15 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,30 +200,22 @@ tools-checkdisk:
###############################################################################
# static checks
###############################################################################
CHECKED_PKGS=$(shell go list ./...)
.PHONY: install-static-check-tools
install-static-check-tools:
@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $GOROOT/bin v1.45.2

CHECKED_PKGS=$(shell go list ./... | grep -v rocksdb)
CHECKED_DIRS=$(subst $(PKGNAME), ,$(subst $(PKGNAME)/, ,$(CHECKED_PKGS))) .
EXTRA_LINTERS=-E misspell -E scopelint -E rowserrcheck -E depguard -E unconvert \
EXTRA_LINTERS=-E misspell -E rowserrcheck -E depguard -E unconvert \
-E prealloc -E gofmt -E stylecheck
.PHONY: static-check
static-check:
@for p in $(CHECKED_PKGS); do \
go vet -tests=false $$p; \
golint $$p; \
errcheck -blank -ignoretests $$p; \
done;
@for p in $(CHECKED_DIRS); do \
ineffassign $$p; \
golangci-lint run $(EXTRA_LINTERS) $$p; \
done;

# -E dupl is not included in regular static check as there are duplicated code
# in auto generated code
.PHONY: extra-static-check
extra-static-check: override EXTRA_LINTERS :=-E dupl
extra-static-check:
for p in $(CHECKED_DIRS); do \
golangci-lint run $(EXTRA_LINTERS) $$p; \
done;
extra-static-check: static-check

###############################################################################
# clean
Expand Down
43 changes: 33 additions & 10 deletions engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -780,12 +780,16 @@ func (w *closeWorker) completed() {
}

func (w *closeWorker) handle(req closeReq) error {
if req.node.destroyed() {
return nil
}
return req.node.destroy()
}

type closeWorkerPool struct {
ready chan closeReq
busy map[uint64]struct{}
busy map[uint64]uint64
processing map[uint64]struct{}
workerStopper *syncutil.Stopper
poolStopper *syncutil.Stopper
workers []*closeWorker
Expand All @@ -796,7 +800,8 @@ func newCloseWorkerPool(closeWorkerCount uint64) *closeWorkerPool {
w := &closeWorkerPool{
workers: make([]*closeWorker, closeWorkerCount),
ready: make(chan closeReq, 1),
busy: make(map[uint64]struct{}, closeWorkerCount),
busy: make(map[uint64]uint64, closeWorkerCount),
processing: make(map[uint64]struct{}, closeWorkerCount),
pending: make([]*node, 0),
workerStopper: syncutil.NewStopper(),
poolStopper: syncutil.NewStopper(),
Expand Down Expand Up @@ -907,14 +912,20 @@ func (p *closeWorkerPool) isIdle() bool {
}

func (p *closeWorkerPool) completed(workerID uint64) {
if _, ok := p.busy[workerID]; !ok {
plog.Panicf("close worker %d is not in busy state")
clusterID, ok := p.busy[workerID]
if !ok {
plog.Panicf("close worker %d is not in busy state", workerID)
}
if _, ok := p.processing[clusterID]; !ok {
plog.Panicf("cluster %d is not being processed", clusterID)
}
delete(p.processing, clusterID)
delete(p.busy, workerID)
}

func (p *closeWorkerPool) setBusy(workerID uint64) {
p.busy[workerID] = struct{}{}
func (p *closeWorkerPool) setBusy(workerID uint64, clusterID uint64) {
p.processing[clusterID] = struct{}{}
p.busy[workerID] = clusterID
}

func (p *closeWorkerPool) getWorker() *closeWorker {
Expand All @@ -934,21 +945,33 @@ func (p *closeWorkerPool) schedule() {
}
}

func (p *closeWorkerPool) canSchedule(n *node) bool {
_, ok := p.processing[n.clusterID]
return !ok
}

func (p *closeWorkerPool) scheduleWorker() bool {
w := p.getWorker()
if w == nil {
return false
}
if len(p.pending) > 0 {
p.scheduleReq(p.pending[0], w)

for i := 0; i < len(p.pending); i++ {
node := p.pending[0]
p.removeFromPending(0)
return true
if p.canSchedule(node) {
p.scheduleReq(node, w)
return true
} else {
p.pending = append(p.pending, node)
}
}

return false
}

func (p *closeWorkerPool) scheduleReq(n *node, w *closeWorker) {
p.setBusy(w.workerID)
p.setBusy(w.workerID, n.clusterID)
select {
case w.requestC <- closeReq{node: n}:
default:
Expand Down
49 changes: 41 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,48 @@
module github.com/lni/dragonboat/v3

require (
github.com/VictoriaMetrics/metrics v1.6.2
github.com/cockroachdb/errors v1.8.2
github.com/cockroachdb/pebble v0.0.0-20210406181039-e3809b89b488
github.com/golang/snappy v0.0.4-0.20210502035320-33fc3d5d8d99
github.com/hashicorp/memberlist v0.2.4
github.com/VictoriaMetrics/metrics v1.18.1
github.com/cockroachdb/errors v1.9.0
github.com/cockroachdb/pebble v0.0.0-20220407171941-2120d145e292
github.com/golang/snappy v0.0.4
github.com/hashicorp/memberlist v0.3.1
github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441
github.com/lni/goutils v1.3.1-0.20210517080819-7f56813dc438
github.com/lni/vfs v0.2.0
github.com/lni/goutils v1.3.1-0.20220404072553-ddb2075d2587
github.com/lni/vfs v0.2.1-0.20220408085249-8be85be1c3c1
github.com/stretchr/testify v1.7.0
)

go 1.14
require (
github.com/DataDog/zstd v1.4.5 // indirect
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/getsentry/sentry-go v0.12.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
github.com/hashicorp/go-msgpack v0.5.3 // indirect
github.com/hashicorp/go-multierror v1.0.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.0 // indirect
github.com/hashicorp/golang-lru v0.5.0 // indirect
github.com/klauspost/compress v1.11.7 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/miekg/dns v1.1.26 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/valyala/fastrand v1.1.0 // indirect
github.com/valyala/histogram v1.2.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
golang.org/x/net v0.0.0-20211008194852-3b03d305991f // indirect
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

go 1.17
Loading

0 comments on commit fa0dbd6

Please sign in to comment.