From 98141935232072bf5f5c17a15c085bcd2f6cd1cc Mon Sep 17 00:00:00 2001 From: "allen.wq" Date: Wed, 15 Apr 2020 16:15:47 +0800 Subject: [PATCH] add bitmap in pkg, the max bitIndex is math.MaxInt32 Signed-off-by: allen.wq --- go.mod | 3 +- go.sum | 9 ++ pkg/bitmap/bitmap.go | 287 ++++++++++++++++++++++++++++++++++++++ pkg/bitmap/bitmap_test.go | 259 ++++++++++++++++++++++++++++++++++ pkg/bitmap/utils.go | 40 ++++++ 5 files changed, 597 insertions(+), 1 deletion(-) create mode 100644 pkg/bitmap/bitmap.go create mode 100644 pkg/bitmap/bitmap_test.go create mode 100644 pkg/bitmap/utils.go diff --git a/go.mod b/go.mod index 3793ab4e8..38103eaf4 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/magiconair/properties v1.8.1 // indirect github.com/mailru/easyjson v0.0.0-20170902151237-2a92e673c9a6 // indirect github.com/mitchellh/mapstructure v1.1.2 + github.com/openacid/low v0.1.10 github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3 github.com/pkg/errors v0.8.0 github.com/prashantv/gostub v1.0.0 @@ -38,7 +39,7 @@ require ( github.com/spf13/cobra v0.0.0-20181021141114-fe5e611709b0 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.4.0 - github.com/stretchr/testify v1.2.2 + github.com/stretchr/testify v1.3.0 github.com/valyala/fasthttp v1.3.0 github.com/willf/bitset v0.0.0-20190228212526-18bd95f470f9 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect diff --git a/go.sum b/go.sum index a89f1fa23..e61e7a04d 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ github.com/asaskevich/govalidator v0.0.0-20170903095215-73945b6115bf/go.mod h1:l github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -25,6 +26,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.7 h1:DVS0EPFHUiaJSaX2EKlaf65HUmk9PXhOl/Xa3Go242Q= github.com/cpuguy83/go-md2man v1.0.7/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -115,6 +117,10 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/openacid/errors v0.8.1/go.mod h1:GUQEJJOJE3W9skHm8E8Y4phdl2LLEN8iD7c5gcGgdx0= +github.com/openacid/low v0.1.10 h1:rKpmB5CHtKoPq9tFiqUvRk8vtWaPympL2D2dNfw3PvI= +github.com/openacid/low v0.1.10/go.mod h1:QCkCiLykPRXaaZV76EsiRePPqQlqraEaV5WdGQh4qKk= +github.com/openacid/must v0.1.3/go.mod h1:luPiXCuJlEo3UUFQngVQokV0MPGryeYvtCbQPs3U1+I= github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3 h1:9J0mOv1rXIBlRjQCiAGyx9C3dZZh5uIa3HU0oTV8v1E= github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -159,9 +165,12 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= diff --git a/pkg/bitmap/bitmap.go b/pkg/bitmap/bitmap.go new file mode 100644 index 000000000..383a9206d --- /dev/null +++ b/pkg/bitmap/bitmap.go @@ -0,0 +1,287 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bitmap + +import ( + "encoding/binary" + "fmt" + "math" + "sync" + + lbm "github.com/openacid/low/bitmap" +) + +const ( + // sizeOf64BitsLimit limits the max size of array of bitmap. + // It is limited by "github.com/openacid/low/bitmap". + sizeOf64BitsLimit = (math.MaxInt32 >> 6) + 1 +) + +// BitMap is a struct which provides the Get or Set of bits map. +type BitMap struct { + lock sync.RWMutex + bm []uint64 + maxBitIndex uint32 +} + +// NewBitMap generates a BitMap. +func NewBitMap(sizeOf64Bits uint32, allSetBit bool) (*BitMap, error) { + if sizeOf64Bits > sizeOf64BitsLimit { + return nil, fmt.Errorf("sizeOf64Bits %d should be range[0, %d]", sizeOf64Bits, sizeOf64BitsLimit) + } + + bm := make([]uint64, sizeOf64Bits) + if allSetBit { + for i := 0; i < int(sizeOf64Bits); i++ { + bm[i] = math.MaxUint64 + } + } + + return &BitMap{ + bm: bm, + maxBitIndex: (sizeOf64Bits << 6) - 1, + }, nil +} + +// RestoreBitMap generate the BitMap by input bytes. +func RestoreBitMap(data []byte) (*BitMap, error) { + if uint64(len(data)) > (sizeOf64BitsLimit << 3) { + return nil, fmt.Errorf("sizeOf64Bits %d should be range[0, %d]", uint64(len(data)), sizeOf64BitsLimit<<3) + } + + bm := DecodeToUintArray(data) + + return &BitMap{ + bm: bm, + maxBitIndex: uint32(len(bm)*64 - 1), + }, nil +} + +// Get gets the bits in range [start, end]. if set is true, return the set bits. +// else return the unset bits. +func (b *BitMap) Get(start uint32, end uint32, setBit bool) ([]*BitsRange, error) { + b.lock.RLock() + defer b.lock.RUnlock() + + return b.getWithoutLock(start, end, setBit) +} + +// Set sets or cleans the bits in range [start, end]. if setBit is true, set bits. else clean bits. +func (b *BitMap) Set(start uint32, end uint32, setBit bool) error { + b.lock.Lock() + defer b.lock.Unlock() + + return b.setWithoutLock(start, end, setBit) +} + +func (b *BitMap) getWithoutLock(start uint32, end uint32, setBit bool) ([]*BitsRange, error) { + if start > end { + return nil, fmt.Errorf("start %d should not bigger than %d", start, end) + } + + if end > b.maxBitIndex { + return nil, fmt.Errorf("end %d should not bigger than %d", end, b.maxBitIndex) + } + + rs := []*BitsRange{} + var lastRange *BitsRange + + second64MinIndex := ((start >> 6) + 1) << 6 + first64MaxIndex := end + if first64MaxIndex >= second64MinIndex { + first64MaxIndex = second64MinIndex - 1 + } + + last64MinIndex := (end >> 6) << 6 + + appendArr, last := b.mergeAndFetchRangeOfUint64(b.bm[start>>6], (start>>6)<<6, setBit, start, first64MaxIndex, lastRange) + rs = append(rs, appendArr...) + lastRange = last + + for i := second64MinIndex; i < last64MinIndex; i += 64 { + appendArr, last := b.mergeAndFetchRangeOfUint64(b.bm[i>>6], (i>>6)<<6, setBit, i, i+63, lastRange) + rs = append(rs, appendArr...) + lastRange = last + } + + // get range of last uint64 + if last64MinIndex >= second64MinIndex { + appendArr, last := b.mergeAndFetchRangeOfUint64(b.bm[end>>6], (end>>6)<<6, setBit, last64MinIndex, end, lastRange) + rs = append(rs, appendArr...) + lastRange = last + } + + if lastRange != nil { + rs = append(rs, lastRange) + } + + return rs, nil +} + +// mergeAndFetchRangeOfUint64 will get range of x, and merge prv range if possible. +// return the array of ranges and last range which may merged by next uint64. +func (b *BitMap) mergeAndFetchRangeOfUint64(x uint64, base uint32, setBit bool, limitStart uint32, limitEnd uint32, prv *BitsRange) (appendArr []*BitsRange, last *BitsRange) { + var ( + start, end uint32 + startIndex, endIndex uint32 + out bool + ) + + appendArr = []*BitsRange{} + + if !setBit { + x = uint64(^x) + } + + for { + if x == 0 || out { + break + } + + // start is the value of the trailing zeros of x + start = uint32(Ctz64(x)) + // remove the trailing zeros of x and counts again + end = uint32(Ctz64(uint64(^(x >> start)))) + start + + x = (x >> end) << end + + startIndex = start + base + endIndex = end + base - 1 + + if endIndex < limitStart { + continue + } + + if startIndex > limitEnd { + continue + } + + if startIndex < limitStart { + startIndex = limitStart + } + + if endIndex > limitEnd { + out = true + endIndex = limitEnd + } + + if prv != nil { + if prv.EndIndex+1 == startIndex { + prv.EndIndex = endIndex + continue + } + appendArr = append(appendArr, prv) + } + + prv = &BitsRange{ + StartIndex: startIndex, + EndIndex: endIndex, + } + } + + return appendArr, prv +} + +func (b *BitMap) setWithoutLock(start uint32, end uint32, setBit bool) error { + if start > end { + return fmt.Errorf("start %d should not bigger than %d", start, end) + } + + if end > b.maxBitIndex { + return fmt.Errorf("end %d should not bigger than %d", end, b.maxBitIndex) + } + + second64MinIndex := ((start >> 6) + 1) << 6 + first64MaxIndex := end + if first64MaxIndex >= second64MinIndex { + first64MaxIndex = second64MinIndex - 1 + } + + for i := start; i <= first64MaxIndex; i++ { + if setBit { + b.bm[i>>6] = b.bm[i>>6] | lbm.Bit[i&63] + } else { + b.bm[i>>6] = b.bm[i>>6] & (^lbm.Bit[i&63]) + } + } + + last64MinIndex := (end >> 6) << 6 + if last64MinIndex < first64MaxIndex { + last64MinIndex = first64MaxIndex + 1 + } + + for i := second64MinIndex; i < last64MinIndex; i += 64 { + if setBit { + b.bm[i>>6] = math.MaxUint64 + } else { + b.bm[i>>6] = 0 + } + } + + for i := last64MinIndex; i <= end; i++ { + if setBit { + b.bm[i>>6] = b.bm[i>>6] | lbm.Bit[i&63] + } else { + b.bm[i>>6] = b.bm[i>>6] & (^lbm.Bit[i&63]) + } + } + + return nil +} + +func (b *BitMap) Encode() []byte { + b.lock.RLock() + defer b.lock.RUnlock() + + return EncodeUintArray(b.bm) +} + +// BitsRange shows the range of bitmap. +type BitsRange struct { + StartIndex uint32 + EndIndex uint32 +} + +// EncodeUintArray encodes []uint64 to bytes. +func EncodeUintArray(input []uint64) []byte { + arrLen := len(input) + data := make([]byte, arrLen*8) + + bytesIndex := 0 + for i := 0; i < arrLen; i++ { + binary.LittleEndian.PutUint64(data[bytesIndex:bytesIndex+8], input[i]) + bytesIndex += 8 + } + + return data[:bytesIndex] +} + +// DecodeToUintArray decodes input bytes to []uint64. +func DecodeToUintArray(data []byte) []uint64 { + var ( + bytesIndex int + ) + + arrLen := len(data) / 8 + out := make([]uint64, arrLen) + for i := 0; i < arrLen; i++ { + out[i] = binary.LittleEndian.Uint64(data[bytesIndex : bytesIndex+8]) + bytesIndex += 8 + } + + return out +} diff --git a/pkg/bitmap/bitmap_test.go b/pkg/bitmap/bitmap_test.go new file mode 100644 index 000000000..a3f042d9c --- /dev/null +++ b/pkg/bitmap/bitmap_test.go @@ -0,0 +1,259 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bitmap + +import ( + "fmt" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "testing" + + "github.com/go-check/check" +) + +func Test(t *testing.T) { + check.TestingT(t) +} + +type BitMapSuite struct { + tmpDir string +} + +func init() { + check.Suite(&BitMapSuite{}) +} + +func (suite *BitMapSuite) SetUpSuite(c *check.C) { + suite.tmpDir = "./testdata" + err := os.MkdirAll(suite.tmpDir, 0774) + c.Assert(err, check.IsNil) +} + +func (suite *BitMapSuite) TearDownSuite(c *check.C) { + if suite.tmpDir != "" { + os.RemoveAll(suite.tmpDir) + } +} + +func (suite *BitMapSuite) TestBitMap(c *check.C) { + // bits are in [0, 100 * 64 - 1] + bm, err := NewBitMap(100, false) + c.Assert(err, check.IsNil) + err = bm.Set(0, 100, true) + c.Assert(err, check.IsNil) + + var i uint32 + fmt.Printf("bp1, bm[0]: %x, bm[1]: %x\n", bm.bm[0], bm.bm[1]) + for i = 0; i <= 100; i++ { + rs, err := bm.Get(uint32(i), uint32(i), true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 1) + c.Assert(rs[0].StartIndex, check.Equals, i) + c.Assert(rs[0].EndIndex, check.Equals, i) + } + + var start, end uint32 + + // random 10000 to set [start, end] + for i = 0; i <= 1000; i++ { + n1 := uint32(rand.Int31n(101)) + n2 := uint32(rand.Int31n(101)) + if n1 < n2 { + start = n1 + end = n2 + } else { + start = n2 + end = n1 + } + + rs, err := bm.Get(uint32(start), uint32(end), true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 1) + c.Assert(rs[0].StartIndex, check.Equals, start) + c.Assert(rs[0].EndIndex, check.Equals, end) + } + + err = bm.Set(200, 300, true) + c.Assert(err, check.IsNil) + rs, err := bm.Get(0, 250, true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 2) + c.Assert(rs[0].StartIndex, check.Equals, uint32(0)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(100)) + c.Assert(rs[1].StartIndex, check.Equals, uint32(200)) + c.Assert(rs[1].EndIndex, check.Equals, uint32(250)) + + rs, err = bm.Get(0, 250, false) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 1) + c.Assert(rs[0].StartIndex, check.Equals, uint32(101)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(199)) + + err = bm.Set(100, 200, false) + c.Assert(err, check.IsNil) + rs, err = bm.Get(0, 250, false) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 1) + c.Assert(rs[0].StartIndex, check.Equals, uint32(100)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(200)) + + err = bm.Set(300, 303, true) + c.Assert(err, check.IsNil) + fmt.Printf("index: %d, bp1, bm[4]: %x, bm[5]: %x\n", -1, bm.bm[4], bm.bm[5]) + rs, err = bm.Get(200, 400, true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 1) + c.Assert(rs[0].StartIndex, check.Equals, uint32(201)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(303)) + + err = bm.Set(305, 308, true) + c.Assert(err, check.IsNil) + fmt.Printf("index: %d, bp1, bm[4]: %x, bm[5]: %x\n", -2, bm.bm[4], bm.bm[5]) + rs, err = bm.Get(200, 400, true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 2) + c.Assert(rs[0].StartIndex, check.Equals, uint32(201)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(303)) + c.Assert(rs[1].StartIndex, check.Equals, uint32(305)) + c.Assert(rs[1].EndIndex, check.Equals, uint32(308)) + + err = bm.Set(310, 313, true) + c.Assert(err, check.IsNil) + fmt.Printf("index: %d, bp1, bm[4]: %x, bm[5]: %x\n", -3, bm.bm[4], bm.bm[5]) + rs, err = bm.Get(200, 400, true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 3) + c.Assert(rs[0].StartIndex, check.Equals, uint32(201)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(303)) + c.Assert(rs[1].StartIndex, check.Equals, uint32(305)) + c.Assert(rs[1].EndIndex, check.Equals, uint32(308)) + c.Assert(rs[2].StartIndex, check.Equals, uint32(310)) + c.Assert(rs[2].EndIndex, check.Equals, uint32(313)) + + err = bm.Set(315, 318, true) + c.Assert(err, check.IsNil) + fmt.Printf("index: %d, bp1, bm[4]: %x, bm[5]: %x\n", -4, bm.bm[4], bm.bm[5]) + rs, err = bm.Get(200, 400, true) + c.Assert(err, check.IsNil) + c.Assert(len(rs), check.Equals, 4) + c.Assert(rs[0].StartIndex, check.Equals, uint32(201)) + c.Assert(rs[0].EndIndex, check.Equals, uint32(303)) + c.Assert(rs[1].StartIndex, check.Equals, uint32(305)) + c.Assert(rs[1].EndIndex, check.Equals, uint32(308)) + c.Assert(rs[2].StartIndex, check.Equals, uint32(310)) + c.Assert(rs[2].EndIndex, check.Equals, uint32(313)) + c.Assert(rs[3].StartIndex, check.Equals, uint32(315)) + c.Assert(rs[3].EndIndex, check.Equals, uint32(318)) + + for i := 0; i < 50; i++ { + err = bm.Set(uint32(300+i*5), uint32(300+i*5+3), true) + c.Assert(err, check.IsNil) + } + + rs, err = bm.Get(0, 1000, true) + c.Assert(err, check.IsNil) + c.Check(len(rs), check.Equals, 51) + c.Check(rs[0].StartIndex, check.Equals, uint32(0)) + c.Check(rs[0].EndIndex, check.Equals, uint32(99)) + c.Check(rs[1].StartIndex, check.Equals, uint32(201)) + c.Check(rs[1].EndIndex, check.Equals, uint32(303)) + + for i := 0; i < 49; i++ { + c.Check(rs[i+2].StartIndex, check.Equals, uint32(305+i*5)) + c.Check(rs[i+2].EndIndex, check.Equals, uint32(308+i*5)) + } +} + +func (suite *BitMapSuite) TestRestoreBitMap(c *check.C) { + bm, err := NewBitMap(100, false) + c.Assert(err, check.IsNil) + err = bm.Set(0, 10, true) + c.Assert(err, check.IsNil) + err = bm.Set(100, 200, true) + c.Assert(err, check.IsNil) + res, err := bm.Get(0, 100*64-1, true) + fmt.Println(res) + c.Assert(err, check.IsNil) + c.Assert(len(res), check.Equals, 2) + c.Assert(res[0].StartIndex, check.Equals, uint32(0)) + c.Assert(res[0].EndIndex, check.Equals, uint32(10)) + c.Assert(res[1].StartIndex, check.Equals, uint32(100)) + c.Assert(res[1].EndIndex, check.Equals, uint32(200)) + + bmPath := filepath.Join(suite.tmpDir, "TestRestoreBitMap.bits") + data := bm.Encode() + + err = ioutil.WriteFile(bmPath, data, 0644) + c.Assert(err, check.IsNil) + + readData, err := ioutil.ReadFile(bmPath) + c.Assert(err, check.IsNil) + bm1, err := RestoreBitMap(readData) + c.Assert(err, check.IsNil) + c.Assert(bm1.maxBitIndex, check.Equals, uint32(100*64-1)) + res, err = bm1.Get(0, 100*64-1, true) + c.Assert(err, check.IsNil) + c.Assert(len(res), check.Equals, 2) + c.Assert(res[0].StartIndex, check.Equals, uint32(0)) + c.Assert(res[0].EndIndex, check.Equals, uint32(10)) + c.Assert(res[1].StartIndex, check.Equals, uint32(100)) + c.Assert(res[1].EndIndex, check.Equals, uint32(200)) +} + +func (suite *BitMapSuite) TestInvalidBitMap(c *check.C) { + _, err := NewBitMap(sizeOf64BitsLimit+1, false) + c.Assert(err, check.NotNil) + + bm1, err := NewBitMap(100000, true) + c.Assert(err, check.IsNil) + + err = bm1.Set(0, bm1.maxBitIndex, true) + c.Assert(err, check.IsNil) + + err = bm1.Set(1, 1, true) + c.Assert(err, check.IsNil) + + err = bm1.Set(1000, 0, true) + c.Assert(err, check.NotNil) + + err = bm1.Set(1000, bm1.maxBitIndex+1, true) + c.Assert(err, check.NotNil) + + _, err = bm1.Get(0, bm1.maxBitIndex, true) + c.Assert(err, check.IsNil) + + _, err = bm1.Get(1, 1, true) + c.Assert(err, check.IsNil) + + _, err = bm1.Get(1000, 0, true) + c.Assert(err, check.NotNil) + + _, err = bm1.Get(1000, bm1.maxBitIndex+1, true) + c.Assert(err, check.NotNil) + + // test restore + data := make([]byte, sizeOf64BitsLimit<<3) + _, err = RestoreBitMap(data) + c.Assert(err, check.IsNil) + + data1 := make([]byte, (sizeOf64BitsLimit<<3)+1) + _, err = RestoreBitMap(data1) + c.Assert(err, check.NotNil) + +} diff --git a/pkg/bitmap/utils.go b/pkg/bitmap/utils.go new file mode 100644 index 000000000..a0985d35d --- /dev/null +++ b/pkg/bitmap/utils.go @@ -0,0 +1,40 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bitmap + +const deBruijn64 = 0x0218a392cd3d5dbf + +var deBruijnIdx64 = [64]byte{ + 0, 1, 2, 7, 3, 13, 8, 19, + 4, 25, 14, 28, 9, 34, 20, 40, + 5, 17, 26, 38, 15, 46, 29, 48, + 10, 31, 35, 54, 21, 50, 41, 57, + 63, 6, 12, 18, 24, 27, 33, 39, + 16, 37, 45, 47, 30, 53, 49, 56, + 62, 11, 23, 32, 36, 44, 52, 55, + 61, 22, 43, 51, 60, 42, 59, 58, +} + +// Ctz64 counts trailing (low-order) zeroes, +// and if all are zero, then 64. +func Ctz64(x uint64) int { + x &= -x // isolate low-order bit + y := x * deBruijn64 >> 58 // extract part of deBruijn sequence + i := int(deBruijnIdx64[y]) // convert to bit index + z := int((x - 1) >> 57 & 64) // adjustment if zero + return i + z +}