Skip to content

Commit

Permalink
Closes: Feature: add ability to delete apps #119 (#221)
Browse files Browse the repository at this point in the history 
* Feature: add ability to delete apps #119

* More tests

* Added proper delete, but waiting for lfu-go fork to be updated.

* Use lfu-go from fork, still temporary solution

* replaced closingMutex with a RWMutex

Co-authored-by: Dmitry Filimonov <dmitry@pyroscope.io>
  • Loading branch information
@AdrK @petethepig
AdrK and petethepig committed Jun 3, 2021
1 parent 99c560a commit 2cc5f0e
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 9 deletions.
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,5 @@ require (
)

replace github.com/mgechev/revive v1.0.3 => github.com/pyroscope-io/revive v1.0.6-0.20210330033039-4a71146f9dc1

replace github.com/dgrijalva/lfu-go => github.com/pyroscope-io/lfu-go v1.0.1
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,6 @@ github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDm
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgrijalva/lfu-go v0.0.0-20141010002404-f174e76c5138 h1:dnHgW/+2AbIFKEIhb3mXw6OuPNPrMmrJ6GZylDT556s=
github.com/dgrijalva/lfu-go v0.0.0-20141010002404-f174e76c5138/go.mod h1:NBHEfzv9sqF3bQdv4L3+buvhN5JtVkuO1EhpELjZxXk=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
Expand Down Expand Up @@ -370,6 +368,8 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/pyroscope-io/dotnetdiag v1.1.0 h1:BIo2cIAH+/iXQgbgUfpLil3E9ODY2HcnBBwQ4smjgAU=
github.com/pyroscope-io/dotnetdiag v1.1.0/go.mod h1:mUudCmW+j2ewS55/FHq0lHj0Xy2AHWc1UhaaRdfSve8=
github.com/pyroscope-io/lfu-go v1.0.1 h1:yGE9tbsAYTr+Bb5KL25WJOh7FX0bnb01MDgIqPvxPAY=
github.com/pyroscope-io/lfu-go v1.0.1/go.mod h1:3W9sGrDLhKFkHZPXkz6c5dAKrxcwkKbFFKnJtDukMDA=
github.com/pyroscope-io/revive v1.0.6-0.20210330033039-4a71146f9dc1 h1:0v9lBNgdmVtpyyk9PP/DfpJlOHkXriu5YgNlrhQw5YE=
github.com/pyroscope-io/revive v1.0.6-0.20210330033039-4a71146f9dc1/go.mod h1:tSw34BaGZ0iF+oVKDOjq1/LuxGifgW7shaJ6+dBYFXg=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
Expand Down
10 changes: 10 additions & 0 deletions pkg/storage/cache/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,16 @@ func (cache *Cache) Flush() {
<-cache.cleanupDone
}

func (cache *Cache) Delete(key string) error {
cache.lfu.Delete(key)

err := cache.db.Update(func(txn *badger.Txn) error {
return txn.Delete([]byte(cache.prefix + key))
})

return err
}

func (cache *Cache) Get(key string) (interface{}, error) {
// find the key from cache first
if cache.lfu.UpperBound > 0 {
Expand Down
71 changes: 64 additions & 7 deletions pkg/storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ var errClosing = errors.New("the db is in closing state")
var errOutOfSpace = errors.New("running out of space")

type Storage struct {
closingMutex sync.Mutex
closingMutex sync.RWMutex
closing bool

cfg *config.Server
Expand Down Expand Up @@ -193,9 +193,8 @@ type PutInput struct {
}

func (s *Storage) Put(po *PutInput) error {
s.closingMutex.Lock()
defer s.closingMutex.Unlock()

s.closingMutex.RLock()
defer s.closingMutex.RUnlock()
if s.closing {
return errClosing
}
Expand Down Expand Up @@ -292,9 +291,8 @@ type GetOutput struct {
}

func (s *Storage) Get(gi *GetInput) (*GetOutput, error) {
s.closingMutex.Lock()
defer s.closingMutex.Unlock()

s.closingMutex.RLock()
defer s.closingMutex.RUnlock()
if s.closing {
return nil, errClosing
}
Expand Down Expand Up @@ -388,6 +386,65 @@ func (s *Storage) Get(gi *GetInput) (*GetOutput, error) {
}, nil
}

type DeleteInput struct {
StartTime time.Time
EndTime time.Time
Key *Key
}

func (s *Storage) Delete(di *DeleteInput) error {
s.closingMutex.RLock()
defer s.closingMutex.RUnlock()
if s.closing {
return errClosing
}

logrus.WithFields(logrus.Fields{
"startTime": di.StartTime.String(),
"endTime": di.EndTime.String(),
"key": di.Key.Normalized(),
}).Info("storage.Delete")

dimensions := []*dimension.Dimension{}
for k, v := range di.Key.labels {
dInt, err := s.dimensions.Get(k + ":" + v)
if err != nil {
return nil
}
d := dInt.(*dimension.Dimension)
dimensions = append(dimensions, d)
}

segmentKeys := dimension.Intersection(dimensions...)

for _, sk := range segmentKeys {
// TODO: refactor, store `Key`s in dimensions
skk, _ := ParseKey(string(sk))
stInt, err := s.segments.Get(skk.SegmentKey())
if err != nil {
return nil
}
st := stInt.(*segment.Segment)
if st == nil {
continue
}

st.Get(di.StartTime, di.EndTime, func(depth int, samples, writes uint64, t time.Time, r *big.Rat) {
k := skk.TreeKey(depth, t)
s.trees.Delete(k)
s.dicts.Delete(FromTreeToMainKey(k))
})

s.segments.Delete(skk.SegmentKey())
}

for k, v := range di.Key.labels {
s.dimensions.Delete(k + ":" + v)
}

return nil
}

func (s *Storage) Close() error {
s.closingMutex.Lock()
s.closing = true
Expand Down
142 changes: 142 additions & 0 deletions pkg/storage/storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,148 @@ var _ = Describe("storage package", func() {
Expect(err).ToNot(HaveOccurred())
})

Context("delete tests", func() {
Context("simple delete", func() {
It("works correctly", func() {
tree := tree.New()
tree.Insert([]byte("a;b"), uint64(1))
tree.Insert([]byte("a;c"), uint64(2))
st := testing.SimpleTime(10)
et := testing.SimpleTime(19)
st2 := testing.SimpleTime(0)
et2 := testing.SimpleTime(30)
key, _ := ParseKey("foo")

s.Put(&PutInput{
StartTime: st,
EndTime: et,
Key: key,
Val: tree,
SpyName: "testspy",
SampleRate: 100,
})

err := s.Delete(&DeleteInput{
StartTime: st,
EndTime: et,
Key: key,
})
Expect(err).ToNot(HaveOccurred())

gOut, err := s.Get(&GetInput{
StartTime: st2,
EndTime: et2,
Key: key,
})

Expect(err).ToNot(HaveOccurred())
Expect(gOut).To(BeNil())
Expect(s.Close()).ToNot(HaveOccurred())
})
})
Context("delete all trees", func() {
It("works correctly", func() {
tree1 := tree.New()
tree1.Insert([]byte("a;b"), uint64(1))
tree1.Insert([]byte("a;c"), uint64(2))
tree2 := tree.New()
tree2.Insert([]byte("c;d"), uint64(1))
tree2.Insert([]byte("e;f"), uint64(2))
st := testing.SimpleTime(10)
et := testing.SimpleTime(19)
st2 := testing.SimpleTime(0)
et2 := testing.SimpleTime(30)
key, _ := ParseKey("foo")

s.Put(&PutInput{
StartTime: st,
EndTime: et,
Key: key,
Val: tree1,
SpyName: "testspy",
SampleRate: 100,
})

s.Put(&PutInput{
StartTime: st,
EndTime: et,
Key: key,
Val: tree2,
SpyName: "testspy",
SampleRate: 100,
})

err := s.Delete(&DeleteInput{
StartTime: st,
EndTime: et,
Key: key,
})
Expect(err).ToNot(HaveOccurred())

gOut, err := s.Get(&GetInput{
StartTime: st2,
EndTime: et2,
Key: key,
})
Expect(err).ToNot(HaveOccurred())
Expect(gOut).To(BeNil())
Expect(s.Close()).ToNot(HaveOccurred())
})
})
Context("put after delete", func() {
It("works correctly", func() {
tree1 := tree.New()
tree1.Insert([]byte("a;b"), uint64(1))
tree1.Insert([]byte("a;c"), uint64(2))
tree2 := tree.New()
tree2.Insert([]byte("c;d"), uint64(1))
tree2.Insert([]byte("e;f"), uint64(2))
st := testing.SimpleTime(10)
et := testing.SimpleTime(19)
st2 := testing.SimpleTime(0)
et2 := testing.SimpleTime(30)
key, _ := ParseKey("foo")

err := s.Put(&PutInput{
StartTime: st,
EndTime: et,
Key: key,
Val: tree1,
SpyName: "testspy",
SampleRate: 100,
})
Expect(err).ToNot(HaveOccurred())

err = s.Delete(&DeleteInput{
StartTime: st,
EndTime: et,
Key: key,
})
Expect(err).ToNot(HaveOccurred())

s.Put(&PutInput{
StartTime: st,
EndTime: et,
Key: key,
Val: tree2,
SpyName: "testspy",
SampleRate: 100,
})

gOut, err := s.Get(&GetInput{
StartTime: st2,
EndTime: et2,
Key: key,
})

Expect(err).ToNot(HaveOccurred())
Expect(gOut.Tree).ToNot(BeNil())
Expect(gOut.Tree.String()).To(Equal(tree2.String()))
Expect(s.Close()).ToNot(HaveOccurred())
})
})
})

Context("smoke tests", func() {
Context("check segment cache", func() {
It("works correctly", func() {
Expand Down

0 comments on commit 2cc5f0e

Please sign in to comment.