Skip to content

Commit

Permalink
Merge df01c19 into 18698ef
Browse files Browse the repository at this point in the history
  • Loading branch information
@darklynx
darklynx committed Feb 22, 2019
2 parents 18698ef + df01c19 commit ea0d19e
Show file tree
Hide file tree
Showing 22 changed files with 1,253 additions and 634 deletions.
82 changes: 82 additions & 0 deletions baskets.go
View file
Expand Up @@ -75,6 +75,26 @@ type BasketNamesQueryPage struct {
HasMore bool `json:"has_more"`
}

// DatabaseStats describes collected statistics of a baskets database
type DatabaseStats struct {
BasketsCount int `json:"baskets_count"`
EmptyBasketsCount int `json:"empty_baskets_count"`
RequestsCount int `json:"requests_count"`
RequestsTotalCount int `json:"requests_total_count"`
MaxBasketSize int `json:"max_basket_size"`
AvgBasketSize int `json:"avg_basket_size"`
TopBasketsBySize []*BasketInfo `json:"top_baskets_size"`
TopBasketsByDate []*BasketInfo `json:"top_baskets_recent"`
}

// BasketInfo describes shorlty a basket for database statistics
type BasketInfo struct {
Name string `json:"name"`
RequestsCount int `json:"requests_count"`
RequestsTotalCount int `json:"requests_total_count"`
LastRequestDate int64 `json:"last_request_date"`
}

// Basket is an interface that represent request basket entity to collects HTTP requests
type Basket interface {
Config() BasketConfig
Expand Down Expand Up @@ -102,6 +122,8 @@ type BasketsDatabase interface {
GetNames(max int, skip int) BasketNamesPage
FindNames(query string, max int, skip int) BasketNamesQueryPage

GetStats(max int) DatabaseStats

Release()
}

Expand Down Expand Up @@ -233,3 +255,63 @@ func (req *RequestData) Matches(query string, in string) bool {

return false
}

// Collect collects information about basket and updates statistics
func (stats *DatabaseStats) Collect(basket *BasketInfo, max int) {
stats.BasketsCount++
if basket.RequestsTotalCount == 0 {
stats.EmptyBasketsCount++
}

stats.RequestsCount += basket.RequestsCount
stats.RequestsTotalCount += basket.RequestsTotalCount
if basket.RequestsTotalCount > stats.MaxBasketSize {
stats.MaxBasketSize = basket.RequestsTotalCount
}

// top baskets by size
stats.TopBasketsBySize = collectConditionally(stats.TopBasketsBySize, basket, max,
func(b1 *BasketInfo, b2 *BasketInfo) bool {
return b1.RequestsTotalCount > b2.RequestsTotalCount
})

// top baskets by recent activity
stats.TopBasketsByDate = collectConditionally(stats.TopBasketsByDate, basket, max,
func(b1 *BasketInfo, b2 *BasketInfo) bool {
return b1.LastRequestDate > b2.LastRequestDate
})
}

func collectConditionally(col []*BasketInfo, basket *BasketInfo, size int,
greater func(*BasketInfo, *BasketInfo) bool) []*BasketInfo {
if col == nil {
col = make([]*BasketInfo, 0, size)
return append(col, basket)
}

for i, b := range col {
if greater(basket, b) {
if len(col) < size {
col = append(col, nil)
}
copy(col[i+1:], col[i:])
col[i] = basket
return col
}
}

if len(col) < size {
return append(col, basket)
}

return col
}

// UpdateAvarage updates avarage statistics counters.
func (stats *DatabaseStats) UpdateAvarage() {
if stats.BasketsCount > stats.EmptyBasketsCount {
stats.AvgBasketSize = stats.RequestsTotalCount / (stats.BasketsCount - stats.EmptyBasketsCount)
} else {
stats.AvgBasketSize = 0
}
}
29 changes: 29 additions & 0 deletions baskets_bolt.go
View file
Expand Up @@ -475,6 +475,35 @@ func (bdb *boltDatabase) FindNames(query string, max int, skip int) BasketNamesQ
return page
}

func (bdb *boltDatabase) GetStats(max int) DatabaseStats {
stats := DatabaseStats{}

bdb.db.View(func(tx *bolt.Tx) error {
cur := tx.Cursor()
for key, _ := cur.First(); key != nil; key, _ = cur.Next() {
if b := tx.Bucket(key); b != nil {
var lastRequestDate int64
if _, val := b.Bucket(boltKeyRequests).Cursor().Last(); val != nil {
request := new(RequestData)
if err := json.Unmarshal(val, request); err == nil {
lastRequestDate = request.Date
}
}

stats.Collect(&BasketInfo{
Name: string(key),
RequestsCount: btoi(b.Get(boltKeyCount)),
RequestsTotalCount: btoi(b.Get(boltKeyTotalCount)),
LastRequestDate: lastRequestDate}, max)
}
}
return nil
})

stats.UpdateAvarage()
return stats
}

func (bdb *boltDatabase) Release() {
log.Print("[info] closing Bolt database")
err := bdb.db.Close()
Expand Down
48 changes: 48 additions & 0 deletions baskets_bolt_test.go
View file
Expand Up @@ -444,6 +444,54 @@ func TestBoltBasket_SetResponse_Update(t *testing.T) {
}
}

func TestBoltDatabase_GetStats(t *testing.T) {
name := "test130"
db := NewBoltDatabase(name + ".db")
defer db.Release()
defer os.Remove(name + ".db")

config := BasketConfig{Capacity: 5}
for i := 0; i < 10; i++ {
bname := fmt.Sprintf("%s_%v", name, i)
db.Create(bname, config)

// fill basket
basket := db.Get(bname)
for j := 0; j < 9-i; j++ {
basket.Add(createTestPOSTRequest(
fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain"))
}
time.Sleep(20 * time.Millisecond)
}

// get stats
stats := db.GetStats(3)
if assert.NotNil(t, stats, "database statistics is expected") {
assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats")
assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats")
assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats")
assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats")
assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats")
assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats")

// top 3 by date
if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") {
assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets")
assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name)
}

// top 3 by size
if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") {
assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets")
assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name)
}
}
}

func TestBoltBasket_InvalidBasket(t *testing.T) {
name := "test199"
db, _ := bolt.Open(name+".db", 0600, &bolt.Options{Timeout: 5 * time.Second})
Expand Down
25 changes: 25 additions & 0 deletions baskets_mem.go
View file
Expand Up @@ -256,6 +256,31 @@ func (db *memoryDatabase) FindNames(query string, max int, skip int) BasketNames
return BasketNamesQueryPage{Names: result, HasMore: false}
}

func (db *memoryDatabase) GetStats(max int) DatabaseStats {
db.RLock()
defer db.RUnlock()

stats := DatabaseStats{}

for _, name := range db.names {
if basket, exists := db.baskets[name]; exists {
var lastRequestDate int64
if basket.Size() > 0 {
lastRequestDate = basket.GetRequests(1, 0).Requests[0].Date
}

stats.Collect(&BasketInfo{
Name: name,
RequestsCount: basket.Size(),
RequestsTotalCount: basket.totalCount,
LastRequestDate: lastRequestDate}, max)
}
}

stats.UpdateAvarage()
return stats
}

func (db *memoryDatabase) Release() {
log.Print("[info] releasing in-memory database resources")
}
Expand Down
48 changes: 48 additions & 0 deletions baskets_mem_test.go
View file
Expand Up @@ -7,6 +7,7 @@ import (
"net/url"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -400,3 +401,50 @@ func TestMemoryBasket_SetResponse_Update(t *testing.T) {
}
}
}

func TestMemoryDatabase_GetStats(t *testing.T) {
name := "test130"
db := NewMemoryDatabase()
defer db.Release()

config := BasketConfig{Capacity: 5}
for i := 0; i < 10; i++ {
bname := fmt.Sprintf("%s_%v", name, i)
db.Create(bname, config)

// fill basket
basket := db.Get(bname)
for j := 0; j < 9-i; j++ {
basket.Add(createTestPOSTRequest(
fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain"))
}
time.Sleep(20 * time.Millisecond)
}

// get stats
stats := db.GetStats(3)
if assert.NotNil(t, stats, "database statistics is expected") {
assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats")
assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats")
assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats")
assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats")
assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats")
assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats")

// top 3 by date
if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") {
assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets")
assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name)
}

// top 3 by size
if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") {
assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets")
assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name)
assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name)
}
}
}

0 comments on commit ea0d19e

Please sign in to comment.