diff --git a/baskets.go b/baskets.go index 5f959ba..c6d2a12 100644 --- a/baskets.go +++ b/baskets.go @@ -75,6 +75,26 @@ type BasketNamesQueryPage struct { HasMore bool `json:"has_more"` } +// DatabaseStats describes collected statistics of a baskets database +type DatabaseStats struct { + BasketsCount int `json:"baskets_count"` + EmptyBasketsCount int `json:"empty_baskets_count"` + RequestsCount int `json:"requests_count"` + RequestsTotalCount int `json:"requests_total_count"` + MaxBasketSize int `json:"max_basket_size"` + AvgBasketSize int `json:"avg_basket_size"` + TopBasketsBySize []*BasketInfo `json:"top_baskets_size"` + TopBasketsByDate []*BasketInfo `json:"top_baskets_recent"` +} + +// BasketInfo describes shorlty a basket for database statistics +type BasketInfo struct { + Name string `json:"name"` + RequestsCount int `json:"requests_count"` + RequestsTotalCount int `json:"requests_total_count"` + LastRequestDate int64 `json:"last_request_date"` +} + // Basket is an interface that represent request basket entity to collects HTTP requests type Basket interface { Config() BasketConfig @@ -102,6 +122,8 @@ type BasketsDatabase interface { GetNames(max int, skip int) BasketNamesPage FindNames(query string, max int, skip int) BasketNamesQueryPage + GetStats(max int) DatabaseStats + Release() } @@ -233,3 +255,63 @@ func (req *RequestData) Matches(query string, in string) bool { return false } + +// Collect collects information about basket and updates statistics +func (stats *DatabaseStats) Collect(basket *BasketInfo, max int) { + stats.BasketsCount++ + if basket.RequestsTotalCount == 0 { + stats.EmptyBasketsCount++ + } + + stats.RequestsCount += basket.RequestsCount + stats.RequestsTotalCount += basket.RequestsTotalCount + if basket.RequestsTotalCount > stats.MaxBasketSize { + stats.MaxBasketSize = basket.RequestsTotalCount + } + + // top baskets by size + stats.TopBasketsBySize = collectConditionally(stats.TopBasketsBySize, basket, max, + func(b1 *BasketInfo, b2 *BasketInfo) bool { + return b1.RequestsTotalCount > b2.RequestsTotalCount + }) + + // top baskets by recent activity + stats.TopBasketsByDate = collectConditionally(stats.TopBasketsByDate, basket, max, + func(b1 *BasketInfo, b2 *BasketInfo) bool { + return b1.LastRequestDate > b2.LastRequestDate + }) +} + +func collectConditionally(col []*BasketInfo, basket *BasketInfo, size int, + greater func(*BasketInfo, *BasketInfo) bool) []*BasketInfo { + if col == nil { + col = make([]*BasketInfo, 0, size) + return append(col, basket) + } + + for i, b := range col { + if greater(basket, b) { + if len(col) < size { + col = append(col, nil) + } + copy(col[i+1:], col[i:]) + col[i] = basket + return col + } + } + + if len(col) < size { + return append(col, basket) + } + + return col +} + +// UpdateAvarage updates avarage statistics counters. +func (stats *DatabaseStats) UpdateAvarage() { + if stats.BasketsCount > stats.EmptyBasketsCount { + stats.AvgBasketSize = stats.RequestsTotalCount / (stats.BasketsCount - stats.EmptyBasketsCount) + } else { + stats.AvgBasketSize = 0 + } +} diff --git a/baskets_bolt.go b/baskets_bolt.go index b73227f..c97f6b7 100644 --- a/baskets_bolt.go +++ b/baskets_bolt.go @@ -475,6 +475,35 @@ func (bdb *boltDatabase) FindNames(query string, max int, skip int) BasketNamesQ return page } +func (bdb *boltDatabase) GetStats(max int) DatabaseStats { + stats := DatabaseStats{} + + bdb.db.View(func(tx *bolt.Tx) error { + cur := tx.Cursor() + for key, _ := cur.First(); key != nil; key, _ = cur.Next() { + if b := tx.Bucket(key); b != nil { + var lastRequestDate int64 + if _, val := b.Bucket(boltKeyRequests).Cursor().Last(); val != nil { + request := new(RequestData) + if err := json.Unmarshal(val, request); err == nil { + lastRequestDate = request.Date + } + } + + stats.Collect(&BasketInfo{ + Name: string(key), + RequestsCount: btoi(b.Get(boltKeyCount)), + RequestsTotalCount: btoi(b.Get(boltKeyTotalCount)), + LastRequestDate: lastRequestDate}, max) + } + } + return nil + }) + + stats.UpdateAvarage() + return stats +} + func (bdb *boltDatabase) Release() { log.Print("[info] closing Bolt database") err := bdb.db.Close() diff --git a/baskets_bolt_test.go b/baskets_bolt_test.go index 0ec6e47..75c13f7 100644 --- a/baskets_bolt_test.go +++ b/baskets_bolt_test.go @@ -444,6 +444,54 @@ func TestBoltBasket_SetResponse_Update(t *testing.T) { } } +func TestBoltDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewBoltDatabase(name + ".db") + defer db.Release() + defer os.Remove(name + ".db") + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} + func TestBoltBasket_InvalidBasket(t *testing.T) { name := "test199" db, _ := bolt.Open(name+".db", 0600, &bolt.Options{Timeout: 5 * time.Second}) diff --git a/baskets_mem.go b/baskets_mem.go index dc1fae5..d80509e 100644 --- a/baskets_mem.go +++ b/baskets_mem.go @@ -256,6 +256,31 @@ func (db *memoryDatabase) FindNames(query string, max int, skip int) BasketNames return BasketNamesQueryPage{Names: result, HasMore: false} } +func (db *memoryDatabase) GetStats(max int) DatabaseStats { + db.RLock() + defer db.RUnlock() + + stats := DatabaseStats{} + + for _, name := range db.names { + if basket, exists := db.baskets[name]; exists { + var lastRequestDate int64 + if basket.Size() > 0 { + lastRequestDate = basket.GetRequests(1, 0).Requests[0].Date + } + + stats.Collect(&BasketInfo{ + Name: name, + RequestsCount: basket.Size(), + RequestsTotalCount: basket.totalCount, + LastRequestDate: lastRequestDate}, max) + } + } + + stats.UpdateAvarage() + return stats +} + func (db *memoryDatabase) Release() { log.Print("[info] releasing in-memory database resources") } diff --git a/baskets_mem_test.go b/baskets_mem_test.go index c0399be..c532176 100644 --- a/baskets_mem_test.go +++ b/baskets_mem_test.go @@ -7,6 +7,7 @@ import ( "net/url" "strings" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -400,3 +401,50 @@ func TestMemoryBasket_SetResponse_Update(t *testing.T) { } } } + +func TestMemoryDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewMemoryDatabase() + defer db.Release() + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} diff --git a/baskets_sql.go b/baskets_sql.go index c0c777e..56972f0 100644 --- a/baskets_sql.go +++ b/baskets_sql.go @@ -8,6 +8,7 @@ import ( "net/http" "regexp" "strings" + "time" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" @@ -87,6 +88,21 @@ func (basket *sqlBasket) applyLimit(capacity int) { } } +func (basket *sqlBasket) getTotalRequestsCount() int { + return basket.getInt("SELECT requests_count FROM rb_baskets WHERE basket_name = $1", 0) +} + +func (basket *sqlBasket) getLastRequestDate() int64 { + var value time.Time + if err := basket.db.QueryRow(unifySQL(basket.dbType, + "SELECT MAX(created_at) FROM rb_requests WHERE basket_name = $1"), basket.name).Scan(&value); err != nil { + log.Printf("[error] failed to get last request date of basket: %s - %s", basket.name, err) + return 0 + } + + return value.UnixNano() / toMs +} + func (basket *sqlBasket) Config() BasketConfig { config := BasketConfig{} @@ -198,8 +214,7 @@ func (basket *sqlBasket) Size() int { } func (basket *sqlBasket) GetRequests(max int, skip int) RequestsPage { - page := RequestsPage{make([]*RequestData, 0, max), basket.Size(), - basket.getInt("SELECT requests_count FROM rb_baskets WHERE basket_name = $1", 0), false} + page := RequestsPage{make([]*RequestData, 0, max), basket.Size(), basket.getTotalRequestsCount(), false} if max > 0 { requests, err := basket.db.Query( @@ -213,9 +228,7 @@ func (basket *sqlBasket) GetRequests(max int, skip int) RequestsPage { var req string for len(page.Requests) < max && requests.Next() { - if err = requests.Scan(&req); err != nil { - log.Printf("[error] failed to get request of basket: %s - %s", basket.name, err) - } else { + if err = requests.Scan(&req); err == nil { request := new(RequestData) if err = json.Unmarshal([]byte(req), request); err != nil { log.Printf("[error] failed to parse HTTP request data in basket: %s - %s", basket.name, err) @@ -247,9 +260,7 @@ func (basket *sqlBasket) FindRequests(query string, in string, max int, skip int skipped := 0 var req string for len(page.Requests) < max && requests.Next() { - if err = requests.Scan(&req); err != nil { - log.Printf("[error] failed to get request of basket: %s - %s", basket.name, err) - } else { + if err = requests.Scan(&req); err == nil { request := new(RequestData) if err = json.Unmarshal([]byte(req), request); err != nil { log.Printf("[error] failed to parse HTTP request data in basket: %s - %s", basket.name, err) @@ -280,6 +291,47 @@ type sqlDatabase struct { dbType string // postgresql, mysql, oracle, etc. } +func (sdb *sqlDatabase) getInt(sql string, defaultValue int) int { + var value int + if err := sdb.db.QueryRow(sql).Scan(&value); err != nil { + log.Printf("[error] failed to query for int result, query: %s - %s", sql, err) + return defaultValue + } + + return value +} + +func (sdb *sqlDatabase) getTopBaskets(sql string, max int) []*BasketInfo { + top := make([]*BasketInfo, 0, max) + names, err := sdb.db.Query(unifySQL(sdb.dbType, sql), max) + if err != nil { + log.Printf("[error] failed to find top baskets: %s", err) + return top + } + defer names.Close() + + var name string + for names.Next() { + if err = names.Scan(&name); err == nil { + basket := &sqlBasket{sdb.db, sdb.dbType, name} + reqCount := basket.Size() + + var lastRequestDate int64 + if reqCount > 0 { + lastRequestDate = basket.getLastRequestDate() + } + + top = append(top, &BasketInfo{ + Name: name, + RequestsCount: reqCount, + RequestsTotalCount: basket.getTotalRequestsCount(), + LastRequestDate: lastRequestDate}) + } + } + + return top +} + func (sdb *sqlDatabase) Create(name string, config BasketConfig) (BasketAuth, error) { auth := BasketAuth{} token, err := GenerateToken() @@ -324,13 +376,7 @@ func (sdb *sqlDatabase) Delete(name string) { } func (sdb *sqlDatabase) Size() int { - var size int - if err := sdb.db.QueryRow("SELECT COUNT(*) FROM rb_baskets").Scan(&size); err != nil { - log.Printf("[error] failed to get the total number of baskets: %s", err) - return 0 - } - - return size + return sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) } func (sdb *sqlDatabase) GetNames(max int, skip int) BasketNamesPage { @@ -345,9 +391,7 @@ func (sdb *sqlDatabase) GetNames(max int, skip int) BasketNamesPage { var name string for len(page.Names) < max && names.Next() { - if err = names.Scan(&name); err != nil { - log.Printf("[error] failed to get basket name: %s", err) - } else { + if err = names.Scan(&name); err == nil { page.Names = append(page.Names, name) } } @@ -371,9 +415,7 @@ func (sdb *sqlDatabase) FindNames(query string, max int, skip int) BasketNamesQu var name string for len(page.Names) < max && names.Next() { - if err = names.Scan(&name); err != nil { - log.Printf("[error] failed to get basket name: %s", err) - } else { + if err = names.Scan(&name); err == nil { page.Names = append(page.Names, name) } } @@ -383,6 +425,22 @@ func (sdb *sqlDatabase) FindNames(query string, max int, skip int) BasketNamesQu return page } +func (sdb *sqlDatabase) GetStats(max int) DatabaseStats { + stats := DatabaseStats{} + + stats.BasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) + stats.EmptyBasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets WHERE requests_count = 0", 0) + stats.RequestsCount = sdb.getInt("SELECT COUNT(*) FROM rb_requests", 0) + stats.RequestsTotalCount = sdb.getInt("SELECT COALESCE(SUM(requests_count), 0) FROM rb_baskets", 0) + stats.MaxBasketSize = sdb.getInt("SELECT COALESCE(MAX(requests_count), 0) FROM rb_baskets", 0) + + stats.TopBasketsBySize = sdb.getTopBaskets("SELECT basket_name FROM rb_baskets ORDER BY requests_count DESC LIMIT $1", max) + stats.TopBasketsByDate = sdb.getTopBaskets("SELECT basket_name FROM rb_requests GROUP BY basket_name ORDER BY MAX(created_at) DESC LIMIT $1", max) + + stats.UpdateAvarage() + return stats +} + func (sdb *sqlDatabase) Release() { log.Printf("[info] closing SQL database, releasing any open resources") sdb.db.Close() diff --git a/baskets_sql_mysql_test.go b/baskets_sql_mysql_test.go index 4c12198..a4f1c95 100644 --- a/baskets_sql_mysql_test.go +++ b/baskets_sql_mysql_test.go @@ -465,3 +465,51 @@ func TestMySQLBasket_SetResponse_Error(t *testing.T) { assert.Nil(t, basket.GetResponse(method), "Response for very long method name is not expected") } } + +func TestMySQLDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewSQLDatabase(mysqlTestConnection) + defer db.Release() + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + defer db.Delete(bname) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} diff --git a/baskets_sql_pg_test.go b/baskets_sql_pg_test.go index 548042a..b1177fa 100644 --- a/baskets_sql_pg_test.go +++ b/baskets_sql_pg_test.go @@ -465,3 +465,51 @@ func TestPgSQLBasket_SetResponse_Error(t *testing.T) { assert.Nil(t, basket.GetResponse(method), "Response for very long method name is not expected") } } + +func TestPgSQLDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewSQLDatabase(pgTestConnection) + defer db.Release() + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + defer db.Delete(bname) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} diff --git a/baskets_test.go b/baskets_test.go index 5f59fdb..ce14a79 100644 --- a/baskets_test.go +++ b/baskets_test.go @@ -139,3 +139,50 @@ func TestExpandURL(t *testing.T) { assert.Equal(t, "/notify/hello/world", expandURL("/notify", "/notify/hello/world", "notify")) assert.Equal(t, "/receive/notification/test/", expandURL("/receive/notification/", "/basket/test/", "basket")) } + +func TestDatabaseStats_Collect(t *testing.T) { + stats := new(DatabaseStats) + stats.Collect(&BasketInfo{"a", 5, 10, 100}, 3) + stats.Collect(&BasketInfo{"b", 5, 30, 200}, 3) + stats.Collect(&BasketInfo{"c", 5, 5, 300}, 3) + stats.Collect(&BasketInfo{"d", 0, 0, 400}, 3) + stats.Collect(&BasketInfo{"e", 5, 20, 500}, 3) + stats.Collect(&BasketInfo{"f", 10, 40, 600}, 3) + stats.Collect(&BasketInfo{"g", 0, 0, 700}, 3) + stats.Collect(&BasketInfo{"h", 5, 5, 800}, 3) + + assert.Equal(t, 8, stats.BasketsCount, "wrong BasketsCount") + assert.Equal(t, 2, stats.EmptyBasketsCount, "wrong EmptyBasketsCount") + assert.Equal(t, 40, stats.MaxBasketSize, "wrong MaxBasketSize") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount") + assert.Equal(t, 110, stats.RequestsTotalCount, "wrong RequestsTotalCount") + + assert.Equal(t, 3, len(stats.TopBasketsByDate), "wrong number of TopBasketsByDate") + assert.Equal(t, "h", stats.TopBasketsByDate[0].Name) + assert.Equal(t, "g", stats.TopBasketsByDate[1].Name) + assert.Equal(t, "f", stats.TopBasketsByDate[2].Name) + + assert.Equal(t, 3, len(stats.TopBasketsBySize), "wrong number of TopBasketsBySize") + assert.Equal(t, "f", stats.TopBasketsBySize[0].Name) + assert.Equal(t, "b", stats.TopBasketsBySize[1].Name) + assert.Equal(t, "e", stats.TopBasketsBySize[2].Name) + + // we do not expect avarage basket size, it is not updated automatically + assert.Equal(t, 0, stats.AvgBasketSize, "unexpected AvgBasketSize") +} + +func TestDatabaseStats_UpdateAvarage(t *testing.T) { + stats := new(DatabaseStats) + stats.Collect(&BasketInfo{"a", 5, 10, 100}, 3) + stats.Collect(&BasketInfo{"b", 5, 20, 200}, 3) + stats.Collect(&BasketInfo{"c", 5, 30, 300}, 3) + + stats.UpdateAvarage() + assert.Equal(t, 20, stats.AvgBasketSize, "wrong AvgBasketSize") +} + +func TestDatabaseStats_UpdateAvarage_Empty(t *testing.T) { + stats := new(DatabaseStats) + stats.UpdateAvarage() + assert.Equal(t, 0, stats.AvgBasketSize, "wrong AvgBasketSize") +} diff --git a/config.go b/config.go index 502fd77..5930d2c 100644 --- a/config.go +++ b/config.go @@ -13,9 +13,12 @@ const ( initBasketCapacity = 200 maxBasketCapacity = 2000 defaultDatabaseType = DbTypeMemory - serviceAPIPath = "baskets" + serviceOldAPIPath = "baskets" + serviceAPIPath = "api" serviceUIPath = "web" + serviceName = "request-baskets" basketNamePattern = `^[\w\d\-_\.]{1,250}$` + sourceCodeURL = "https://github.com/darklynx/request-baskets" ) // ServerConfig describes server configuration. diff --git a/doc/api-swagger.json b/doc/api-swagger.json deleted file mode 100644 index 8d55831..0000000 --- a/doc/api-swagger.json +++ /dev/null @@ -1,538 +0,0 @@ -{ - "swagger" : "2.0", - "info" : { - "description" : "RESTful API of [Request Baskets](https://rbaskets.in) service.\n\nRequest Baskets is an open source project of a service to collect HTTP requests and inspect them via RESTful\nAPI or web UI.\n\nCheck out the [project page](https://github.com/darklynx/request-baskets) for more detailed description.\n", - "version" : "0.8", - "title" : "Request Baskets API", - "contact" : { - "name" : "darklynx", - "url" : "https://github.com/darklynx" - }, - "license" : { - "name" : "MIT", - "url" : "https://github.com/darklynx/request-baskets/blob/master/LICENSE" - } - }, - "host" : "rbaskets.in", - "basePath" : "/", - "tags" : [ { - "name" : "baskets", - "description" : "Manage baskets" - }, { - "name" : "responses", - "description" : "Configure basket responses" - }, { - "name" : "requests", - "description" : "Manage collected requests" - } ], - "schemes" : [ "https" ], - "consumes" : [ "application/json" ], - "produces" : [ "application/json" ], - "paths" : { - "/baskets" : { - "get" : { - "tags" : [ "baskets" ], - "summary" : "Get baskets", - "description" : "Fetches a list of basket names managed by service. Require master token.", - "parameters" : [ { - "name" : "max", - "in" : "query", - "description" : "Maximum number of basket names to return; default 20", - "required" : false, - "type" : "integer" - }, { - "name" : "skip", - "in" : "query", - "description" : "Number of basket names to skip; default 0", - "required" : false, - "type" : "integer" - }, { - "name" : "q", - "in" : "query", - "description" : "Query string to filter result, only those basket names that match the query will be included in response", - "required" : false, - "type" : "string" - } ], - "responses" : { - "200" : { - "description" : "OK. Returns list of available baskets.", - "schema" : { - "$ref" : "#/definitions/Baskets" - } - }, - "204" : { - "description" : "No Content. No baskets available for specified limits" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing master token" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - }, - "/baskets/{name}" : { - "get" : { - "tags" : [ "baskets" ], - "summary" : "Get basket settings", - "description" : "Retrieves configuration settings of this basket.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - } ], - "responses" : { - "200" : { - "description" : "OK. Returns basket configuration", - "schema" : { - "$ref" : "#/definitions/Config" - } - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "post" : { - "tags" : [ "baskets" ], - "summary" : "Create new basket", - "description" : "Creates a new basket with this name.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The name of new basket", - "required" : true, - "type" : "string" - }, { - "in" : "body", - "name" : "config", - "description" : "Basket configuration", - "required" : false, - "schema" : { - "$ref" : "#/definitions/Config" - } - } ], - "responses" : { - "201" : { - "description" : "Created. Indicates that basket is successfully created", - "schema" : { - "$ref" : "#/definitions/Token" - } - }, - "400" : { - "description" : "Bad Request. Failed to parse JSON into basket configuration object." - }, - "403" : { - "description" : "Forbidden. Indicates that basket name conflicts with reserved paths; e.g. `baskets`, `web`, etc." - }, - "409" : { - "description" : "Conflict. Indicates that basket with such name already exists" - }, - "422" : { - "description" : "Unprocessable Entity. Basket configuration is not valid." - } - } - }, - "put" : { - "tags" : [ "baskets" ], - "summary" : "Update basket settings", - "description" : "Updates configuration settings of this basket.\n\nSpecial configuration parameters for request forwarding:\n * `insecure_tls` controls certificate verification when forwarding requests. Setting this parameter to `true`\n allows to forward collected HTTP requests via HTTPS protocol even if the forward end-point is configured with\n self-signed TLS/SSL certificate. **Warning:** enabling this feature has known security implications.\n * `expand_path` changes the logic of constructing taget URL when forwarding requests. If this parameter is\n set to `true` the forward URL path will be expanded when original HTTP request contains compound path. For\n example, a basket with name **server1** is configured to forward all requests to `http://server1.intranet:8001/myservice`\n and it has received an HTTP request like `GET http://baskets.example.com/server1/component/123/events?status=OK`\n then depending on `expand_path` settings the request will be forwarded to:\n * `true` => `GET http://server1.intranet:8001/myservice/component/123/events?status=OK`\n * `false` => `GET http://server1.intranet:8001/myservice?status=OK`\n", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "in" : "body", - "name" : "config", - "description" : "New configuration to apply", - "required" : true, - "schema" : { - "$ref" : "#/definitions/Config" - } - } ], - "responses" : { - "204" : { - "description" : "No Content. Basket configuration is updated" - }, - "400" : { - "description" : "Bad Request. Failed to parse JSON into basket configuration object." - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - }, - "422" : { - "description" : "Unprocessable Entity. Basket configuration is not valid." - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "delete" : { - "tags" : [ "baskets" ], - "summary" : "Delete basket", - "description" : "Permanently deletes this basket and all collected requests.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - } ], - "responses" : { - "204" : { - "description" : "No Content. Basket is deleted" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - }, - "/baskets/{name}/responses/{method}" : { - "get" : { - "tags" : [ "responses" ], - "summary" : "Get response settings", - "description" : "Retrieves information about configured response of the basket. Service will reply with this response to any\nHTTP request sent to the basket with appropriate HTTP method.\n\nIf nothing is configured, the default response is HTTP 200 - OK with empty content.\n", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "name" : "method", - "in" : "path", - "description" : "The HTTP method this response is configured for", - "required" : true, - "type" : "string", - "enum" : [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] - } ], - "responses" : { - "200" : { - "description" : "OK. Returns configured response information", - "schema" : { - "$ref" : "#/definitions/Response" - } - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "put" : { - "tags" : [ "responses" ], - "summary" : "Update response settings", - "description" : "Allows to configure HTTP response of this basket. The service will reply with configured response to any HTTP\nrequest sent to the basket with appropriate HTTP method.\n\nIf nothing is configured, the default response is HTTP 200 - OK with empty content.\n", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "name" : "method", - "in" : "path", - "description" : "The HTTP method this response is configured for", - "required" : true, - "type" : "string", - "enum" : [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] - }, { - "in" : "body", - "name" : "response", - "description" : "HTTP response configuration", - "required" : true, - "schema" : { - "$ref" : "#/definitions/Response" - } - } ], - "responses" : { - "204" : { - "description" : "No Content. Response configuration is updated" - }, - "400" : { - "description" : "Bad Request. Failed to parse JSON into response configuration object." - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - }, - "422" : { - "description" : "Unprocessable Entity. Response configuration is not valid." - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - }, - "/baskets/{name}/requests" : { - "get" : { - "tags" : [ "requests" ], - "summary" : "Get collected requests", - "description" : "Fetches collection of requests collected by this basket.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "name" : "max", - "in" : "query", - "description" : "Maximum number of requests to return; default 20", - "required" : false, - "type" : "integer" - }, { - "name" : "skip", - "in" : "query", - "description" : "Number of requests to skip; default 0", - "required" : false, - "type" : "integer" - }, { - "name" : "q", - "in" : "query", - "description" : "Query string to filter result, only requests that match the query will be included in response", - "required" : false, - "type" : "string" - }, { - "name" : "in", - "in" : "query", - "description" : "Defines what is taken into account when filtering is applied: `body` - search in content body of collected requests,\n`query` - search among query parameters of collected requests, `headers` - search among request header values,\n`any` - search anywhere; default `any`\n", - "required" : false, - "type" : "string", - "enum" : [ "any", "body", "query", "headers" ] - } ], - "responses" : { - "200" : { - "description" : "OK. Returns list of basket requests.", - "schema" : { - "$ref" : "#/definitions/Requests" - } - }, - "204" : { - "description" : "No Content. No requests found for specified limits" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "delete" : { - "tags" : [ "requests" ], - "summary" : "Delete all requests", - "description" : "Deletes all requests collected by this basket.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - } ], - "responses" : { - "204" : { - "description" : "No Content. Basket requests are cleared" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - } - }, - "securityDefinitions" : { - "basket_token" : { - "description" : "Basket assigned secure token", - "type" : "apiKey", - "name" : "Authorization", - "in" : "header" - } - }, - "definitions" : { - "Baskets" : { - "type" : "object", - "required" : [ "has_more", "names" ], - "properties" : { - "names" : { - "type" : "array", - "description" : "Collection of basket names", - "items" : { - "type" : "string" - } - }, - "count" : { - "type" : "integer", - "description" : "Total number of baskets in the system; not present if query is applied" - }, - "has_more" : { - "type" : "boolean", - "description" : "Indicates if there are more baskets to fetch" - } - } - }, - "Config" : { - "type" : "object", - "properties" : { - "forward_url" : { - "type" : "string", - "description" : "URL to forward all incoming requests of the basket, `empty` value disables forwarding" - }, - "proxy_response" : { - "type" : "boolean", - "description" : "If set to `true` this basket behaves as a full proxy: responses from underlying service configured in `forward_url`\nare passed back to clients of original requests. The configuration of basket responses is ignored in this case.\n" - }, - "insecure_tls" : { - "type" : "boolean", - "description" : "If set to `true` the certificate verification will be disabled if forward URL indicates HTTPS scheme.\n**Warning:** enabling this feature has known security implications.\n" - }, - "expand_path" : { - "type" : "boolean", - "description" : "If set to `true` the forward URL path will be expanded when original HTTP request contains compound path." - }, - "capacity" : { - "type" : "integer", - "description" : "Baskets capacity, defines maximum number of requests to store" - } - } - }, - "Token" : { - "type" : "object", - "required" : [ "token" ], - "properties" : { - "token" : { - "type" : "string", - "description" : "Secure token to manage the basket, generated by system" - } - } - }, - "Requests" : { - "type" : "object", - "required" : [ "has_more", "requests" ], - "properties" : { - "requests" : { - "type" : "array", - "description" : "Collection of collected requests", - "items" : { - "$ref" : "#/definitions/Request" - } - }, - "count" : { - "type" : "integer", - "description" : "Current number of collected requests hold by basket; not present if query is applied" - }, - "total_count" : { - "type" : "integer", - "description" : "Total number of all requests passed through this basket; not present if query is applied" - }, - "has_more" : { - "type" : "boolean", - "description" : "Indicates if there are more requests collected by basket to fetch" - } - } - }, - "Request" : { - "type" : "object", - "properties" : { - "date" : { - "type" : "integer", - "format" : "int64", - "description" : "Date and time of request in Unix time ms. format (number of miliseconds elapsed since January 1, 1970 UTC)" - }, - "headers" : { - "$ref" : "#/definitions/Headers" - }, - "content_length" : { - "type" : "integer", - "description" : "Content lenght of request" - }, - "body" : { - "type" : "string", - "description" : "Content of request body" - }, - "method" : { - "type" : "string", - "description" : "HTTP methof of request" - }, - "path" : { - "type" : "string", - "description" : "URL path of request" - }, - "query" : { - "type" : "string", - "description" : "Query parameters of request" - } - } - }, - "Headers" : { - "type" : "object", - "description" : "Map of HTTP headers, key represents name, value is array of values", - "additionalProperties" : { - "type" : "array", - "description" : "Collection of header values", - "items" : { - "type" : "string" - } - } - }, - "Response" : { - "type" : "object", - "properties" : { - "status" : { - "type" : "integer", - "description" : "The HTTP status code to reply with" - }, - "headers" : { - "$ref" : "#/definitions/Headers" - }, - "body" : { - "type" : "string", - "description" : "Content of response body" - }, - "is_template" : { - "type" : "boolean", - "description" : "If set to `true` the body is treated as [HTML template](https://golang.org/pkg/html/template) that accepts\ninput from request parameters.\n" - } - } - } - } -} diff --git a/doc/api-swagger.yaml b/doc/api-swagger.yaml index 2ce4d07..c9965dc 100644 --- a/doc/api-swagger.yaml +++ b/doc/api-swagger.yaml @@ -4,7 +4,7 @@ swagger: '2.0' info: - version: '0.8' + version: '1.0.0' title: Request Baskets API description: | RESTful API of [Request Baskets](https://rbaskets.in) service. @@ -31,6 +31,8 @@ produces: # Groups and their descriptions tags: + - name: service + description: Service information - name: baskets description: Manage baskets - name: responses @@ -48,11 +50,341 @@ securityDefinitions: # URL patterns paths: + /api/version: + get: + tags: + - service + summary: Get service version + description: Get service version. + responses: + 200: + description: OK. Returns service version. + schema: + $ref: '#/definitions/Version' + + /api/stats: + get: + tags: + - baskets + summary: Get baskets statistics + description: Get service statistics about baskets and collected HTTP requests. Require master token. + parameters: + - name: max + in: query + type: integer + description: Maximum number of basket names to return; default 5 + required: false + responses: + 200: + description: OK. Returns service statistics. + schema: + $ref: '#/definitions/ServiceStats' + 401: + description: Unauthorized. Invalid or missing master token + security: + - basket_token: [] + + /api/baskets: + get: + tags: + - baskets + summary: Get baskets + description: Fetches a list of basket names managed by service. Require master token. + parameters: + - name: max + in: query + type: integer + description: Maximum number of basket names to return; default 20 + required: false + - name: skip + in: query + type: integer + description: Number of basket names to skip; default 0 + required: false + - name: q + in: query + type: string + description: Query string to filter result, only those basket names that match the query will be included in response + required: false + responses: + 200: + description: OK. Returns list of available baskets. + schema: + $ref: '#/definitions/Baskets' + 204: + description: No Content. No baskets available for specified limits + 401: + description: Unauthorized. Invalid or missing master token + security: + - basket_token: [] + + /api/baskets/{name}: + post: + tags: + - baskets + summary: Create new basket + description: Creates a new basket with this name. + parameters: + - name: name + in: path + type: string + description: The name of new basket + required: true + - name: config + in: body + description: Basket configuration + required: false + schema: + $ref: '#/definitions/Config' + responses: + 201: + description: Created. Indicates that basket is successfully created + schema: + $ref: '#/definitions/Token' + 400: + description: Bad Request. Failed to parse JSON into basket configuration object. + 403: + description: Forbidden. Indicates that basket name conflicts with reserved paths; e.g. `baskets`, `web`, etc. + 409: + description: Conflict. Indicates that basket with such name already exists + 422: + description: Unprocessable Entity. Basket configuration is not valid. + get: + tags: + - baskets + summary: Get basket settings + description: Retrieves configuration settings of this basket. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + responses: + 200: + description: OK. Returns basket configuration + schema: + $ref: '#/definitions/Config' + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + put: + tags: + - baskets + summary: Update basket settings + description: | + Updates configuration settings of this basket. + + Special configuration parameters for request forwarding: + * `insecure_tls` controls certificate verification when forwarding requests. Setting this parameter to `true` + allows to forward collected HTTP requests via HTTPS protocol even if the forward end-point is configured with + self-signed TLS/SSL certificate. **Warning:** enabling this feature has known security implications. + * `expand_path` changes the logic of constructing taget URL when forwarding requests. If this parameter is + set to `true` the forward URL path will be expanded when original HTTP request contains compound path. For + example, a basket with name **server1** is configured to forward all requests to `http://server1.intranet:8001/myservice` + and it has received an HTTP request like `GET http://baskets.example.com/server1/component/123/events?status=OK` + then depending on `expand_path` settings the request will be forwarded to: + * `true` => `GET http://server1.intranet:8001/myservice/component/123/events?status=OK` + * `false` => `GET http://server1.intranet:8001/myservice?status=OK` + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: config + in: body + description: New configuration to apply + required: true + schema: + $ref: '#/definitions/Config' + responses: + 204: + description: No Content. Basket configuration is updated + 400: + description: Bad Request. Failed to parse JSON into basket configuration object. + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + 422: + description: Unprocessable Entity. Basket configuration is not valid. + security: + - basket_token: [] + delete: + tags: + - baskets + summary: Delete basket + description: Permanently deletes this basket and all collected requests. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + responses: + 204: + description: No Content. Basket is deleted + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + + /api/baskets/{name}/responses/{method}: + get: + tags: + - responses + summary: Get response settings + description: | + Retrieves information about configured response of the basket. Service will reply with this response to any + HTTP request sent to the basket with appropriate HTTP method. + + If nothing is configured, the default response is HTTP 200 - OK with empty content. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: method + in: path + type: string + enum: [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] + description: The HTTP method this response is configured for + required: true + responses: + 200: + description: OK. Returns configured response information + schema: + $ref: '#/definitions/Response' + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + put: + tags: + - responses + summary: Update response settings + description: | + Allows to configure HTTP response of this basket. The service will reply with configured response to any HTTP + request sent to the basket with appropriate HTTP method. + + If nothing is configured, the default response is HTTP 200 - OK with empty content. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: method + in: path + type: string + enum: [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] + description: The HTTP method this response is configured for + required: true + - name: response + in: body + description: HTTP response configuration + required: true + schema: + $ref: '#/definitions/Response' + responses: + 204: + description: No Content. Response configuration is updated + 400: + description: Bad Request. Failed to parse JSON into response configuration object. + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + 422: + description: Unprocessable Entity. Response configuration is not valid. + security: + - basket_token: [] + + /api/baskets/{name}/requests: + get: + tags: + - requests + summary: Get collected requests + description: Fetches collection of requests collected by this basket. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: max + in: query + type: integer + description: Maximum number of requests to return; default 20 + required: false + - name: skip + in: query + type: integer + description: Number of requests to skip; default 0 + required: false + - name: q + in: query + type: string + description: Query string to filter result, only requests that match the query will be included in response + required: false + - name: in + in: query + type: string + enum: [ 'any', 'body', 'query', 'headers' ] + description: | + Defines what is taken into account when filtering is applied: `body` - search in content body of collected requests, + `query` - search among query parameters of collected requests, `headers` - search among request header values, + `any` - search anywhere; default `any` + required: false + responses: + 200: + description: OK. Returns list of basket requests. + schema: + $ref: '#/definitions/Requests' + 204: + description: No Content. No requests found for specified limits + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + delete: + tags: + - requests + summary: Delete all requests + description: Deletes all requests collected by this basket. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + responses: + 204: + description: No Content. Basket requests are cleared + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + /baskets: get: tags: - baskets summary: Get baskets + deprecated: true description: Fetches a list of basket names managed by service. Require master token. parameters: - name: max @@ -87,6 +419,7 @@ paths: tags: - baskets summary: Create new basket + deprecated: true description: Creates a new basket with this name. parameters: - name: name @@ -117,6 +450,7 @@ paths: tags: - baskets summary: Get basket settings + deprecated: true description: Retrieves configuration settings of this basket. parameters: - name: name @@ -139,6 +473,7 @@ paths: tags: - baskets summary: Update basket settings + deprecated: true description: | Updates configuration settings of this basket. @@ -182,6 +517,7 @@ paths: tags: - baskets summary: Delete basket + deprecated: true description: Permanently deletes this basket and all collected requests. parameters: - name: name @@ -204,6 +540,7 @@ paths: tags: - responses summary: Get response settings + deprecated: true description: | Retrieves information about configured response of the basket. Service will reply with this response to any HTTP request sent to the basket with appropriate HTTP method. @@ -236,6 +573,7 @@ paths: tags: - responses summary: Update response settings + deprecated: true description: | Allows to configure HTTP response of this basket. The service will reply with configured response to any HTTP request sent to the basket with appropriate HTTP method. @@ -278,6 +616,7 @@ paths: tags: - requests summary: Get collected requests + deprecated: true description: Fetches collection of requests collected by this basket. parameters: - name: name @@ -326,6 +665,7 @@ paths: tags: - requests summary: Delete all requests + deprecated: true description: Deletes all requests collected by this basket. parameters: - name: name @@ -345,6 +685,78 @@ paths: # Model definitions: + Version: + type: object + properties: + name: + type: string + description: Service name + version: + type: string + description: Service version + commit: + type: string + description: Git commit this service is build from + commit_short: + type: string + description: Short form of git commit this service is build from + source_code: + type: string + description: URL of the source code repository + + ServiceStats: + type: object + properties: + baskets_count: + type: integer + description: Total number of baskets managed by service + empty_baskets_count: + type: integer + description: Number of empty baskets + requests_count: + type: integer + description: Number of HTTP requests currently stored by service + requests_total_count: + type: integer + description: Total number of HTTP requests processed by service + max_basket_size: + type: integer + description: Size of the biggest basket that processed the top most number of HTTP requests + avg_basket_size: + type: integer + description: Average size of a basket in the system, empty baskets are not taken into account + top_baskets_size: + type: array + description: Collection of top basket by size + items: + $ref: '#/definitions/BasketInfo' + top_baskets_recent: + type: array + description: Collection of top baskets recently active + items: + $ref: '#/definitions/BasketInfo' + + BasketInfo: + type: object + properties: + name: + type: string + description: Basket name + requests_count: + type: integer + description: Current number of collected HTTP requests held by basket + requests_total_count: + type: integer + description: Total number of all HTTP requests passed through this basket + last_request_date: + type: integer + format: int64 + description: | + Date and time of last request processed through this basket in Unix time ms. + format (number of milliseconds elapsed since January 1, 1970 UTC). + + If no requests were collected by this basket `0` is returned. + Baskets: type: object required: @@ -422,18 +834,18 @@ definitions: date: type: integer format: int64 - description: Date and time of request in Unix time ms. format (number of miliseconds elapsed since January 1, 1970 UTC) + description: Date and time of request in Unix time ms. format (number of milliseconds elapsed since January 1, 1970 UTC) headers: $ref: '#/definitions/Headers' content_length: type: integer - description: Content lenght of request + description: Content length of request body: type: string description: Content of request body method: type: string - description: HTTP methof of request + description: HTTP method of request path: type: string description: URL path of request diff --git a/handlers.go b/handlers.go index 3f53876..3980bd2 100644 --- a/handlers.go +++ b/handlers.go @@ -143,18 +143,37 @@ func GetBaskets(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } else { values := r.URL.Query() if query := values.Get("q"); len(query) > 0 { - // Find names + // find names max, skip := getPage(values) json, err := json.Marshal(basketsDb.FindNames(query, max, skip)) writeJSON(w, http.StatusOK, json, err) } else { - // Get basket names page + // get basket names page json, err := json.Marshal(basketsDb.GetNames(getPage(values))) writeJSON(w, http.StatusOK, json, err) } } } +// GetStats handles HTTP request to get database statistics +func GetStats(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if r.Header.Get("Authorization") != serverConfig.MasterToken { + w.WriteHeader(http.StatusUnauthorized) + } else { + // get database stats + max := parseInt(r.URL.Query().Get("max"), 1, 100, 5) + json, err := json.Marshal(basketsDb.GetStats(max)) + writeJSON(w, http.StatusOK, json, err) + } +} + +// GetVersion handles HTTP request to get service version details +func GetVersion(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + // get database stats + json, err := json.Marshal(version) + writeJSON(w, http.StatusOK, json, err) +} + // GetBasket handles HTTP request to get basket configuration func GetBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if _, basket := getAuthenticatedBasket(w, r, ps); basket != nil { @@ -166,7 +185,7 @@ func GetBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { // CreateBasket handles HTTP request to create a new basket func CreateBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { name := ps.ByName("basket") - if name == serviceAPIPath || name == serviceUIPath { + if name == serviceOldAPIPath || name == serviceAPIPath || name == serviceUIPath { http.Error(w, "This basket name conflicts with reserved system path: "+name, http.StatusForbidden) return } @@ -302,12 +321,12 @@ func GetBasketRequests(w http.ResponseWriter, r *http.Request, ps httprouter.Par if _, basket := getAuthenticatedBasket(w, r, ps); basket != nil { values := r.URL.Query() if query := values.Get("q"); len(query) > 0 { - // Find requests + // find requests max, skip := getPage(values) json, err := json.Marshal(basket.FindRequests(query, values.Get("in"), max, skip)) writeJSON(w, http.StatusOK, json, err) } else { - // Get requests page + // get requests page json, err := json.Marshal(basket.GetRequests(getPage(values))) writeJSON(w, http.StatusOK, json, err) } @@ -337,7 +356,7 @@ func WebIndexPage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) func WebBasketPage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if name := ps.ByName("basket"); validBasketName.MatchString(name) { switch name { - case serviceAPIPath: + case serviceOldAPIPath: // admin page to access all baskets w.Header().Set("Content-Type", "text/html; charset=utf-8") basketsPageTemplate.Execute(w, version) diff --git a/handlers_test.go b/handlers_test.go index f6be45f..44bd504 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -704,6 +704,78 @@ func TestGetBaskets_Unauthorized(t *testing.T) { } } +func TestGetStats(t *testing.T) { + // create 3 baskets + for i := 0; i < 3; i++ { + basket := fmt.Sprintf("forstats0%v", i) + r, err := http.NewRequest("POST", "http://localhost:55555/baskets/"+basket, strings.NewReader("")) + if assert.NoError(t, err) { + w := httptest.NewRecorder() + ps := append(make(httprouter.Params, 0), httprouter.Param{Key: "basket", Value: basket}) + CreateBasket(w, r, ps) + assert.Equal(t, 201, w.Code, "wrong HTTP result code") + } + } + + // get stats + r, err := http.NewRequest("GET", "http://localhost:55555/api/stats", strings.NewReader("")) + if assert.NoError(t, err) { + r.Header.Add("Authorization", serverConfig.MasterToken) + w := httptest.NewRecorder() + GetStats(w, r, make(httprouter.Params, 0)) + // HTTP 200 - OK + assert.Equal(t, 200, w.Code, "wrong HTTP result code") + + stats := new(DatabaseStats) + err = json.Unmarshal(w.Body.Bytes(), stats) + if assert.NoError(t, err) { + // validate response + assert.NotEmpty(t, stats.TopBasketsByDate, "top baskets are expected") + assert.NotEmpty(t, stats.TopBasketsBySize, "top baskets are expected") + assert.True(t, stats.BasketsCount > 0, "baskets count should be greater than 0") + assert.True(t, stats.EmptyBasketsCount > 0, "empty baskets count should be greater than 0") + } + } +} + +func TestGetStats_Unauthorized(t *testing.T) { + r, err := http.NewRequest("GET", "http://localhost:55555/api/stats", strings.NewReader("")) + if assert.NoError(t, err) { + // no authorization at all: 401 - unauthorized + w := httptest.NewRecorder() + GetStats(w, r, make(httprouter.Params, 0)) + assert.Equal(t, 401, w.Code, "wrong HTTP result code") + + // invalid master token: 401 - unauthorized + r.Header.Add("Authorization", "123-wrong-token") + w = httptest.NewRecorder() + GetStats(w, r, make(httprouter.Params, 0)) + assert.Equal(t, 401, w.Code, "wrong HTTP result code") + } +} + +func TestGetVersion(t *testing.T) { + // get version + r, err := http.NewRequest("GET", "http://localhost:55555/api/version", strings.NewReader("")) + if assert.NoError(t, err) { + w := httptest.NewRecorder() + GetVersion(w, r, make(httprouter.Params, 0)) + // HTTP 200 - OK + assert.Equal(t, 200, w.Code, "wrong HTTP result code") + + ver := new(Version) + err = json.Unmarshal(w.Body.Bytes(), ver) + if assert.NoError(t, err) { + // validate response + assert.Equal(t, serviceName, ver.Name) + assert.Equal(t, sourceCodeURL, ver.SourceCode) + assert.NotEmpty(t, ver.Version, "version is expected") + assert.NotEmpty(t, ver.Commit, "commit is expected") + assert.NotEmpty(t, ver.CommitShort, "commit short is expected") + } + } +} + func TestGetBaskets_Query(t *testing.T) { // create 10 baskets for i := 0; i < 10; i++ { @@ -1032,10 +1104,10 @@ func TestWebBasketPage(t *testing.T) { } func TestWebBasketsPage(t *testing.T) { - r, err := http.NewRequest("GET", "http://localhost:55555/web/"+serviceAPIPath, strings.NewReader("")) + r, err := http.NewRequest("GET", "http://localhost:55555/web/"+serviceOldAPIPath, strings.NewReader("")) if assert.NoError(t, err) { w := httptest.NewRecorder() - ps := append(make(httprouter.Params, 0), httprouter.Param{Key: "basket", Value: serviceAPIPath}) + ps := append(make(httprouter.Params, 0), httprouter.Param{Key: "basket", Value: serviceOldAPIPath}) WebBasketPage(w, r, ps) // validate response: 200 - OK diff --git a/server.go b/server.go index 0201df2..28af486 100644 --- a/server.go +++ b/server.go @@ -19,7 +19,12 @@ var version *Version // CreateServer creates an instance of Request Baskets server func CreateServer(config *ServerConfig) *http.Server { - version = &Version{Version: GitVersion, Commit: GitCommit, CommitShort: GitCommitShort} + version = &Version{ + Name: serviceName, + Version: GitVersion, + Commit: GitCommit, + CommitShort: GitCommitShort, + SourceCode: sourceCodeURL} log.Printf("[info] service version: %s from commit: %s (%s)", version.Version, version.CommitShort, version.Commit) // create database @@ -38,21 +43,36 @@ func CreateServer(config *ServerConfig) *http.Server { // configure service HTTP router router := httprouter.New() + //// Old API mapping //// // basket names - router.GET("/"+serviceAPIPath, GetBaskets) - + router.GET("/"+serviceOldAPIPath, GetBaskets) // basket management - router.GET("/"+serviceAPIPath+"/:basket", GetBasket) - router.POST("/"+serviceAPIPath+"/:basket", CreateBasket) - router.PUT("/"+serviceAPIPath+"/:basket", UpdateBasket) - router.DELETE("/"+serviceAPIPath+"/:basket", DeleteBasket) - - router.GET("/"+serviceAPIPath+"/:basket/responses/:method", GetBasketResponse) - router.PUT("/"+serviceAPIPath+"/:basket/responses/:method", UpdateBasketResponse) + router.GET("/"+serviceOldAPIPath+"/:basket", GetBasket) + router.POST("/"+serviceOldAPIPath+"/:basket", CreateBasket) + router.PUT("/"+serviceOldAPIPath+"/:basket", UpdateBasket) + router.DELETE("/"+serviceOldAPIPath+"/:basket", DeleteBasket) + router.GET("/"+serviceOldAPIPath+"/:basket/responses/:method", GetBasketResponse) + router.PUT("/"+serviceOldAPIPath+"/:basket/responses/:method", UpdateBasketResponse) + // requests management + router.GET("/"+serviceOldAPIPath+"/:basket/requests", GetBasketRequests) + router.DELETE("/"+serviceOldAPIPath+"/:basket/requests", ClearBasket) + //// New API mapping //// + // service details + router.GET("/"+serviceAPIPath+"/stats", GetStats) + router.GET("/"+serviceAPIPath+"/version", GetVersion) + // basket names + router.GET("/"+serviceAPIPath+"/baskets", GetBaskets) + // basket management + router.GET("/"+serviceAPIPath+"/baskets/:basket", GetBasket) + router.POST("/"+serviceAPIPath+"/baskets/:basket", CreateBasket) + router.PUT("/"+serviceAPIPath+"/baskets/:basket", UpdateBasket) + router.DELETE("/"+serviceAPIPath+"/baskets/:basket", DeleteBasket) + router.GET("/"+serviceAPIPath+"/baskets/:basket/responses/:method", GetBasketResponse) + router.PUT("/"+serviceAPIPath+"/baskets/:basket/responses/:method", UpdateBasketResponse) // requests management - router.GET("/"+serviceAPIPath+"/:basket/requests", GetBasketRequests) - router.DELETE("/"+serviceAPIPath+"/:basket/requests", ClearBasket) + router.GET("/"+serviceAPIPath+"/baskets/:basket/requests", GetBasketRequests) + router.DELETE("/"+serviceAPIPath+"/baskets/:basket/requests", ClearBasket) // web pages router.GET("/", ForwardToWeb) diff --git a/version.go b/version.go index 9a4d753..d73eaef 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,9 @@ var ( // Version describes application version type Version struct { - Version string - Commit string - CommitShort string + Name string `json:"name"` + Version string `json:"version"` + Commit string `json:"commit"` + CommitShort string `json:"commit_short"` + SourceCode string `json:"source_code"` } diff --git a/web/basket.html b/web/basket.html index f2286d2..4a52d6f 100644 --- a/web/basket.html +++ b/web/basket.html @@ -257,7 +257,7 @@ function fetchRequests() { $.ajax({ method: "GET", - url: "/baskets/" + name + "/requests?skip=" + fetchedCount, + url: "/api/baskets/" + name + "/requests?skip=" + fetchedCount, headers: { "Authorization" : getToken() } @@ -267,7 +267,7 @@ function fetchTotalCount() { $.ajax({ method: "GET", - url: "/baskets/" + name + "/requests?max=0", + url: "/api/baskets/" + name + "/requests?max=0", headers: { "Authorization" : getToken() } @@ -283,7 +283,7 @@ $("#response_method").val(method); $.ajax({ method: "GET", - url: "/baskets/" + name + "/responses/" + method, + url: "/api/baskets/" + name + "/responses/" + method, headers: { "Authorization" : getToken() } @@ -357,7 +357,7 @@ $.ajax({ method: "PUT", - url: "/baskets/" + name + "/responses/" + method, + url: "/api/baskets/" + name + "/responses/" + method, dataType: "json", data: JSON.stringify(response), headers: { @@ -382,7 +382,7 @@ $.ajax({ method: "PUT", - url: "/baskets/" + name, + url: "/api/baskets/" + name, dataType: "json", data: JSON.stringify(currentConfig), headers: { @@ -421,7 +421,7 @@ function config() { $.ajax({ method: "GET", - url: "/baskets/" + name, + url: "/api/baskets/" + name, headers: { "Authorization" : getToken() } @@ -444,7 +444,7 @@ function deleteRequests() { $.ajax({ method: "DELETE", - url: "/baskets/" + name + "/requests", + url: "/api/baskets/" + name + "/requests", headers: { "Authorization" : getToken() } @@ -459,7 +459,7 @@ $.ajax({ method: "DELETE", - url: "/baskets/" + name, + url: "/api/baskets/" + name, headers: { "Authorization" : getToken() } @@ -507,7 +507,7 @@ "already stored in your browser.\n\n" + "If you trust this link choose 'OK' and existing token will be \n" + "replaced with the new one, otherwise choose 'Cancel'.\n\n" + - "Do you want to replace the access token of this basket?")))) { + "Do you want to replace the access token of this basket?")) { localStorage.setItem("basket_" + name, token); } } diff --git a/web/baskets.html b/web/baskets.html index 6b760ea..c68a579 100644 --- a/web/baskets.html +++ b/web/baskets.html @@ -19,8 +19,9 @@ h1 { margin-top: 2px; } #more { margin-left: 60px; padding-bottom: 10px; } #all_baskets ul { width: 100%; } - #all_baskets li { padding: 0 0 5px 20px; float: left; display: inline; position: relative; width: 25%; } - #all_baskets li:before { content: "\f291"; font-family: "FontAwesome"; position: absolute; left: 0px; top:0px; } + #all_baskets li { float: left; display: inline; position: relative; width: 25%; } + .baskets li { padding: 0 0 5px 20px; } + .baskets li:before { content: "\f291"; font-family: "FontAwesome"; position: absolute; left: 0px; top:0px; } @@ -227,12 +272,65 @@
-

All Baskets

+
+
+

Basic Statistics

+
+
    +
  • + ? + Total baskets +
  • +
  • + ? + Empty baskets +
  • +
  • + ? + Total requests +
  • +
  • + ? + Current requests +
  • +
  • + ? + Max. basket size +
  • +
  • + ? + Avg. basket size +
  • +
+
+
+
+
+
+

Top Baskets: by size

+
+
    +
+
+
+
+
+
+

Top Baskets: recently active

+
+
    +
+
+
+
+
+
+

All Baskets


-