From 857b1c94294b095885b35cdcbd72794103df0c99 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Wed, 13 Feb 2019 01:25:56 +0100 Subject: [PATCH 01/10] baskets db stats basic implementation + service end-point via /stats --- baskets.go | 22 ++++++++++++++++++++++ baskets_bolt.go | 37 +++++++++++++++++++++++++++++++++++++ baskets_mem.go | 33 +++++++++++++++++++++++++++++++++ baskets_sql.go | 40 +++++++++++++++++++++++++++++++++------- config.go | 1 + handlers.go | 13 ++++++++++++- server.go | 1 + 7 files changed, 139 insertions(+), 8 deletions(-) diff --git a/baskets.go b/baskets.go index 5f959ba..b088a92 100644 --- a/baskets.go +++ b/baskets.go @@ -75,6 +75,26 @@ type BasketNamesQueryPage struct { HasMore bool `json:"has_more"` } +// DatabaseStats describes collected statistics of a baskets database +type DatabaseStats struct { + BasketsCount int `json:"baskets_count"` + EmptyBasketsCount int `json:"empty_baskets_count"` + RequestsCount int `json:"requests_count"` + RequestsTotalCount int `json:"requests_total_count"` + MaxBasketSize int `json:"max_basket_size"` + AvgBasketSize int `json:"avg_basket_size"` + TopBasketsBySize []*BasketInfo `json:"top_baskets_size"` + TopBasketsByDate []*BasketInfo `json:"top_baskets_recent"` +} + +// BasketInfo describes shorlty a basket for database statistics +type BasketInfo struct { + Name string `json:"name"` + RequestsCount int `json:"requests_count"` + RequestsTotalCount int `json:"requests_total_count"` + LastRequestDate int64 `json:"last_request_date"` +} + // Basket is an interface that represent request basket entity to collects HTTP requests type Basket interface { Config() BasketConfig @@ -102,6 +122,8 @@ type BasketsDatabase interface { GetNames(max int, skip int) BasketNamesPage FindNames(query string, max int, skip int) BasketNamesQueryPage + GetStats() DatabaseStats + Release() } diff --git a/baskets_bolt.go b/baskets_bolt.go index b73227f..48715a1 100644 --- a/baskets_bolt.go +++ b/baskets_bolt.go @@ -475,6 +475,43 @@ func (bdb *boltDatabase) FindNames(query string, max int, skip int) BasketNamesQ return page } +func (bdb *boltDatabase) GetStats() DatabaseStats { + var basketsCount, emptyBasketsCount, requestsCount, requestsTotalCount, maxBasketSize, avgBasketSize int + + bdb.db.View(func(tx *bolt.Tx) error { + cur := tx.Cursor() + for key, _ := cur.First(); key != nil; key, _ = cur.Next() { + if b := tx.Bucket(key); b != nil { + count := btoi(b.Get(boltKeyCount)) + total := btoi(b.Get(boltKeyTotalCount)) + + basketsCount++ + if total == 0 { + emptyBasketsCount++ + } + requestsCount += count + requestsTotalCount += total + if total > maxBasketSize { + maxBasketSize = total + } + } + } + return nil + }) + + if basketsCount > emptyBasketsCount { + avgBasketSize = requestsTotalCount / (basketsCount - emptyBasketsCount) + } + + return DatabaseStats{ + BasketsCount: basketsCount, + EmptyBasketsCount: emptyBasketsCount, + RequestsCount: requestsCount, + RequestsTotalCount: requestsTotalCount, + MaxBasketSize: maxBasketSize, + AvgBasketSize: avgBasketSize} +} + func (bdb *boltDatabase) Release() { log.Print("[info] closing Bolt database") err := bdb.db.Close() diff --git a/baskets_mem.go b/baskets_mem.go index dc1fae5..9bde293 100644 --- a/baskets_mem.go +++ b/baskets_mem.go @@ -256,6 +256,39 @@ func (db *memoryDatabase) FindNames(query string, max int, skip int) BasketNames return BasketNamesQueryPage{Names: result, HasMore: false} } +func (db *memoryDatabase) GetStats() DatabaseStats { + db.RLock() + defer db.RUnlock() + + var basketsCount, emptyBasketsCount, requestsCount, requestsTotalCount, maxBasketSize, avgBasketSize int + + for _, name := range db.names { + if basket, exists := db.baskets[name]; exists { + basketsCount++ + if basket.totalCount == 0 { + emptyBasketsCount++ + } + requestsCount += basket.Size() + requestsTotalCount += basket.totalCount + if basket.totalCount > maxBasketSize { + maxBasketSize = basket.totalCount + } + } + } + + if basketsCount > emptyBasketsCount { + avgBasketSize = requestsTotalCount / (basketsCount - emptyBasketsCount) + } + + return DatabaseStats{ + BasketsCount: basketsCount, + EmptyBasketsCount: emptyBasketsCount, + RequestsCount: requestsCount, + RequestsTotalCount: requestsTotalCount, + MaxBasketSize: maxBasketSize, + AvgBasketSize: avgBasketSize} +} + func (db *memoryDatabase) Release() { log.Print("[info] releasing in-memory database resources") } diff --git a/baskets_sql.go b/baskets_sql.go index c0c777e..f10a156 100644 --- a/baskets_sql.go +++ b/baskets_sql.go @@ -280,6 +280,16 @@ type sqlDatabase struct { dbType string // postgresql, mysql, oracle, etc. } +func (sdb *sqlDatabase) getInt(sql string, defaultValue int) int { + var value int + if err := sdb.db.QueryRow(unifySQL(sdb.dbType, sql)).Scan(&value); err != nil { + log.Printf("[error] failed to query for int result: %s", err) + return defaultValue + } + + return value +} + func (sdb *sqlDatabase) Create(name string, config BasketConfig) (BasketAuth, error) { auth := BasketAuth{} token, err := GenerateToken() @@ -324,13 +334,7 @@ func (sdb *sqlDatabase) Delete(name string) { } func (sdb *sqlDatabase) Size() int { - var size int - if err := sdb.db.QueryRow("SELECT COUNT(*) FROM rb_baskets").Scan(&size); err != nil { - log.Printf("[error] failed to get the total number of baskets: %s", err) - return 0 - } - - return size + return sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) } func (sdb *sqlDatabase) GetNames(max int, skip int) BasketNamesPage { @@ -383,6 +387,28 @@ func (sdb *sqlDatabase) FindNames(query string, max int, skip int) BasketNamesQu return page } +func (sdb *sqlDatabase) GetStats() DatabaseStats { + var basketsCount, emptyBasketsCount, requestsCount, requestsTotalCount, maxBasketSize, avgBasketSize int + + basketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) + emptyBasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets WHERE requests_count = 0", 0) + requestsCount = sdb.getInt("SELECT COUNT(*) FROM rb_requests", 0) + requestsTotalCount = sdb.getInt("SELECT SUM(requests_count) FROM rb_baskets", 0) + maxBasketSize = sdb.getInt("SELECT MAX(requests_count) FROM rb_baskets", 0) + + if basketsCount > emptyBasketsCount { + avgBasketSize = requestsTotalCount / (basketsCount - emptyBasketsCount) + } + + return DatabaseStats{ + BasketsCount: basketsCount, + EmptyBasketsCount: emptyBasketsCount, + RequestsCount: requestsCount, + RequestsTotalCount: requestsTotalCount, + MaxBasketSize: maxBasketSize, + AvgBasketSize: avgBasketSize} +} + func (sdb *sqlDatabase) Release() { log.Printf("[info] closing SQL database, releasing any open resources") sdb.db.Close() diff --git a/config.go b/config.go index 502fd77..b147c89 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ const ( maxBasketCapacity = 2000 defaultDatabaseType = DbTypeMemory serviceAPIPath = "baskets" + serviceStatsPath = "stats" serviceUIPath = "web" basketNamePattern = `^[\w\d\-_\.]{1,250}$` ) diff --git a/handlers.go b/handlers.go index 3f53876..b75079a 100644 --- a/handlers.go +++ b/handlers.go @@ -155,6 +155,17 @@ func GetBaskets(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } } +// GetStats handles HTTP request to get database statistics +func GetStats(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if r.Header.Get("Authorization") != serverConfig.MasterToken { + w.WriteHeader(http.StatusUnauthorized) + } else { + // Get database stats + json, err := json.Marshal(basketsDb.GetStats()) + writeJSON(w, http.StatusOK, json, err) + } +} + // GetBasket handles HTTP request to get basket configuration func GetBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if _, basket := getAuthenticatedBasket(w, r, ps); basket != nil { @@ -166,7 +177,7 @@ func GetBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { // CreateBasket handles HTTP request to create a new basket func CreateBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { name := ps.ByName("basket") - if name == serviceAPIPath || name == serviceUIPath { + if name == serviceAPIPath || name == serviceStatsPath || name == serviceUIPath { http.Error(w, "This basket name conflicts with reserved system path: "+name, http.StatusForbidden) return } diff --git a/server.go b/server.go index 0201df2..ee385b4 100644 --- a/server.go +++ b/server.go @@ -40,6 +40,7 @@ func CreateServer(config *ServerConfig) *http.Server { // basket names router.GET("/"+serviceAPIPath, GetBaskets) + router.GET("/"+serviceStatsPath, GetStats) // basket management router.GET("/"+serviceAPIPath+"/:basket", GetBasket) From 36879139ed12999973f6b91b3ec6e6035ee37088 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Thu, 14 Feb 2019 00:43:49 +0100 Subject: [PATCH 02/10] fixed JavaScript bug --- web/basket.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/basket.html b/web/basket.html index f2286d2..cc71064 100644 --- a/web/basket.html +++ b/web/basket.html @@ -507,7 +507,7 @@ "already stored in your browser.\n\n" + "If you trust this link choose 'OK' and existing token will be \n" + "replaced with the new one, otherwise choose 'Cancel'.\n\n" + - "Do you want to replace the access token of this basket?")))) { + "Do you want to replace the access token of this basket?")) { localStorage.setItem("basket_" + name, token); } } From 2a2f2b6f7184e8749cb10a0f2114b5c5accb7421 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Fri, 15 Feb 2019 15:22:27 +0100 Subject: [PATCH 03/10] list top 5 (top n) biggest and recent baskets, UI for stats in admin area --- baskets.go | 62 ++++++++++++++++++++++- baskets_bolt.go | 40 ++++++--------- baskets_mem.go | 34 +++++-------- baskets_sql.go | 27 ++++------ handlers.go | 13 ++--- web/baskets.html | 118 ++++++++++++++++++++++++++++++++++++++++---- web_baskets.html.go | 118 ++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 322 insertions(+), 90 deletions(-) diff --git a/baskets.go b/baskets.go index b088a92..c6d2a12 100644 --- a/baskets.go +++ b/baskets.go @@ -122,7 +122,7 @@ type BasketsDatabase interface { GetNames(max int, skip int) BasketNamesPage FindNames(query string, max int, skip int) BasketNamesQueryPage - GetStats() DatabaseStats + GetStats(max int) DatabaseStats Release() } @@ -255,3 +255,63 @@ func (req *RequestData) Matches(query string, in string) bool { return false } + +// Collect collects information about basket and updates statistics +func (stats *DatabaseStats) Collect(basket *BasketInfo, max int) { + stats.BasketsCount++ + if basket.RequestsTotalCount == 0 { + stats.EmptyBasketsCount++ + } + + stats.RequestsCount += basket.RequestsCount + stats.RequestsTotalCount += basket.RequestsTotalCount + if basket.RequestsTotalCount > stats.MaxBasketSize { + stats.MaxBasketSize = basket.RequestsTotalCount + } + + // top baskets by size + stats.TopBasketsBySize = collectConditionally(stats.TopBasketsBySize, basket, max, + func(b1 *BasketInfo, b2 *BasketInfo) bool { + return b1.RequestsTotalCount > b2.RequestsTotalCount + }) + + // top baskets by recent activity + stats.TopBasketsByDate = collectConditionally(stats.TopBasketsByDate, basket, max, + func(b1 *BasketInfo, b2 *BasketInfo) bool { + return b1.LastRequestDate > b2.LastRequestDate + }) +} + +func collectConditionally(col []*BasketInfo, basket *BasketInfo, size int, + greater func(*BasketInfo, *BasketInfo) bool) []*BasketInfo { + if col == nil { + col = make([]*BasketInfo, 0, size) + return append(col, basket) + } + + for i, b := range col { + if greater(basket, b) { + if len(col) < size { + col = append(col, nil) + } + copy(col[i+1:], col[i:]) + col[i] = basket + return col + } + } + + if len(col) < size { + return append(col, basket) + } + + return col +} + +// UpdateAvarage updates avarage statistics counters. +func (stats *DatabaseStats) UpdateAvarage() { + if stats.BasketsCount > stats.EmptyBasketsCount { + stats.AvgBasketSize = stats.RequestsTotalCount / (stats.BasketsCount - stats.EmptyBasketsCount) + } else { + stats.AvgBasketSize = 0 + } +} diff --git a/baskets_bolt.go b/baskets_bolt.go index 48715a1..c97f6b7 100644 --- a/baskets_bolt.go +++ b/baskets_bolt.go @@ -475,41 +475,33 @@ func (bdb *boltDatabase) FindNames(query string, max int, skip int) BasketNamesQ return page } -func (bdb *boltDatabase) GetStats() DatabaseStats { - var basketsCount, emptyBasketsCount, requestsCount, requestsTotalCount, maxBasketSize, avgBasketSize int +func (bdb *boltDatabase) GetStats(max int) DatabaseStats { + stats := DatabaseStats{} bdb.db.View(func(tx *bolt.Tx) error { cur := tx.Cursor() for key, _ := cur.First(); key != nil; key, _ = cur.Next() { if b := tx.Bucket(key); b != nil { - count := btoi(b.Get(boltKeyCount)) - total := btoi(b.Get(boltKeyTotalCount)) - - basketsCount++ - if total == 0 { - emptyBasketsCount++ - } - requestsCount += count - requestsTotalCount += total - if total > maxBasketSize { - maxBasketSize = total + var lastRequestDate int64 + if _, val := b.Bucket(boltKeyRequests).Cursor().Last(); val != nil { + request := new(RequestData) + if err := json.Unmarshal(val, request); err == nil { + lastRequestDate = request.Date + } } + + stats.Collect(&BasketInfo{ + Name: string(key), + RequestsCount: btoi(b.Get(boltKeyCount)), + RequestsTotalCount: btoi(b.Get(boltKeyTotalCount)), + LastRequestDate: lastRequestDate}, max) } } return nil }) - if basketsCount > emptyBasketsCount { - avgBasketSize = requestsTotalCount / (basketsCount - emptyBasketsCount) - } - - return DatabaseStats{ - BasketsCount: basketsCount, - EmptyBasketsCount: emptyBasketsCount, - RequestsCount: requestsCount, - RequestsTotalCount: requestsTotalCount, - MaxBasketSize: maxBasketSize, - AvgBasketSize: avgBasketSize} + stats.UpdateAvarage() + return stats } func (bdb *boltDatabase) Release() { diff --git a/baskets_mem.go b/baskets_mem.go index 9bde293..d80509e 100644 --- a/baskets_mem.go +++ b/baskets_mem.go @@ -256,37 +256,29 @@ func (db *memoryDatabase) FindNames(query string, max int, skip int) BasketNames return BasketNamesQueryPage{Names: result, HasMore: false} } -func (db *memoryDatabase) GetStats() DatabaseStats { +func (db *memoryDatabase) GetStats(max int) DatabaseStats { db.RLock() defer db.RUnlock() - var basketsCount, emptyBasketsCount, requestsCount, requestsTotalCount, maxBasketSize, avgBasketSize int + stats := DatabaseStats{} for _, name := range db.names { if basket, exists := db.baskets[name]; exists { - basketsCount++ - if basket.totalCount == 0 { - emptyBasketsCount++ + var lastRequestDate int64 + if basket.Size() > 0 { + lastRequestDate = basket.GetRequests(1, 0).Requests[0].Date } - requestsCount += basket.Size() - requestsTotalCount += basket.totalCount - if basket.totalCount > maxBasketSize { - maxBasketSize = basket.totalCount - } - } - } - if basketsCount > emptyBasketsCount { - avgBasketSize = requestsTotalCount / (basketsCount - emptyBasketsCount) + stats.Collect(&BasketInfo{ + Name: name, + RequestsCount: basket.Size(), + RequestsTotalCount: basket.totalCount, + LastRequestDate: lastRequestDate}, max) + } } - return DatabaseStats{ - BasketsCount: basketsCount, - EmptyBasketsCount: emptyBasketsCount, - RequestsCount: requestsCount, - RequestsTotalCount: requestsTotalCount, - MaxBasketSize: maxBasketSize, - AvgBasketSize: avgBasketSize} + stats.UpdateAvarage() + return stats } func (db *memoryDatabase) Release() { diff --git a/baskets_sql.go b/baskets_sql.go index f10a156..3d170a5 100644 --- a/baskets_sql.go +++ b/baskets_sql.go @@ -387,26 +387,17 @@ func (sdb *sqlDatabase) FindNames(query string, max int, skip int) BasketNamesQu return page } -func (sdb *sqlDatabase) GetStats() DatabaseStats { - var basketsCount, emptyBasketsCount, requestsCount, requestsTotalCount, maxBasketSize, avgBasketSize int +func (sdb *sqlDatabase) GetStats(max int) DatabaseStats { + stats := DatabaseStats{} - basketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) - emptyBasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets WHERE requests_count = 0", 0) - requestsCount = sdb.getInt("SELECT COUNT(*) FROM rb_requests", 0) - requestsTotalCount = sdb.getInt("SELECT SUM(requests_count) FROM rb_baskets", 0) - maxBasketSize = sdb.getInt("SELECT MAX(requests_count) FROM rb_baskets", 0) + stats.BasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) + stats.EmptyBasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets WHERE requests_count = 0", 0) + stats.RequestsCount = sdb.getInt("SELECT COUNT(*) FROM rb_requests", 0) + stats.RequestsTotalCount = sdb.getInt("SELECT SUM(requests_count) FROM rb_baskets", 0) + stats.MaxBasketSize = sdb.getInt("SELECT MAX(requests_count) FROM rb_baskets", 0) - if basketsCount > emptyBasketsCount { - avgBasketSize = requestsTotalCount / (basketsCount - emptyBasketsCount) - } - - return DatabaseStats{ - BasketsCount: basketsCount, - EmptyBasketsCount: emptyBasketsCount, - RequestsCount: requestsCount, - RequestsTotalCount: requestsTotalCount, - MaxBasketSize: maxBasketSize, - AvgBasketSize: avgBasketSize} + stats.UpdateAvarage() + return stats } func (sdb *sqlDatabase) Release() { diff --git a/handlers.go b/handlers.go index b75079a..697348e 100644 --- a/handlers.go +++ b/handlers.go @@ -143,12 +143,12 @@ func GetBaskets(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } else { values := r.URL.Query() if query := values.Get("q"); len(query) > 0 { - // Find names + // find names max, skip := getPage(values) json, err := json.Marshal(basketsDb.FindNames(query, max, skip)) writeJSON(w, http.StatusOK, json, err) } else { - // Get basket names page + // get basket names page json, err := json.Marshal(basketsDb.GetNames(getPage(values))) writeJSON(w, http.StatusOK, json, err) } @@ -160,8 +160,9 @@ func GetStats(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if r.Header.Get("Authorization") != serverConfig.MasterToken { w.WriteHeader(http.StatusUnauthorized) } else { - // Get database stats - json, err := json.Marshal(basketsDb.GetStats()) + // get database stats + max := parseInt(r.URL.Query().Get("max"), 1, 100, 5) + json, err := json.Marshal(basketsDb.GetStats(max)) writeJSON(w, http.StatusOK, json, err) } } @@ -313,12 +314,12 @@ func GetBasketRequests(w http.ResponseWriter, r *http.Request, ps httprouter.Par if _, basket := getAuthenticatedBasket(w, r, ps); basket != nil { values := r.URL.Query() if query := values.Get("q"); len(query) > 0 { - // Find requests + // find requests max, skip := getPage(values) json, err := json.Marshal(basket.FindRequests(query, values.Get("in"), max, skip)) writeJSON(w, http.StatusOK, json, err) } else { - // Get requests page + // get requests page json, err := json.Marshal(basket.GetRequests(getPage(values))) writeJSON(w, http.StatusOK, json, err) } diff --git a/web/baskets.html b/web/baskets.html index 6b760ea..f6c7022 100644 --- a/web/baskets.html +++ b/web/baskets.html @@ -19,8 +19,9 @@ h1 { margin-top: 2px; } #more { margin-left: 60px; padding-bottom: 10px; } #all_baskets ul { width: 100%; } - #all_baskets li { padding: 0 0 5px 20px; float: left; display: inline; position: relative; width: 25%; } - #all_baskets li:before { content: "\f291"; font-family: "FontAwesome"; position: absolute; left: 0px; top:0px; } + #all_baskets li { float: left; display: inline; position: relative; width: 25%; } + .baskets li { padding: 0 0 5px 20px; } + .baskets li:before { content: "\f291"; font-family: "FontAwesome"; position: absolute; left: 0px; top:0px; } @@ -227,12 +272,65 @@

Master Token

-

All Baskets

+
+
+

Basic Statistics

+
+
    +
  • + ? + Total baskets +
    • +
  • + ? + Empty baskets +
    • +
  • + ? + Total requests +
    • +
  • + ? + Current requests +
    • +
  • + ? + Max. basket size +
    • +
  • + ? + Avg. basket size +
    • +
+
+
+
+
+
+

Top Baskets: by size

+
+
    +
+
+
+
+
+
+

Top Baskets: recently active

+
+
    +
+
+
+
+
+
+

All Baskets

-
    +
    diff --git a/web_baskets.html.go b/web_baskets.html.go index 1efdd23..b35a644 100644 --- a/web_baskets.html.go +++ b/web_baskets.html.go @@ -19,8 +19,9 @@ var ( h1 { margin-top: 2px; } #more { margin-left: 60px; padding-bottom: 10px; } #all_baskets ul { width: 100%; } - #all_baskets li { padding: 0 0 5px 20px; float: left; display: inline; position: relative; width: 25%; } - #all_baskets li:before { content: "\f291"; font-family: "FontAwesome"; position: absolute; left: 0px; top:0px; } + #all_baskets li { float: left; display: inline; position: relative; width: 25%; } + .baskets li { padding: 0 0 5px 20px; } + .baskets li:before { content: "\f291"; font-family: "FontAwesome"; position: absolute; left: 0px; top:0px; } @@ -227,12 +272,65 @@ var (
    -

    All Baskets

    +
    +
    +

    Basic Statistics

    +
    +
      +
    • + ? + Total baskets +
      • +
    • + ? + Empty baskets +
      • +
    • + ? + Total requests +
      • +
    • + ? + Current requests +
      • +
    • + ? + Max. basket size +
      • +
    • + ? + Avg. basket size +
      • +
    +
    +
    +
    +
    +
    +

    Top Baskets: by size

    +
    +
      +
    +
    +
    +
    +
    +
    +

    Top Baskets: recently active

    +
    +
      +
    +
    +
    +
    +
    +
    +

    All Baskets

    -
      +
    From ab5b92c992d00818b4a933d5dd92d0275a5ff8d0 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Sat, 16 Feb 2019 18:14:56 +0100 Subject: [PATCH 04/10] implemented top baskests stats for SQL database type, simplified error handling in SQL storage in some cases --- baskets_sql.go | 77 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/baskets_sql.go b/baskets_sql.go index 3d170a5..56972f0 100644 --- a/baskets_sql.go +++ b/baskets_sql.go @@ -8,6 +8,7 @@ import ( "net/http" "regexp" "strings" + "time" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" @@ -87,6 +88,21 @@ func (basket *sqlBasket) applyLimit(capacity int) { } } +func (basket *sqlBasket) getTotalRequestsCount() int { + return basket.getInt("SELECT requests_count FROM rb_baskets WHERE basket_name = $1", 0) +} + +func (basket *sqlBasket) getLastRequestDate() int64 { + var value time.Time + if err := basket.db.QueryRow(unifySQL(basket.dbType, + "SELECT MAX(created_at) FROM rb_requests WHERE basket_name = $1"), basket.name).Scan(&value); err != nil { + log.Printf("[error] failed to get last request date of basket: %s - %s", basket.name, err) + return 0 + } + + return value.UnixNano() / toMs +} + func (basket *sqlBasket) Config() BasketConfig { config := BasketConfig{} @@ -198,8 +214,7 @@ func (basket *sqlBasket) Size() int { } func (basket *sqlBasket) GetRequests(max int, skip int) RequestsPage { - page := RequestsPage{make([]*RequestData, 0, max), basket.Size(), - basket.getInt("SELECT requests_count FROM rb_baskets WHERE basket_name = $1", 0), false} + page := RequestsPage{make([]*RequestData, 0, max), basket.Size(), basket.getTotalRequestsCount(), false} if max > 0 { requests, err := basket.db.Query( @@ -213,9 +228,7 @@ func (basket *sqlBasket) GetRequests(max int, skip int) RequestsPage { var req string for len(page.Requests) < max && requests.Next() { - if err = requests.Scan(&req); err != nil { - log.Printf("[error] failed to get request of basket: %s - %s", basket.name, err) - } else { + if err = requests.Scan(&req); err == nil { request := new(RequestData) if err = json.Unmarshal([]byte(req), request); err != nil { log.Printf("[error] failed to parse HTTP request data in basket: %s - %s", basket.name, err) @@ -247,9 +260,7 @@ func (basket *sqlBasket) FindRequests(query string, in string, max int, skip int skipped := 0 var req string for len(page.Requests) < max && requests.Next() { - if err = requests.Scan(&req); err != nil { - log.Printf("[error] failed to get request of basket: %s - %s", basket.name, err) - } else { + if err = requests.Scan(&req); err == nil { request := new(RequestData) if err = json.Unmarshal([]byte(req), request); err != nil { log.Printf("[error] failed to parse HTTP request data in basket: %s - %s", basket.name, err) @@ -282,14 +293,45 @@ type sqlDatabase struct { func (sdb *sqlDatabase) getInt(sql string, defaultValue int) int { var value int - if err := sdb.db.QueryRow(unifySQL(sdb.dbType, sql)).Scan(&value); err != nil { - log.Printf("[error] failed to query for int result: %s", err) + if err := sdb.db.QueryRow(sql).Scan(&value); err != nil { + log.Printf("[error] failed to query for int result, query: %s - %s", sql, err) return defaultValue } return value } +func (sdb *sqlDatabase) getTopBaskets(sql string, max int) []*BasketInfo { + top := make([]*BasketInfo, 0, max) + names, err := sdb.db.Query(unifySQL(sdb.dbType, sql), max) + if err != nil { + log.Printf("[error] failed to find top baskets: %s", err) + return top + } + defer names.Close() + + var name string + for names.Next() { + if err = names.Scan(&name); err == nil { + basket := &sqlBasket{sdb.db, sdb.dbType, name} + reqCount := basket.Size() + + var lastRequestDate int64 + if reqCount > 0 { + lastRequestDate = basket.getLastRequestDate() + } + + top = append(top, &BasketInfo{ + Name: name, + RequestsCount: reqCount, + RequestsTotalCount: basket.getTotalRequestsCount(), + LastRequestDate: lastRequestDate}) + } + } + + return top +} + func (sdb *sqlDatabase) Create(name string, config BasketConfig) (BasketAuth, error) { auth := BasketAuth{} token, err := GenerateToken() @@ -349,9 +391,7 @@ func (sdb *sqlDatabase) GetNames(max int, skip int) BasketNamesPage { var name string for len(page.Names) < max && names.Next() { - if err = names.Scan(&name); err != nil { - log.Printf("[error] failed to get basket name: %s", err) - } else { + if err = names.Scan(&name); err == nil { page.Names = append(page.Names, name) } } @@ -375,9 +415,7 @@ func (sdb *sqlDatabase) FindNames(query string, max int, skip int) BasketNamesQu var name string for len(page.Names) < max && names.Next() { - if err = names.Scan(&name); err != nil { - log.Printf("[error] failed to get basket name: %s", err) - } else { + if err = names.Scan(&name); err == nil { page.Names = append(page.Names, name) } } @@ -393,8 +431,11 @@ func (sdb *sqlDatabase) GetStats(max int) DatabaseStats { stats.BasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets", 0) stats.EmptyBasketsCount = sdb.getInt("SELECT COUNT(*) FROM rb_baskets WHERE requests_count = 0", 0) stats.RequestsCount = sdb.getInt("SELECT COUNT(*) FROM rb_requests", 0) - stats.RequestsTotalCount = sdb.getInt("SELECT SUM(requests_count) FROM rb_baskets", 0) - stats.MaxBasketSize = sdb.getInt("SELECT MAX(requests_count) FROM rb_baskets", 0) + stats.RequestsTotalCount = sdb.getInt("SELECT COALESCE(SUM(requests_count), 0) FROM rb_baskets", 0) + stats.MaxBasketSize = sdb.getInt("SELECT COALESCE(MAX(requests_count), 0) FROM rb_baskets", 0) + + stats.TopBasketsBySize = sdb.getTopBaskets("SELECT basket_name FROM rb_baskets ORDER BY requests_count DESC LIMIT $1", max) + stats.TopBasketsByDate = sdb.getTopBaskets("SELECT basket_name FROM rb_requests GROUP BY basket_name ORDER BY MAX(created_at) DESC LIMIT $1", max) stats.UpdateAvarage() return stats From d69c180cb40e0c4d2e3f880d8c7c5c2ee900ad91 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Wed, 20 Feb 2019 01:30:30 +0100 Subject: [PATCH 05/10] added unit tests for DatabaseStats methods --- baskets_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/baskets_test.go b/baskets_test.go index 5f59fdb..ce14a79 100644 --- a/baskets_test.go +++ b/baskets_test.go @@ -139,3 +139,50 @@ func TestExpandURL(t *testing.T) { assert.Equal(t, "/notify/hello/world", expandURL("/notify", "/notify/hello/world", "notify")) assert.Equal(t, "/receive/notification/test/", expandURL("/receive/notification/", "/basket/test/", "basket")) } + +func TestDatabaseStats_Collect(t *testing.T) { + stats := new(DatabaseStats) + stats.Collect(&BasketInfo{"a", 5, 10, 100}, 3) + stats.Collect(&BasketInfo{"b", 5, 30, 200}, 3) + stats.Collect(&BasketInfo{"c", 5, 5, 300}, 3) + stats.Collect(&BasketInfo{"d", 0, 0, 400}, 3) + stats.Collect(&BasketInfo{"e", 5, 20, 500}, 3) + stats.Collect(&BasketInfo{"f", 10, 40, 600}, 3) + stats.Collect(&BasketInfo{"g", 0, 0, 700}, 3) + stats.Collect(&BasketInfo{"h", 5, 5, 800}, 3) + + assert.Equal(t, 8, stats.BasketsCount, "wrong BasketsCount") + assert.Equal(t, 2, stats.EmptyBasketsCount, "wrong EmptyBasketsCount") + assert.Equal(t, 40, stats.MaxBasketSize, "wrong MaxBasketSize") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount") + assert.Equal(t, 110, stats.RequestsTotalCount, "wrong RequestsTotalCount") + + assert.Equal(t, 3, len(stats.TopBasketsByDate), "wrong number of TopBasketsByDate") + assert.Equal(t, "h", stats.TopBasketsByDate[0].Name) + assert.Equal(t, "g", stats.TopBasketsByDate[1].Name) + assert.Equal(t, "f", stats.TopBasketsByDate[2].Name) + + assert.Equal(t, 3, len(stats.TopBasketsBySize), "wrong number of TopBasketsBySize") + assert.Equal(t, "f", stats.TopBasketsBySize[0].Name) + assert.Equal(t, "b", stats.TopBasketsBySize[1].Name) + assert.Equal(t, "e", stats.TopBasketsBySize[2].Name) + + // we do not expect avarage basket size, it is not updated automatically + assert.Equal(t, 0, stats.AvgBasketSize, "unexpected AvgBasketSize") +} + +func TestDatabaseStats_UpdateAvarage(t *testing.T) { + stats := new(DatabaseStats) + stats.Collect(&BasketInfo{"a", 5, 10, 100}, 3) + stats.Collect(&BasketInfo{"b", 5, 20, 200}, 3) + stats.Collect(&BasketInfo{"c", 5, 30, 300}, 3) + + stats.UpdateAvarage() + assert.Equal(t, 20, stats.AvgBasketSize, "wrong AvgBasketSize") +} + +func TestDatabaseStats_UpdateAvarage_Empty(t *testing.T) { + stats := new(DatabaseStats) + stats.UpdateAvarage() + assert.Equal(t, 0, stats.AvgBasketSize, "wrong AvgBasketSize") +} From 7ea100695e957d8d56ce7e6fe19661e55522ff87 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Fri, 22 Feb 2019 00:41:27 +0100 Subject: [PATCH 06/10] unit tests to get stats from every supported type of baskets storage --- baskets_bolt_test.go | 48 +++++++++++++++++++++++++++++++++++++ baskets_mem_test.go | 48 +++++++++++++++++++++++++++++++++++++ baskets_sql_mysql_test.go | 48 +++++++++++++++++++++++++++++++++++++ baskets_sql_pg_test.go | 48 +++++++++++++++++++++++++++++++++++++ handlers_test.go | 50 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+) diff --git a/baskets_bolt_test.go b/baskets_bolt_test.go index 0ec6e47..75c13f7 100644 --- a/baskets_bolt_test.go +++ b/baskets_bolt_test.go @@ -444,6 +444,54 @@ func TestBoltBasket_SetResponse_Update(t *testing.T) { } } +func TestBoltDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewBoltDatabase(name + ".db") + defer db.Release() + defer os.Remove(name + ".db") + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} + func TestBoltBasket_InvalidBasket(t *testing.T) { name := "test199" db, _ := bolt.Open(name+".db", 0600, &bolt.Options{Timeout: 5 * time.Second}) diff --git a/baskets_mem_test.go b/baskets_mem_test.go index c0399be..c532176 100644 --- a/baskets_mem_test.go +++ b/baskets_mem_test.go @@ -7,6 +7,7 @@ import ( "net/url" "strings" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -400,3 +401,50 @@ func TestMemoryBasket_SetResponse_Update(t *testing.T) { } } } + +func TestMemoryDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewMemoryDatabase() + defer db.Release() + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} diff --git a/baskets_sql_mysql_test.go b/baskets_sql_mysql_test.go index 4c12198..a4f1c95 100644 --- a/baskets_sql_mysql_test.go +++ b/baskets_sql_mysql_test.go @@ -465,3 +465,51 @@ func TestMySQLBasket_SetResponse_Error(t *testing.T) { assert.Nil(t, basket.GetResponse(method), "Response for very long method name is not expected") } } + +func TestMySQLDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewSQLDatabase(mysqlTestConnection) + defer db.Release() + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + defer db.Delete(bname) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} diff --git a/baskets_sql_pg_test.go b/baskets_sql_pg_test.go index 548042a..b1177fa 100644 --- a/baskets_sql_pg_test.go +++ b/baskets_sql_pg_test.go @@ -465,3 +465,51 @@ func TestPgSQLBasket_SetResponse_Error(t *testing.T) { assert.Nil(t, basket.GetResponse(method), "Response for very long method name is not expected") } } + +func TestPgSQLDatabase_GetStats(t *testing.T) { + name := "test130" + db := NewSQLDatabase(pgTestConnection) + defer db.Release() + + config := BasketConfig{Capacity: 5} + for i := 0; i < 10; i++ { + bname := fmt.Sprintf("%s_%v", name, i) + db.Create(bname, config) + defer db.Delete(bname) + + // fill basket + basket := db.Get(bname) + for j := 0; j < 9-i; j++ { + basket.Add(createTestPOSTRequest( + fmt.Sprintf("http://localhost/%v?id=%v", bname, j), fmt.Sprintf("req%v", j), "text/plain")) + } + time.Sleep(20 * time.Millisecond) + } + + // get stats + stats := db.GetStats(3) + if assert.NotNil(t, stats, "database statistics is expected") { + assert.Equal(t, 10, stats.BasketsCount, "wrong BasketsCount stats") + assert.Equal(t, 1, stats.EmptyBasketsCount, "wrong EmptyBasketsCount stats") + assert.Equal(t, 9, stats.MaxBasketSize, "wrong MaxBasketSize stats") + assert.Equal(t, 35, stats.RequestsCount, "wrong RequestsCount stats") + assert.Equal(t, 45, stats.RequestsTotalCount, "wrong RequestsTotalCount stats") + assert.Equal(t, 5, stats.AvgBasketSize, "wrong AvgBasketSize stats") + + // top 3 by date + if assert.NotNil(t, stats.TopBasketsByDate, "top baskets by date are expected") { + assert.Equal(t, 3, len(stats.TopBasketsByDate), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 8), stats.TopBasketsByDate[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 7), stats.TopBasketsByDate[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 6), stats.TopBasketsByDate[2].Name) + } + + // top 3 by size + if assert.NotNil(t, stats.TopBasketsBySize, "top baskets by size are expected") { + assert.Equal(t, 3, len(stats.TopBasketsBySize), "unexpected number of top baskets") + assert.Equal(t, fmt.Sprintf("%s_%v", name, 0), stats.TopBasketsBySize[0].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 1), stats.TopBasketsBySize[1].Name) + assert.Equal(t, fmt.Sprintf("%s_%v", name, 2), stats.TopBasketsBySize[2].Name) + } + } +} diff --git a/handlers_test.go b/handlers_test.go index f6be45f..e8383f5 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -704,6 +704,56 @@ func TestGetBaskets_Unauthorized(t *testing.T) { } } +func TestGetStats(t *testing.T) { + // create 3 baskets + for i := 0; i < 3; i++ { + basket := fmt.Sprintf("forstats0%v", i) + r, err := http.NewRequest("POST", "http://localhost:55555/baskets/"+basket, strings.NewReader("")) + if assert.NoError(t, err) { + w := httptest.NewRecorder() + ps := append(make(httprouter.Params, 0), httprouter.Param{Key: "basket", Value: basket}) + CreateBasket(w, r, ps) + assert.Equal(t, 201, w.Code, "wrong HTTP result code") + } + } + + // get stats + r, err := http.NewRequest("GET", "http://localhost:55555/stats", strings.NewReader("")) + if assert.NoError(t, err) { + r.Header.Add("Authorization", serverConfig.MasterToken) + w := httptest.NewRecorder() + GetStats(w, r, make(httprouter.Params, 0)) + // HTTP 200 - OK + assert.Equal(t, 200, w.Code, "wrong HTTP result code") + + stats := new(DatabaseStats) + err = json.Unmarshal(w.Body.Bytes(), stats) + if assert.NoError(t, err) { + // validate response + assert.NotEmpty(t, stats.TopBasketsByDate, "top baskets are expected") + assert.NotEmpty(t, stats.TopBasketsBySize, "top baskets are expected") + assert.True(t, stats.BasketsCount > 0, "baskets count should be greater than 0") + assert.True(t, stats.EmptyBasketsCount > 0, "empty baskets count should be greater than 0") + } + } +} + +func TestGetStats_Unauthorized(t *testing.T) { + r, err := http.NewRequest("GET", "http://localhost:55555/stats", strings.NewReader("")) + if assert.NoError(t, err) { + // no authorization at all: 401 - unauthorized + w := httptest.NewRecorder() + GetStats(w, r, make(httprouter.Params, 0)) + assert.Equal(t, 401, w.Code, "wrong HTTP result code") + + // invalid master token: 401 - unauthorized + r.Header.Add("Authorization", "123-wrong-token") + w = httptest.NewRecorder() + GetStats(w, r, make(httprouter.Params, 0)) + assert.Equal(t, 401, w.Code, "wrong HTTP result code") + } +} + func TestGetBaskets_Query(t *testing.T) { // create 10 baskets for i := 0; i < 10; i++ { From ef4c1e640076da061cbf144c66c8e47563cdd692 Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Fri, 22 Feb 2019 01:12:39 +0100 Subject: [PATCH 07/10] move REST API end-points under /api/* path, still keep old /baskets/* path for compatibility --- config.go | 6 ++++-- handlers.go | 11 +++++++++-- handlers_test.go | 30 ++++++++++++++++++++++++++---- server.go | 45 ++++++++++++++++++++++++++++++++------------- version.go | 8 +++++--- 5 files changed, 76 insertions(+), 24 deletions(-) diff --git a/config.go b/config.go index b147c89..5930d2c 100644 --- a/config.go +++ b/config.go @@ -13,10 +13,12 @@ const ( initBasketCapacity = 200 maxBasketCapacity = 2000 defaultDatabaseType = DbTypeMemory - serviceAPIPath = "baskets" - serviceStatsPath = "stats" + serviceOldAPIPath = "baskets" + serviceAPIPath = "api" serviceUIPath = "web" + serviceName = "request-baskets" basketNamePattern = `^[\w\d\-_\.]{1,250}$` + sourceCodeURL = "https://github.com/darklynx/request-baskets" ) // ServerConfig describes server configuration. diff --git a/handlers.go b/handlers.go index 697348e..3980bd2 100644 --- a/handlers.go +++ b/handlers.go @@ -167,6 +167,13 @@ func GetStats(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } } +// GetVersion handles HTTP request to get service version details +func GetVersion(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + // get database stats + json, err := json.Marshal(version) + writeJSON(w, http.StatusOK, json, err) +} + // GetBasket handles HTTP request to get basket configuration func GetBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if _, basket := getAuthenticatedBasket(w, r, ps); basket != nil { @@ -178,7 +185,7 @@ func GetBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { // CreateBasket handles HTTP request to create a new basket func CreateBasket(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { name := ps.ByName("basket") - if name == serviceAPIPath || name == serviceStatsPath || name == serviceUIPath { + if name == serviceOldAPIPath || name == serviceAPIPath || name == serviceUIPath { http.Error(w, "This basket name conflicts with reserved system path: "+name, http.StatusForbidden) return } @@ -349,7 +356,7 @@ func WebIndexPage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) func WebBasketPage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if name := ps.ByName("basket"); validBasketName.MatchString(name) { switch name { - case serviceAPIPath: + case serviceOldAPIPath: // admin page to access all baskets w.Header().Set("Content-Type", "text/html; charset=utf-8") basketsPageTemplate.Execute(w, version) diff --git a/handlers_test.go b/handlers_test.go index e8383f5..44bd504 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -718,7 +718,7 @@ func TestGetStats(t *testing.T) { } // get stats - r, err := http.NewRequest("GET", "http://localhost:55555/stats", strings.NewReader("")) + r, err := http.NewRequest("GET", "http://localhost:55555/api/stats", strings.NewReader("")) if assert.NoError(t, err) { r.Header.Add("Authorization", serverConfig.MasterToken) w := httptest.NewRecorder() @@ -739,7 +739,7 @@ func TestGetStats(t *testing.T) { } func TestGetStats_Unauthorized(t *testing.T) { - r, err := http.NewRequest("GET", "http://localhost:55555/stats", strings.NewReader("")) + r, err := http.NewRequest("GET", "http://localhost:55555/api/stats", strings.NewReader("")) if assert.NoError(t, err) { // no authorization at all: 401 - unauthorized w := httptest.NewRecorder() @@ -754,6 +754,28 @@ func TestGetStats_Unauthorized(t *testing.T) { } } +func TestGetVersion(t *testing.T) { + // get version + r, err := http.NewRequest("GET", "http://localhost:55555/api/version", strings.NewReader("")) + if assert.NoError(t, err) { + w := httptest.NewRecorder() + GetVersion(w, r, make(httprouter.Params, 0)) + // HTTP 200 - OK + assert.Equal(t, 200, w.Code, "wrong HTTP result code") + + ver := new(Version) + err = json.Unmarshal(w.Body.Bytes(), ver) + if assert.NoError(t, err) { + // validate response + assert.Equal(t, serviceName, ver.Name) + assert.Equal(t, sourceCodeURL, ver.SourceCode) + assert.NotEmpty(t, ver.Version, "version is expected") + assert.NotEmpty(t, ver.Commit, "commit is expected") + assert.NotEmpty(t, ver.CommitShort, "commit short is expected") + } + } +} + func TestGetBaskets_Query(t *testing.T) { // create 10 baskets for i := 0; i < 10; i++ { @@ -1082,10 +1104,10 @@ func TestWebBasketPage(t *testing.T) { } func TestWebBasketsPage(t *testing.T) { - r, err := http.NewRequest("GET", "http://localhost:55555/web/"+serviceAPIPath, strings.NewReader("")) + r, err := http.NewRequest("GET", "http://localhost:55555/web/"+serviceOldAPIPath, strings.NewReader("")) if assert.NoError(t, err) { w := httptest.NewRecorder() - ps := append(make(httprouter.Params, 0), httprouter.Param{Key: "basket", Value: serviceAPIPath}) + ps := append(make(httprouter.Params, 0), httprouter.Param{Key: "basket", Value: serviceOldAPIPath}) WebBasketPage(w, r, ps) // validate response: 200 - OK diff --git a/server.go b/server.go index ee385b4..28af486 100644 --- a/server.go +++ b/server.go @@ -19,7 +19,12 @@ var version *Version // CreateServer creates an instance of Request Baskets server func CreateServer(config *ServerConfig) *http.Server { - version = &Version{Version: GitVersion, Commit: GitCommit, CommitShort: GitCommitShort} + version = &Version{ + Name: serviceName, + Version: GitVersion, + Commit: GitCommit, + CommitShort: GitCommitShort, + SourceCode: sourceCodeURL} log.Printf("[info] service version: %s from commit: %s (%s)", version.Version, version.CommitShort, version.Commit) // create database @@ -38,22 +43,36 @@ func CreateServer(config *ServerConfig) *http.Server { // configure service HTTP router router := httprouter.New() + //// Old API mapping //// // basket names - router.GET("/"+serviceAPIPath, GetBaskets) - router.GET("/"+serviceStatsPath, GetStats) - + router.GET("/"+serviceOldAPIPath, GetBaskets) // basket management - router.GET("/"+serviceAPIPath+"/:basket", GetBasket) - router.POST("/"+serviceAPIPath+"/:basket", CreateBasket) - router.PUT("/"+serviceAPIPath+"/:basket", UpdateBasket) - router.DELETE("/"+serviceAPIPath+"/:basket", DeleteBasket) - - router.GET("/"+serviceAPIPath+"/:basket/responses/:method", GetBasketResponse) - router.PUT("/"+serviceAPIPath+"/:basket/responses/:method", UpdateBasketResponse) + router.GET("/"+serviceOldAPIPath+"/:basket", GetBasket) + router.POST("/"+serviceOldAPIPath+"/:basket", CreateBasket) + router.PUT("/"+serviceOldAPIPath+"/:basket", UpdateBasket) + router.DELETE("/"+serviceOldAPIPath+"/:basket", DeleteBasket) + router.GET("/"+serviceOldAPIPath+"/:basket/responses/:method", GetBasketResponse) + router.PUT("/"+serviceOldAPIPath+"/:basket/responses/:method", UpdateBasketResponse) + // requests management + router.GET("/"+serviceOldAPIPath+"/:basket/requests", GetBasketRequests) + router.DELETE("/"+serviceOldAPIPath+"/:basket/requests", ClearBasket) + //// New API mapping //// + // service details + router.GET("/"+serviceAPIPath+"/stats", GetStats) + router.GET("/"+serviceAPIPath+"/version", GetVersion) + // basket names + router.GET("/"+serviceAPIPath+"/baskets", GetBaskets) + // basket management + router.GET("/"+serviceAPIPath+"/baskets/:basket", GetBasket) + router.POST("/"+serviceAPIPath+"/baskets/:basket", CreateBasket) + router.PUT("/"+serviceAPIPath+"/baskets/:basket", UpdateBasket) + router.DELETE("/"+serviceAPIPath+"/baskets/:basket", DeleteBasket) + router.GET("/"+serviceAPIPath+"/baskets/:basket/responses/:method", GetBasketResponse) + router.PUT("/"+serviceAPIPath+"/baskets/:basket/responses/:method", UpdateBasketResponse) // requests management - router.GET("/"+serviceAPIPath+"/:basket/requests", GetBasketRequests) - router.DELETE("/"+serviceAPIPath+"/:basket/requests", ClearBasket) + router.GET("/"+serviceAPIPath+"/baskets/:basket/requests", GetBasketRequests) + router.DELETE("/"+serviceAPIPath+"/baskets/:basket/requests", ClearBasket) // web pages router.GET("/", ForwardToWeb) diff --git a/version.go b/version.go index 9a4d753..d73eaef 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,9 @@ var ( // Version describes application version type Version struct { - Version string - Commit string - CommitShort string + Name string `json:"name"` + Version string `json:"version"` + Commit string `json:"commit"` + CommitShort string `json:"commit_short"` + SourceCode string `json:"source_code"` } From 1f9e4beb9950866cf5009f49a42f8ac0316fddcd Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Fri, 22 Feb 2019 01:26:34 +0100 Subject: [PATCH 08/10] updated UI to use new REST API end-points --- web/basket.html | 16 ++++++++-------- web/baskets.html | 8 ++++---- web/index.html | 2 +- web_basket.html.go | 16 ++++++++-------- web_baskets.html.go | 10 +++++----- web_index.html.go | 4 ++-- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/web/basket.html b/web/basket.html index cc71064..4a52d6f 100644 --- a/web/basket.html +++ b/web/basket.html @@ -257,7 +257,7 @@ function fetchRequests() { $.ajax({ method: "GET", - url: "/baskets/" + name + "/requests?skip=" + fetchedCount, + url: "/api/baskets/" + name + "/requests?skip=" + fetchedCount, headers: { "Authorization" : getToken() } @@ -267,7 +267,7 @@ function fetchTotalCount() { $.ajax({ method: "GET", - url: "/baskets/" + name + "/requests?max=0", + url: "/api/baskets/" + name + "/requests?max=0", headers: { "Authorization" : getToken() } @@ -283,7 +283,7 @@ $("#response_method").val(method); $.ajax({ method: "GET", - url: "/baskets/" + name + "/responses/" + method, + url: "/api/baskets/" + name + "/responses/" + method, headers: { "Authorization" : getToken() } @@ -357,7 +357,7 @@ $.ajax({ method: "PUT", - url: "/baskets/" + name + "/responses/" + method, + url: "/api/baskets/" + name + "/responses/" + method, dataType: "json", data: JSON.stringify(response), headers: { @@ -382,7 +382,7 @@ $.ajax({ method: "PUT", - url: "/baskets/" + name, + url: "/api/baskets/" + name, dataType: "json", data: JSON.stringify(currentConfig), headers: { @@ -421,7 +421,7 @@ function config() { $.ajax({ method: "GET", - url: "/baskets/" + name, + url: "/api/baskets/" + name, headers: { "Authorization" : getToken() } @@ -444,7 +444,7 @@ function deleteRequests() { $.ajax({ method: "DELETE", - url: "/baskets/" + name + "/requests", + url: "/api/baskets/" + name + "/requests", headers: { "Authorization" : getToken() } @@ -459,7 +459,7 @@ $.ajax({ method: "DELETE", - url: "/baskets/" + name, + url: "/api/baskets/" + name, headers: { "Authorization" : getToken() } diff --git a/web/baskets.html b/web/baskets.html index f6c7022..c68a579 100644 --- a/web/baskets.html +++ b/web/baskets.html @@ -102,7 +102,7 @@ function fetchBaskets() { $.ajax({ method: "GET", - url: "/baskets?skip=" + basketsCount, + url: "/api/baskets?skip=" + basketsCount, headers: { "Authorization" : sessionStorage.getItem("master_token") } @@ -114,7 +114,7 @@ function fetchStats() { $.ajax({ method: "GET", - url: "/stats", + url: "/api/stats", headers: { "Authorization" : sessionStorage.getItem("master_token") } @@ -127,14 +127,14 @@ function fetchBasketDetails(name, basketRowId) { $.ajax({ method: "GET", - url: "/baskets/" + name + "/requests?max=1", + url: "/api/baskets/" + name + "/requests?max=1", headers: { "Authorization" : sessionStorage.getItem("master_token") } }).done(function(requests) { $.ajax({ method: "GET", - url: "/baskets/" + name, + url: "/api/baskets/" + name, headers: { "Authorization" : sessionStorage.getItem("master_token") } diff --git a/web/index.html b/web/index.html index e1fc5da..f1bb249 100644 --- a/web/index.html +++ b/web/index.html @@ -53,7 +53,7 @@ function createBasket() { var basket = $.trim($("#basket_name").val()); if (basket) { - $.post("/baskets/" + basket, function(data) { + $.post("/api/baskets/" + basket, function(data) { localStorage.setItem("basket_" + basket, data.token); $("#created_message_text").html("

    Basket '" + basket + "' is successfully created!

    Your token is: " + data.token + "

    "); diff --git a/web_basket.html.go b/web_basket.html.go index 3aedcf9..f0b4ca6 100644 --- a/web_basket.html.go +++ b/web_basket.html.go @@ -256,7 +256,7 @@ const ( function fetchRequests() { $.ajax({ method: "GET", - url: "/baskets/{{.}}/requests?skip=" + fetchedCount, + url: "/api/baskets/{{.}}/requests?skip=" + fetchedCount, headers: { "Authorization" : getToken() } @@ -266,7 +266,7 @@ const ( function fetchTotalCount() { $.ajax({ method: "GET", - url: "/baskets/{{.}}/requests?max=0", + url: "/api/baskets/{{.}}/requests?max=0", headers: { "Authorization" : getToken() } @@ -282,7 +282,7 @@ const ( $("#response_method").val(method); $.ajax({ method: "GET", - url: "/baskets/{{.}}/responses/" + method, + url: "/api/baskets/{{.}}/responses/" + method, headers: { "Authorization" : getToken() } @@ -356,7 +356,7 @@ const ( $.ajax({ method: "PUT", - url: "/baskets/{{.}}/responses/" + method, + url: "/api/baskets/{{.}}/responses/" + method, dataType: "json", data: JSON.stringify(response), headers: { @@ -383,7 +383,7 @@ const ( $.ajax({ method: "PUT", - url: "/baskets/{{.}}", + url: "/api/baskets/{{.}}", dataType: "json", data: JSON.stringify(currentConfig), headers: { @@ -422,7 +422,7 @@ const ( function config() { $.ajax({ method: "GET", - url: "/baskets/{{.}}", + url: "/api/baskets/{{.}}", headers: { "Authorization" : getToken() } @@ -446,7 +446,7 @@ const ( function deleteRequests() { $.ajax({ method: "DELETE", - url: "/baskets/{{.}}/requests", + url: "/api/baskets/{{.}}/requests", headers: { "Authorization" : getToken() } @@ -461,7 +461,7 @@ const ( $.ajax({ method: "DELETE", - url: "/baskets/{{.}}", + url: "/api/baskets/{{.}}", headers: { "Authorization" : getToken() } diff --git a/web_baskets.html.go b/web_baskets.html.go index b35a644..d55375b 100644 --- a/web_baskets.html.go +++ b/web_baskets.html.go @@ -102,7 +102,7 @@ var ( function fetchBaskets() { $.ajax({ method: "GET", - url: "/baskets?skip=" + basketsCount, + url: "/api/baskets?skip=" + basketsCount, headers: { "Authorization" : sessionStorage.getItem("master_token") } @@ -114,7 +114,7 @@ var ( function fetchStats() { $.ajax({ method: "GET", - url: "/stats", + url: "/api/stats", headers: { "Authorization" : sessionStorage.getItem("master_token") } @@ -127,14 +127,14 @@ var ( function fetchBasketDetails(name, basketRowId) { $.ajax({ method: "GET", - url: "/baskets/" + name + "/requests?max=1", + url: "/api/baskets/" + name + "/requests?max=1", headers: { "Authorization" : sessionStorage.getItem("master_token") } }).done(function(requests) { $.ajax({ method: "GET", - url: "/baskets/" + name, + url: "/api/baskets/" + name, headers: { "Authorization" : sessionStorage.getItem("master_token") } @@ -353,7 +353,7 @@ var (

    - Powered by request-baskets | + Powered by {{.Name}} | Version: {{.Version}}

    diff --git a/web_index.html.go b/web_index.html.go index 5d35c37..31af2ef 100644 --- a/web_index.html.go +++ b/web_index.html.go @@ -53,7 +53,7 @@ var ( function createBasket() { var basket = $.trim($("#basket_name").val()); if (basket) { - $.post("/baskets/" + basket, function(data) { + $.post("/api/baskets/" + basket, function(data) { localStorage.setItem("basket_" + basket, data.token); $("#created_message_text").html("

    Basket '" + basket + "' is successfully created!

    Your token is: " + data.token + "

    "); @@ -174,7 +174,7 @@ var (

    - Powered by request-baskets | + Powered by {{.Name}} | Version: {{.Version}}

    From 6e07d6ca5e4f1ed169cb27f3521d63e7a951975d Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Fri, 22 Feb 2019 02:13:45 +0100 Subject: [PATCH 09/10] got rid of JSON swagger description, YAML is a source of API documentation --- doc/api-swagger.json | 538 ------------------------------------------- 1 file changed, 538 deletions(-) delete mode 100644 doc/api-swagger.json diff --git a/doc/api-swagger.json b/doc/api-swagger.json deleted file mode 100644 index 8d55831..0000000 --- a/doc/api-swagger.json +++ /dev/null @@ -1,538 +0,0 @@ -{ - "swagger" : "2.0", - "info" : { - "description" : "RESTful API of [Request Baskets](https://rbaskets.in) service.\n\nRequest Baskets is an open source project of a service to collect HTTP requests and inspect them via RESTful\nAPI or web UI.\n\nCheck out the [project page](https://github.com/darklynx/request-baskets) for more detailed description.\n", - "version" : "0.8", - "title" : "Request Baskets API", - "contact" : { - "name" : "darklynx", - "url" : "https://github.com/darklynx" - }, - "license" : { - "name" : "MIT", - "url" : "https://github.com/darklynx/request-baskets/blob/master/LICENSE" - } - }, - "host" : "rbaskets.in", - "basePath" : "/", - "tags" : [ { - "name" : "baskets", - "description" : "Manage baskets" - }, { - "name" : "responses", - "description" : "Configure basket responses" - }, { - "name" : "requests", - "description" : "Manage collected requests" - } ], - "schemes" : [ "https" ], - "consumes" : [ "application/json" ], - "produces" : [ "application/json" ], - "paths" : { - "/baskets" : { - "get" : { - "tags" : [ "baskets" ], - "summary" : "Get baskets", - "description" : "Fetches a list of basket names managed by service. Require master token.", - "parameters" : [ { - "name" : "max", - "in" : "query", - "description" : "Maximum number of basket names to return; default 20", - "required" : false, - "type" : "integer" - }, { - "name" : "skip", - "in" : "query", - "description" : "Number of basket names to skip; default 0", - "required" : false, - "type" : "integer" - }, { - "name" : "q", - "in" : "query", - "description" : "Query string to filter result, only those basket names that match the query will be included in response", - "required" : false, - "type" : "string" - } ], - "responses" : { - "200" : { - "description" : "OK. Returns list of available baskets.", - "schema" : { - "$ref" : "#/definitions/Baskets" - } - }, - "204" : { - "description" : "No Content. No baskets available for specified limits" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing master token" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - }, - "/baskets/{name}" : { - "get" : { - "tags" : [ "baskets" ], - "summary" : "Get basket settings", - "description" : "Retrieves configuration settings of this basket.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - } ], - "responses" : { - "200" : { - "description" : "OK. Returns basket configuration", - "schema" : { - "$ref" : "#/definitions/Config" - } - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "post" : { - "tags" : [ "baskets" ], - "summary" : "Create new basket", - "description" : "Creates a new basket with this name.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The name of new basket", - "required" : true, - "type" : "string" - }, { - "in" : "body", - "name" : "config", - "description" : "Basket configuration", - "required" : false, - "schema" : { - "$ref" : "#/definitions/Config" - } - } ], - "responses" : { - "201" : { - "description" : "Created. Indicates that basket is successfully created", - "schema" : { - "$ref" : "#/definitions/Token" - } - }, - "400" : { - "description" : "Bad Request. Failed to parse JSON into basket configuration object." - }, - "403" : { - "description" : "Forbidden. Indicates that basket name conflicts with reserved paths; e.g. `baskets`, `web`, etc." - }, - "409" : { - "description" : "Conflict. Indicates that basket with such name already exists" - }, - "422" : { - "description" : "Unprocessable Entity. Basket configuration is not valid." - } - } - }, - "put" : { - "tags" : [ "baskets" ], - "summary" : "Update basket settings", - "description" : "Updates configuration settings of this basket.\n\nSpecial configuration parameters for request forwarding:\n * `insecure_tls` controls certificate verification when forwarding requests. Setting this parameter to `true`\n allows to forward collected HTTP requests via HTTPS protocol even if the forward end-point is configured with\n self-signed TLS/SSL certificate. **Warning:** enabling this feature has known security implications.\n * `expand_path` changes the logic of constructing taget URL when forwarding requests. If this parameter is\n set to `true` the forward URL path will be expanded when original HTTP request contains compound path. For\n example, a basket with name **server1** is configured to forward all requests to `http://server1.intranet:8001/myservice`\n and it has received an HTTP request like `GET http://baskets.example.com/server1/component/123/events?status=OK`\n then depending on `expand_path` settings the request will be forwarded to:\n * `true` => `GET http://server1.intranet:8001/myservice/component/123/events?status=OK`\n * `false` => `GET http://server1.intranet:8001/myservice?status=OK`\n", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "in" : "body", - "name" : "config", - "description" : "New configuration to apply", - "required" : true, - "schema" : { - "$ref" : "#/definitions/Config" - } - } ], - "responses" : { - "204" : { - "description" : "No Content. Basket configuration is updated" - }, - "400" : { - "description" : "Bad Request. Failed to parse JSON into basket configuration object." - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - }, - "422" : { - "description" : "Unprocessable Entity. Basket configuration is not valid." - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "delete" : { - "tags" : [ "baskets" ], - "summary" : "Delete basket", - "description" : "Permanently deletes this basket and all collected requests.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - } ], - "responses" : { - "204" : { - "description" : "No Content. Basket is deleted" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - }, - "/baskets/{name}/responses/{method}" : { - "get" : { - "tags" : [ "responses" ], - "summary" : "Get response settings", - "description" : "Retrieves information about configured response of the basket. Service will reply with this response to any\nHTTP request sent to the basket with appropriate HTTP method.\n\nIf nothing is configured, the default response is HTTP 200 - OK with empty content.\n", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "name" : "method", - "in" : "path", - "description" : "The HTTP method this response is configured for", - "required" : true, - "type" : "string", - "enum" : [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] - } ], - "responses" : { - "200" : { - "description" : "OK. Returns configured response information", - "schema" : { - "$ref" : "#/definitions/Response" - } - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "put" : { - "tags" : [ "responses" ], - "summary" : "Update response settings", - "description" : "Allows to configure HTTP response of this basket. The service will reply with configured response to any HTTP\nrequest sent to the basket with appropriate HTTP method.\n\nIf nothing is configured, the default response is HTTP 200 - OK with empty content.\n", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "name" : "method", - "in" : "path", - "description" : "The HTTP method this response is configured for", - "required" : true, - "type" : "string", - "enum" : [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] - }, { - "in" : "body", - "name" : "response", - "description" : "HTTP response configuration", - "required" : true, - "schema" : { - "$ref" : "#/definitions/Response" - } - } ], - "responses" : { - "204" : { - "description" : "No Content. Response configuration is updated" - }, - "400" : { - "description" : "Bad Request. Failed to parse JSON into response configuration object." - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - }, - "422" : { - "description" : "Unprocessable Entity. Response configuration is not valid." - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - }, - "/baskets/{name}/requests" : { - "get" : { - "tags" : [ "requests" ], - "summary" : "Get collected requests", - "description" : "Fetches collection of requests collected by this basket.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - }, { - "name" : "max", - "in" : "query", - "description" : "Maximum number of requests to return; default 20", - "required" : false, - "type" : "integer" - }, { - "name" : "skip", - "in" : "query", - "description" : "Number of requests to skip; default 0", - "required" : false, - "type" : "integer" - }, { - "name" : "q", - "in" : "query", - "description" : "Query string to filter result, only requests that match the query will be included in response", - "required" : false, - "type" : "string" - }, { - "name" : "in", - "in" : "query", - "description" : "Defines what is taken into account when filtering is applied: `body` - search in content body of collected requests,\n`query` - search among query parameters of collected requests, `headers` - search among request header values,\n`any` - search anywhere; default `any`\n", - "required" : false, - "type" : "string", - "enum" : [ "any", "body", "query", "headers" ] - } ], - "responses" : { - "200" : { - "description" : "OK. Returns list of basket requests.", - "schema" : { - "$ref" : "#/definitions/Requests" - } - }, - "204" : { - "description" : "No Content. No requests found for specified limits" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - }, - "delete" : { - "tags" : [ "requests" ], - "summary" : "Delete all requests", - "description" : "Deletes all requests collected by this basket.", - "parameters" : [ { - "name" : "name", - "in" : "path", - "description" : "The basket name", - "required" : true, - "type" : "string" - } ], - "responses" : { - "204" : { - "description" : "No Content. Basket requests are cleared" - }, - "401" : { - "description" : "Unauthorized. Invalid or missing basket token" - }, - "404" : { - "description" : "Not Found. No basket with such name" - } - }, - "security" : [ { - "basket_token" : [ ] - } ] - } - } - }, - "securityDefinitions" : { - "basket_token" : { - "description" : "Basket assigned secure token", - "type" : "apiKey", - "name" : "Authorization", - "in" : "header" - } - }, - "definitions" : { - "Baskets" : { - "type" : "object", - "required" : [ "has_more", "names" ], - "properties" : { - "names" : { - "type" : "array", - "description" : "Collection of basket names", - "items" : { - "type" : "string" - } - }, - "count" : { - "type" : "integer", - "description" : "Total number of baskets in the system; not present if query is applied" - }, - "has_more" : { - "type" : "boolean", - "description" : "Indicates if there are more baskets to fetch" - } - } - }, - "Config" : { - "type" : "object", - "properties" : { - "forward_url" : { - "type" : "string", - "description" : "URL to forward all incoming requests of the basket, `empty` value disables forwarding" - }, - "proxy_response" : { - "type" : "boolean", - "description" : "If set to `true` this basket behaves as a full proxy: responses from underlying service configured in `forward_url`\nare passed back to clients of original requests. The configuration of basket responses is ignored in this case.\n" - }, - "insecure_tls" : { - "type" : "boolean", - "description" : "If set to `true` the certificate verification will be disabled if forward URL indicates HTTPS scheme.\n**Warning:** enabling this feature has known security implications.\n" - }, - "expand_path" : { - "type" : "boolean", - "description" : "If set to `true` the forward URL path will be expanded when original HTTP request contains compound path." - }, - "capacity" : { - "type" : "integer", - "description" : "Baskets capacity, defines maximum number of requests to store" - } - } - }, - "Token" : { - "type" : "object", - "required" : [ "token" ], - "properties" : { - "token" : { - "type" : "string", - "description" : "Secure token to manage the basket, generated by system" - } - } - }, - "Requests" : { - "type" : "object", - "required" : [ "has_more", "requests" ], - "properties" : { - "requests" : { - "type" : "array", - "description" : "Collection of collected requests", - "items" : { - "$ref" : "#/definitions/Request" - } - }, - "count" : { - "type" : "integer", - "description" : "Current number of collected requests hold by basket; not present if query is applied" - }, - "total_count" : { - "type" : "integer", - "description" : "Total number of all requests passed through this basket; not present if query is applied" - }, - "has_more" : { - "type" : "boolean", - "description" : "Indicates if there are more requests collected by basket to fetch" - } - } - }, - "Request" : { - "type" : "object", - "properties" : { - "date" : { - "type" : "integer", - "format" : "int64", - "description" : "Date and time of request in Unix time ms. format (number of miliseconds elapsed since January 1, 1970 UTC)" - }, - "headers" : { - "$ref" : "#/definitions/Headers" - }, - "content_length" : { - "type" : "integer", - "description" : "Content lenght of request" - }, - "body" : { - "type" : "string", - "description" : "Content of request body" - }, - "method" : { - "type" : "string", - "description" : "HTTP methof of request" - }, - "path" : { - "type" : "string", - "description" : "URL path of request" - }, - "query" : { - "type" : "string", - "description" : "Query parameters of request" - } - } - }, - "Headers" : { - "type" : "object", - "description" : "Map of HTTP headers, key represents name, value is array of values", - "additionalProperties" : { - "type" : "array", - "description" : "Collection of header values", - "items" : { - "type" : "string" - } - } - }, - "Response" : { - "type" : "object", - "properties" : { - "status" : { - "type" : "integer", - "description" : "The HTTP status code to reply with" - }, - "headers" : { - "$ref" : "#/definitions/Headers" - }, - "body" : { - "type" : "string", - "description" : "Content of response body" - }, - "is_template" : { - "type" : "boolean", - "description" : "If set to `true` the body is treated as [HTML template](https://golang.org/pkg/html/template) that accepts\ninput from request parameters.\n" - } - } - } - } -} From df01c198baeaf6ddb3fd244266b84ece105a8aaa Mon Sep 17 00:00:00 2001 From: Vladimir Lyubitelev Date: Fri, 22 Feb 2019 02:15:05 +0100 Subject: [PATCH 10/10] described new end-point, /baskets/* paths marked as deprecated, /api/* paths should be used instead --- doc/api-swagger.yaml | 420 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 416 insertions(+), 4 deletions(-) diff --git a/doc/api-swagger.yaml b/doc/api-swagger.yaml index 2ce4d07..c9965dc 100644 --- a/doc/api-swagger.yaml +++ b/doc/api-swagger.yaml @@ -4,7 +4,7 @@ swagger: '2.0' info: - version: '0.8' + version: '1.0.0' title: Request Baskets API description: | RESTful API of [Request Baskets](https://rbaskets.in) service. @@ -31,6 +31,8 @@ produces: # Groups and their descriptions tags: + - name: service + description: Service information - name: baskets description: Manage baskets - name: responses @@ -48,11 +50,341 @@ securityDefinitions: # URL patterns paths: + /api/version: + get: + tags: + - service + summary: Get service version + description: Get service version. + responses: + 200: + description: OK. Returns service version. + schema: + $ref: '#/definitions/Version' + + /api/stats: + get: + tags: + - baskets + summary: Get baskets statistics + description: Get service statistics about baskets and collected HTTP requests. Require master token. + parameters: + - name: max + in: query + type: integer + description: Maximum number of basket names to return; default 5 + required: false + responses: + 200: + description: OK. Returns service statistics. + schema: + $ref: '#/definitions/ServiceStats' + 401: + description: Unauthorized. Invalid or missing master token + security: + - basket_token: [] + + /api/baskets: + get: + tags: + - baskets + summary: Get baskets + description: Fetches a list of basket names managed by service. Require master token. + parameters: + - name: max + in: query + type: integer + description: Maximum number of basket names to return; default 20 + required: false + - name: skip + in: query + type: integer + description: Number of basket names to skip; default 0 + required: false + - name: q + in: query + type: string + description: Query string to filter result, only those basket names that match the query will be included in response + required: false + responses: + 200: + description: OK. Returns list of available baskets. + schema: + $ref: '#/definitions/Baskets' + 204: + description: No Content. No baskets available for specified limits + 401: + description: Unauthorized. Invalid or missing master token + security: + - basket_token: [] + + /api/baskets/{name}: + post: + tags: + - baskets + summary: Create new basket + description: Creates a new basket with this name. + parameters: + - name: name + in: path + type: string + description: The name of new basket + required: true + - name: config + in: body + description: Basket configuration + required: false + schema: + $ref: '#/definitions/Config' + responses: + 201: + description: Created. Indicates that basket is successfully created + schema: + $ref: '#/definitions/Token' + 400: + description: Bad Request. Failed to parse JSON into basket configuration object. + 403: + description: Forbidden. Indicates that basket name conflicts with reserved paths; e.g. `baskets`, `web`, etc. + 409: + description: Conflict. Indicates that basket with such name already exists + 422: + description: Unprocessable Entity. Basket configuration is not valid. + get: + tags: + - baskets + summary: Get basket settings + description: Retrieves configuration settings of this basket. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + responses: + 200: + description: OK. Returns basket configuration + schema: + $ref: '#/definitions/Config' + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + put: + tags: + - baskets + summary: Update basket settings + description: | + Updates configuration settings of this basket. + + Special configuration parameters for request forwarding: + * `insecure_tls` controls certificate verification when forwarding requests. Setting this parameter to `true` + allows to forward collected HTTP requests via HTTPS protocol even if the forward end-point is configured with + self-signed TLS/SSL certificate. **Warning:** enabling this feature has known security implications. + * `expand_path` changes the logic of constructing taget URL when forwarding requests. If this parameter is + set to `true` the forward URL path will be expanded when original HTTP request contains compound path. For + example, a basket with name **server1** is configured to forward all requests to `http://server1.intranet:8001/myservice` + and it has received an HTTP request like `GET http://baskets.example.com/server1/component/123/events?status=OK` + then depending on `expand_path` settings the request will be forwarded to: + * `true` => `GET http://server1.intranet:8001/myservice/component/123/events?status=OK` + * `false` => `GET http://server1.intranet:8001/myservice?status=OK` + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: config + in: body + description: New configuration to apply + required: true + schema: + $ref: '#/definitions/Config' + responses: + 204: + description: No Content. Basket configuration is updated + 400: + description: Bad Request. Failed to parse JSON into basket configuration object. + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + 422: + description: Unprocessable Entity. Basket configuration is not valid. + security: + - basket_token: [] + delete: + tags: + - baskets + summary: Delete basket + description: Permanently deletes this basket and all collected requests. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + responses: + 204: + description: No Content. Basket is deleted + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + + /api/baskets/{name}/responses/{method}: + get: + tags: + - responses + summary: Get response settings + description: | + Retrieves information about configured response of the basket. Service will reply with this response to any + HTTP request sent to the basket with appropriate HTTP method. + + If nothing is configured, the default response is HTTP 200 - OK with empty content. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: method + in: path + type: string + enum: [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] + description: The HTTP method this response is configured for + required: true + responses: + 200: + description: OK. Returns configured response information + schema: + $ref: '#/definitions/Response' + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + put: + tags: + - responses + summary: Update response settings + description: | + Allows to configure HTTP response of this basket. The service will reply with configured response to any HTTP + request sent to the basket with appropriate HTTP method. + + If nothing is configured, the default response is HTTP 200 - OK with empty content. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: method + in: path + type: string + enum: [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE" ] + description: The HTTP method this response is configured for + required: true + - name: response + in: body + description: HTTP response configuration + required: true + schema: + $ref: '#/definitions/Response' + responses: + 204: + description: No Content. Response configuration is updated + 400: + description: Bad Request. Failed to parse JSON into response configuration object. + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + 422: + description: Unprocessable Entity. Response configuration is not valid. + security: + - basket_token: [] + + /api/baskets/{name}/requests: + get: + tags: + - requests + summary: Get collected requests + description: Fetches collection of requests collected by this basket. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + - name: max + in: query + type: integer + description: Maximum number of requests to return; default 20 + required: false + - name: skip + in: query + type: integer + description: Number of requests to skip; default 0 + required: false + - name: q + in: query + type: string + description: Query string to filter result, only requests that match the query will be included in response + required: false + - name: in + in: query + type: string + enum: [ 'any', 'body', 'query', 'headers' ] + description: | + Defines what is taken into account when filtering is applied: `body` - search in content body of collected requests, + `query` - search among query parameters of collected requests, `headers` - search among request header values, + `any` - search anywhere; default `any` + required: false + responses: + 200: + description: OK. Returns list of basket requests. + schema: + $ref: '#/definitions/Requests' + 204: + description: No Content. No requests found for specified limits + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + delete: + tags: + - requests + summary: Delete all requests + description: Deletes all requests collected by this basket. + parameters: + - name: name + in: path + type: string + description: The basket name + required: true + responses: + 204: + description: No Content. Basket requests are cleared + 401: + description: Unauthorized. Invalid or missing basket token + 404: + description: Not Found. No basket with such name + security: + - basket_token: [] + /baskets: get: tags: - baskets summary: Get baskets + deprecated: true description: Fetches a list of basket names managed by service. Require master token. parameters: - name: max @@ -87,6 +419,7 @@ paths: tags: - baskets summary: Create new basket + deprecated: true description: Creates a new basket with this name. parameters: - name: name @@ -117,6 +450,7 @@ paths: tags: - baskets summary: Get basket settings + deprecated: true description: Retrieves configuration settings of this basket. parameters: - name: name @@ -139,6 +473,7 @@ paths: tags: - baskets summary: Update basket settings + deprecated: true description: | Updates configuration settings of this basket. @@ -182,6 +517,7 @@ paths: tags: - baskets summary: Delete basket + deprecated: true description: Permanently deletes this basket and all collected requests. parameters: - name: name @@ -204,6 +540,7 @@ paths: tags: - responses summary: Get response settings + deprecated: true description: | Retrieves information about configured response of the basket. Service will reply with this response to any HTTP request sent to the basket with appropriate HTTP method. @@ -236,6 +573,7 @@ paths: tags: - responses summary: Update response settings + deprecated: true description: | Allows to configure HTTP response of this basket. The service will reply with configured response to any HTTP request sent to the basket with appropriate HTTP method. @@ -278,6 +616,7 @@ paths: tags: - requests summary: Get collected requests + deprecated: true description: Fetches collection of requests collected by this basket. parameters: - name: name @@ -326,6 +665,7 @@ paths: tags: - requests summary: Delete all requests + deprecated: true description: Deletes all requests collected by this basket. parameters: - name: name @@ -345,6 +685,78 @@ paths: # Model definitions: + Version: + type: object + properties: + name: + type: string + description: Service name + version: + type: string + description: Service version + commit: + type: string + description: Git commit this service is build from + commit_short: + type: string + description: Short form of git commit this service is build from + source_code: + type: string + description: URL of the source code repository + + ServiceStats: + type: object + properties: + baskets_count: + type: integer + description: Total number of baskets managed by service + empty_baskets_count: + type: integer + description: Number of empty baskets + requests_count: + type: integer + description: Number of HTTP requests currently stored by service + requests_total_count: + type: integer + description: Total number of HTTP requests processed by service + max_basket_size: + type: integer + description: Size of the biggest basket that processed the top most number of HTTP requests + avg_basket_size: + type: integer + description: Average size of a basket in the system, empty baskets are not taken into account + top_baskets_size: + type: array + description: Collection of top basket by size + items: + $ref: '#/definitions/BasketInfo' + top_baskets_recent: + type: array + description: Collection of top baskets recently active + items: + $ref: '#/definitions/BasketInfo' + + BasketInfo: + type: object + properties: + name: + type: string + description: Basket name + requests_count: + type: integer + description: Current number of collected HTTP requests held by basket + requests_total_count: + type: integer + description: Total number of all HTTP requests passed through this basket + last_request_date: + type: integer + format: int64 + description: | + Date and time of last request processed through this basket in Unix time ms. + format (number of milliseconds elapsed since January 1, 1970 UTC). + + If no requests were collected by this basket `0` is returned. + Baskets: type: object required: @@ -422,18 +834,18 @@ definitions: date: type: integer format: int64 - description: Date and time of request in Unix time ms. format (number of miliseconds elapsed since January 1, 1970 UTC) + description: Date and time of request in Unix time ms. format (number of milliseconds elapsed since January 1, 1970 UTC) headers: $ref: '#/definitions/Headers' content_length: type: integer - description: Content lenght of request + description: Content length of request body: type: string description: Content of request body method: type: string - description: HTTP methof of request + description: HTTP method of request path: type: string description: URL path of request