Merge d171549 into 9d4a4e0

apache · Dec 26, 2020 · 706e0ce · 706e0ce
2 parents 9d4a4e0 + d171549
commit 706e0ce
Show file tree

Hide file tree

Showing 9 changed files with 70 additions and 52 deletions.
diff --git a/deployments/db.js b/deployments/db.js
@@ -83,15 +83,15 @@ db.createCollection( "polling_detail", {
     max: 100,
     validator: { $jsonSchema: {
             bsonType: "object",
-            required: [ "id","session_id","domain","url_path" ],
+            required: [ "id","session_id","revision","url_path" ],
             properties: {
                 id: {
                     bsonType: "string",
                 },
                 domain: {
                     bsonType: "string",
                 },
-                params: {
+                polling_data: {
                     bsonType: "object"
                 },
                 ip: {
@@ -103,8 +103,8 @@ db.createCollection( "polling_detail", {
                 response_body: {
                     bsonType: "object"
                 },
-                response_header: {
-                    bsonType: "object"
+                revision: {
+                    bsonType: "string"
                 },
                 response_code: {
                     bsonType: "number"
@@ -114,15 +114,14 @@ db.createCollection( "polling_detail", {
 } );
 
 //index
-db.kv.createIndex({"id": 1}, { unique: true } );
-db.kv.createIndex({key: 1, label_format: 1,domain:1,project:1},{ unique: true });
-db.kv_revision.createIndex( { "delete_time": 1 }, { expireAfterSeconds: 7 * 24 * 3600 } );
-db.label.createIndex({"id": 1}, { unique: true } );
-db.label.createIndex({format: 1,domain:1,project:1},{ unique: true });
-db.polling_detail.createIndex({"id": 1}, { unique: true } );
-db.polling_detail.createIndex({session_id:1,domain:1}, { unique: true } );
-db.counter.createIndex({name: 1,domain:1},{ unique: true });
-db.view.createIndex({"id": 1}, { unique: true } );
-db.view.createIndex({display:1,domain:1,project:1},{ unique: true });
+db.kv.createIndex({"id": 1}, {unique: true});
+db.kv.createIndex({key: 1, label_format: 1, domain: 1, project: 1}, {unique: true});
+db.kv_revision.createIndex({"delete_time": 1}, {expireAfterSeconds: 7 * 24 * 3600});
+db.label.createIndex({"id": 1}, {unique: true});
+db.label.createIndex({format: 1, domain: 1, project: 1}, {unique: true});
+db.polling_detail.createIndex({timestamp: 1}, {expireAfterSeconds: 7 * 24 * 3600});
+db.counter.createIndex({name: 1, domain: 1}, {unique: true});
+db.view.createIndex({"id": 1}, {unique: true});
+db.view.createIndex({display: 1, domain: 1, project: 1}, {unique: true});
 //db config
-db.setProfilingLevel(1, {slowms: 80, sampleRate: 1} );
+db.setProfilingLevel(1, {slowms: 80, sampleRate: 1});
diff --git a/go.sum b/go.sum
@@ -87,6 +87,7 @@ github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4 h1:wx8JXvg
 github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4/go.mod h1:21/ajGtgJlWTCeM0TxGJdRhO8bJkKirWyV8Stlh6g6c=
 github.com/go-chassis/foundation v0.1.1-0.20200825060850-b16bf420f7b3 h1:c+bwT0qLY69jSU8TmzuNcb9UL/QFAiU96kjuX5TMiQc=
 github.com/go-chassis/foundation v0.1.1-0.20200825060850-b16bf420f7b3/go.mod h1:21/ajGtgJlWTCeM0TxGJdRhO8bJkKirWyV8Stlh6g6c=
+github.com/go-chassis/foundation v0.2.2-0.20201208060254-d5e8e5beb1f0 h1:6oazFyZKryG3r7K20UuWx+bclHMFGXjKBSMJpCefmMI=
 github.com/go-chassis/foundation v0.2.2-0.20201208060254-d5e8e5beb1f0/go.mod h1:2PjwqpVwYEVaAldl5A58a08viH8p27pNeYaiE3ZxOBA=
 github.com/go-chassis/go-archaius v1.2.1-0.20200309104817-8c3d4e87d33c h1:pimEM4Oy/Uf4xG4G7TrRUQbIRFAfHiarxDQQS2gmKaM=
 github.com/go-chassis/go-archaius v1.2.1-0.20200309104817-8c3d4e87d33c/go.mod h1:gVP52u/jCU0fgUjXdUW1VLp5YLLJ+Yl2zoOPrLM/WOM=
@@ -134,6 +135,7 @@ github.com/go-chassis/go-chassis/v2 v2.0.2 h1:FMiZgAa6IjDobtZN5oql+eSgMIlDq5+12o
 github.com/go-chassis/go-chassis/v2 v2.0.2/go.mod h1:xgXXrRUgn5sybtecTDn60ypepHKIV9QpSqR3CIEBNt0=
 github.com/go-chassis/go-chassis/v2 v2.1.0 h1:NnVUBj3U/pPijpZGooH+3QIRlOB2nAuZtx9b71aiaig=
 github.com/go-chassis/go-chassis/v2 v2.1.0/go.mod h1:DHA83QKVmuOtdkaWCP5yCDOh+GIRH+YpX3uZ2BlGMWw=
+github.com/go-chassis/go-chassis/v2 v2.1.1-0.20201208062518-9c2e86bd7a6c h1:k+D2w1oZ/fQjingJc1KzuwGv0cJKfaGDgaOW/4T4Q1c=
 github.com/go-chassis/go-chassis/v2 v2.1.1-0.20201208062518-9c2e86bd7a6c/go.mod h1:C5j2kuBAdch4DsSjKQxiYJr3Xy0PoZu2Bxx9QIJIArI=
 github.com/go-chassis/go-restful-swagger20 v1.0.3-0.20200310030431-17d80f34264f h1:5QmmNpVcGqIc6tuKNe5EAI4PA8Yn2EL9Oee7YdcJ4PE=
 github.com/go-chassis/go-restful-swagger20 v1.0.3-0.20200310030431-17d80f34264f/go.mod h1:eW62fYuzlNFDvIacB6AV8bhUDCTy4myfTCv0bT9Gbb0=
@@ -432,6 +434,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -598,6 +601,7 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

diff --git a/pkg/common/common.go b/pkg/common/common.go
@@ -21,23 +21,23 @@ import "time"
 
 //match mode
 const (
-	QueryParamQ          = "q"
-	QueryByLabelsCon     = "&"
-	QueryParamWait       = "wait"
-	QueryParamRev        = "revision"
-	QueryParamMatch      = "match"
-	QueryParamKey        = "key"
-	QueryParamLabel      = "label"
-	QueryParamStatus     = "status"
-	QueryParamOffset     = "offset"
-	QueryParamLimit      = "limit"
-	PathParamKVID        = "kv_id"
-	PathParameterProject = "project"
-	//polling data
-	QueryParamSessionID = "sessionId"
-	QueryParamIP        = "ip"
-	QueryParamURLPath   = "urlPath"
-	QueryParamUserAgent = "userAgent"
+	QueryParamQ            = "q"
+	QueryByLabelsCon       = "&"
+	QueryParamWait         = "wait"
+	QueryParamRev          = "revision"
+	QueryParamMatch        = "match"
+	QueryParamKey          = "key"
+	QueryParamLabel        = "label"
+	QueryParamStatus       = "status"
+	QueryParamOffset       = "offset"
+	QueryParamLimit        = "limit"
+	PathParamKVID          = "kv_id"
+	PathParameterProject   = "project"
+	QueryParamSessionID    = "sessionId"
+	QueryParamSessionGroup = "sessionGroup"
+	QueryParamIP           = "ip"
+	QueryParamURLPath      = "urlPath"
+	QueryParamUserAgent    = "userAgent"
 )
 
 //http headers

diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go
@@ -17,6 +17,8 @@
 
 package model
 
+import "time"
+
 //LabelDoc is database struct to store labels
 type LabelDoc struct {
 	ID      string            `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"`
@@ -57,16 +59,18 @@ type ViewDoc struct {
 
 //PollingDetail is db struct, it record operation history
 type PollingDetail struct {
-	ID             string                 `json:"id,omitempty" yaml:"id,omitempty"`
-	SessionID      string                 `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"`
-	Domain         string                 `json:"domain,omitempty" yaml:"domain,omitempty"`
-	PollingData    map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"`
-	IP             string                 `json:"ip,omitempty" yaml:"ip,omitempty"`
-	UserAgent      string                 `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"`
-	URLPath        string                 `json:"url_path,omitempty"  bson:"url_path,"  yaml:"url_path,omitempty"`
-	ResponseBody   []*KVDoc               `json:"kv,omitempty"  bson:"kv,"  yaml:"kv,omitempty"`
-	ResponseHeader map[string][]string    `json:"response_header,omitempty"  bson:"response_header,"  yaml:"response_header,omitempty"`
-	ResponseCode   int                    `json:"response_code,omitempty"  bson:"response_code,"  yaml:"response_code,omitempty"`
+	ID           string                 `json:"id,omitempty" yaml:"id,omitempty"`
+	SessionID    string                 `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"`
+	SessionGroup string                 `json:"session_group,omitempty" bson:"session_group," yaml:"session_group,omitempty"`
+	Domain       string                 `json:"domain,omitempty" yaml:"domain,omitempty"`
+	PollingData  map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"`
+	Revision     string                 `json:"revision,omitempty" yaml:"revision,omitempty"`
+	IP           string                 `json:"ip,omitempty" yaml:"ip,omitempty"`
+	UserAgent    string                 `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"`
+	URLPath      string                 `json:"url_path,omitempty"  bson:"url_path,"  yaml:"url_path,omitempty"`
+	ResponseBody []*KVDoc               `json:"kv,omitempty"  bson:"kv,"  yaml:"kv,omitempty"`
+	ResponseCode int                    `json:"response_code,omitempty"  bson:"response_code,"  yaml:"response_code,omitempty"`
+	Timestamp    time.Time              `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
 }
 
 // UpdateKVRequest is db struct, it contains kv update request params

diff --git a/server/handler/track_handler.go b/server/handler/track_handler.go
@@ -29,6 +29,7 @@ import (
 	"github.com/go-chassis/openlog"
 	"net/http"
 	"strings"
+	"time"
 )
 
 //const of noop auth handler
@@ -66,13 +67,15 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation,
 		data := &model.PollingDetail{}
 		data.URLPath = req.Request.Method + " " + req.Request.URL.Path
 		data.SessionID = sessionID
+		data.SessionGroup = req.HeaderParameter(v1.HeaderSessionGroup)
 		data.UserAgent = req.HeaderParameter(v1.HeaderUserAgent)
 		data.Domain = v1.ReadDomain(req.Request.Context())
 		data.IP = iputil.ClientIP(req.Request)
 		data.ResponseBody = req.Attribute(common.RespBodyContextKey).([]*model.KVDoc)
 		data.ResponseCode = ir.Status
+		data.Timestamp = time.Now()
 		if resp != nil {
-			data.ResponseHeader = resp.Header()
+			data.Revision = resp.Header().Get(common.HeaderRevision)
 		}
 		data.PollingData = map[string]interface{}{
 			"revision": revStr,

diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go
@@ -43,6 +43,7 @@ import (
 const (
 	HeaderUserAgent    = "User-Agent"
 	HeaderSessionID    = "X-Session-Id"
+	HeaderSessionGroup = "X-Session-Group"
 	AttributeDomainKey = "domain"
 
 	FmtReadRequestError = "decode request body failed: %v"

diff --git a/server/resource/v1/history_resource.go b/server/resource/v1/history_resource.go
@@ -72,6 +72,14 @@ func (r *HistoryResource) GetPollingData(context *restful.Context) {
 	if sessionID != "" {
 		query.SessionID = sessionID
 	}
+	sessionCroup := context.ReadQueryParameter(common.QueryParamSessionGroup)
+	if sessionCroup != "" {
+		query.SessionGroup = sessionCroup
+	}
+	revision := context.ReadQueryParameter(common.QueryParamRev)
+	if revision != "" {
+		query.Revision = revision
+	}
 	ip := context.ReadQueryParameter(common.QueryParamIP)
 	if ip != "" {
 		query.IP = ip

diff --git a/server/service/mongo/session/session.go b/server/service/mongo/session/session.go
@@ -222,19 +222,14 @@ func ensureKVLongPolling(session *mgo.Session) {
 	c := session.DB(DBName).C(CollectionPollingDetail)
 	err := c.Create(&mgo.CollectionInfo{Validator: bson.M{
 		"id":         bson.M{"$exists": true},
-		"params":     bson.M{"$exists": true},
+		"revision":   bson.M{"$exists": true},
 		"session_id": bson.M{"$exists": true},
 		"url_path":   bson.M{"$exists": true},
 	}})
 	wrapError(err, MsgDBExists)
 	err = c.EnsureIndex(mgo.Index{
-		Key:    []string{"id"},
-		Unique: true,
-	})
-	wrapError(err)
-	err = c.EnsureIndex(mgo.Index{
-		Key:    []string{"session_id", "domain"},
-		Unique: true,
+		Key:         []string{"timestamp"},
+		ExpireAfter: 7 * 24 * time.Hour,
 	})
 	wrapError(err)
 }

diff --git a/server/service/mongo/track/polling_detail_dao.go b/server/service/mongo/track/polling_detail_dao.go
@@ -29,9 +29,10 @@ import (
 )
 
 //CreateOrUpdate create a record or update exist record
+//If revision and session_id is exist: update else:insert
 func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) {
 	collection := session.GetDB().Collection(session.CollectionPollingDetail)
-	queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID}
+	queryFilter := bson.M{"revision": detail.Domain, "session_id": detail.SessionID}
 	res := collection.FindOne(ctx, queryFilter)
 	if res.Err() != nil {
 		if res.Err() == mongo.ErrNoDocuments {
@@ -67,6 +68,9 @@ func Get(ctx context.Context, detail *model.PollingDetail) ([]*model.PollingDeta
 	if detail.URLPath != "" {
 		queryFilter["url_path"] = detail.URLPath
 	}
+	if detail.Revision != "" {
+		queryFilter["revision"] = detail.Revision
+	}
 	cur, err := collection.Find(ctx, queryFilter)
 	if err != nil {
 		return nil, err