Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#79 record polling history #90

Closed
wants to merge 10 commits into from
Closed

#79 record polling history #90

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ce1007f
record polling history
GuoYL123 Jan 21, 2020
f3fffdb
test CI
GuoYL123 Feb 11, 2020
cd7058f
delete ut
GuoYL123 Feb 11, 2020
ec27487
fix bug cause exist error
GuoYL123 Feb 11, 2020
1d491fc
modify as comment
GuoYL123 Feb 12, 2020
ec61e3d
Merge remote-tracking branch 'remotes/origin/master' into record-new
GuoYL123 Feb 12, 2020
205d991
add session id
GuoYL123 Feb 12, 2020
7eeeede
rename insID
GuoYL123 Feb 12, 2020
01f662a
avoid use waitGroup and delete handler
GuoYL123 Feb 12, 2020
432f862
Golint checker
GuoYL123 Feb 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 42 additions & 0 deletions deployments/db.js
View file
Open in desktop
Expand Up @@ -96,11 +96,53 @@ db.createCollection( "view", {
}
} }
} );

db.createCollection( "polling_detail", {
validator: { $jsonSchema: {
bsonType: "object",
required: [ "id","polling_date","ip","user_agent","url_path","response_body","response_header" ],
properties: {
id: {
bsonType: "string",
},
session_id: {
bsonType: "string",
},
domain: {
bsonType: "string",
},
polling_date: {
bsonType: "string"
},
ip: {
bsonType: "string"
},
user_agent: {
bsonType: "string"
},
url_path: {
bsonType: "string"
},
response_body: {
bsonType: "string"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不应该是个string应该是个bson object

},
response_header: {
bsonType: "string"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不应该是个string应该是个bson object

},
response_code: {
bsonType: "string"
}
}
} }
} );

//index
db.kv.createIndex({"id": 1}, { unique: true } );
db.kv.createIndex({key: 1, label_id: 1,domain:1,project:1},{ unique: true });
db.label.createIndex({"id": 1}, { unique: true } );
db.label.createIndex({format: 1,domain:1,project:1},{ unique: true });
db.polling_detail.createIndex({"id": 1}, { unique: true } );
db.polling_detail.createIndex({session:1,domain:1}, { unique: true } );
db.view.createIndex({"id": 1}, { unique: true } );
db.view.createIndex({display:1,domain:1,project:1},{ unique: true });
//db config
Expand Down
24 changes: 24 additions & 0 deletions pkg/iputil/ip_util.go
View file
Open in desktop
@@ -0,0 +1,24 @@
package iputil

import (
"net"
"net/http"
"strings"
)

//ClientIP try to get ip from http header
func ClientIP(r *http.Request) string {
xForwardedFor := r.Header.Get("X-Forwarded-For")
ip := strings.TrimSpace(strings.Split(xForwardedFor, ",")[0])
if ip != "" {
return ip
}
ip = strings.TrimSpace(r.Header.Get("X-Real-Ip"))
if ip != "" {
return ip
}
if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil {
return ip
}
return ""
}
14 changes: 14 additions & 0 deletions pkg/model/kv.go
View file
Open in desktop
Expand Up @@ -54,3 +54,17 @@ type ViewResponse struct {
Total int `json:"total,omitempty"`
Data []*ViewDoc `json:"data,omitempty"`
}

//PollingDetail record operation history
type PollingDetail struct {
ID string `json:"id,omitempty" yaml:"id,omitempty"`
SessionID string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
PollingData map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"`
IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
UserAgent string `json:"user_agent,omitempty" yaml:"user_agent,omitempty"`
URLPath string `json:"url_path,omitempty" yaml:"url_path,omitempty"`
ResponseBody string `json:"response_body,omitempty" yaml:"response_body,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ResponseBody 用 struct

ResponseHeader string `json:"response_header,omitempty" yaml:"response_header,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ResponseHeader 用map[string][]string

ResponseCode int `json:"response_code,omitempty" yaml:"response_code,omitempty"`
}
24 changes: 20 additions & 4 deletions server/resource/v1/common.go
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@ import (
"context"
"encoding/json"
"errors"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/pubsub"
"github.com/apache/servicecomb-kie/server/service"
uuid "github.com/satori/go.uuid"
Expand Down Expand Up @@ -86,6 +87,15 @@ func ReadLabelCombinations(req *goRestful.Request) ([]map[string]string, error)
return labelCombinations, nil
}

//WriteErrResponseWithRecord write error message to client and record
func WriteErrResponseWithRecord(context *restful.Context, status int, msg, contentType string, record *model.PollingDetail) {
WriteErrResponse(context, status, msg, contentType)
_, recordErr := service.RecordService.RecordFailed(context.Ctx, record, status, msg)
if recordErr != nil {
openlogging.Error("record detail error error:" + recordErr.Error())
}
}

//WriteErrResponse write error message to client
func WriteErrResponse(context *restful.Context, status int, msg, contentType string) {
context.WriteHeader(status)
Expand Down Expand Up @@ -209,7 +219,7 @@ func checkStatus(status string) (string, error) {
}

func queryAndResponse(rctx *restful.Context,
domain interface{}, project string, key string, labels map[string]string, limit, offset int64, status string) {
domain interface{}, project string, key string, labels map[string]string, limit, offset int64, status string, record *model.PollingDetail) {
m := getMatchPattern(rctx)
opts := []service.FindOption{
service.WithKey(key),
Expand All @@ -226,20 +236,26 @@ func queryAndResponse(rctx *restful.Context,
kv, err := service.KVService.List(rctx.Ctx, domain.(string), project, opts...)
if err != nil {
if err == service.ErrKeyNotExists {
WriteErrResponse(rctx, http.StatusNotFound, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusNotFound, err.Error(), common.ContentTypeText, record)
return
}
WriteErrResponse(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText, record)
return
}
rev, err := service.RevisionService.GetRevision(rctx.Ctx, domain.(string))
if err != nil {
WriteErrResponse(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText, record)
return
}
rctx.ReadResponseWriter().Header().Set(common.HeaderRevision, strconv.FormatInt(rev, 10))
err = writeResponse(rctx, kv)
if err != nil {
openlogging.Error(err.Error())
}
respData, _ := json.Marshal(kv)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不需要序列化，直接存数据

respHeader, _ := json.Marshal(rctx.Resp.Header())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不要序列化直接存map

_, recordErr := service.RecordService.RecordSuccess(rctx.Ctx, record, http.StatusOK, string(respData), string(respHeader))
if recordErr != nil {
openlogging.Error("record detail error error:" + recordErr.Error())
}
}
57 changes: 45 additions & 12 deletions server/resource/v1/kv_resource.go
View file
Open in desktop
Expand Up @@ -21,12 +21,14 @@ package v1
import (
"fmt"
"github.com/apache/servicecomb-kie/pkg/common"
"github.com/apache/servicecomb-kie/pkg/iputil"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/pubsub"
"github.com/apache/servicecomb-kie/server/service"
goRestful "github.com/emicklei/go-restful"
"github.com/go-chassis/go-chassis/server/restful"
"github.com/go-mesh/openlogging"
uuid "github.com/satori/go.uuid"
"net/http"
)

Expand Down Expand Up @@ -108,13 +110,14 @@ func (r *KVResource) GetByKey(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
insID := rctx.ReadHeader("insId")
statusStr := rctx.ReadQueryParameter("status")
status, err := checkStatus(statusStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
returnData(rctx, domain, project, labels, limit, offset, status)
returnData(rctx, domain, project, labels, limit, offset, status, insID)
}

//List response kv list
Expand All @@ -138,21 +141,24 @@ func (r *KVResource) List(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
insID := rctx.ReadHeader("insId")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

改参数名

statusStr := rctx.ReadQueryParameter("status")
status, err := checkStatus(statusStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
returnData(rctx, domain, project, labels, limit, offset, status)
returnData(rctx, domain, project, labels, limit, offset, status, insID)
}

func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, limit, offset int64, status string) {
func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, limit, offset int64, status, insID string) {
revStr := rctx.ReadQueryParameter(common.QueryParamRev)
wait := rctx.ReadQueryParameter(common.QueryParamWait)
var recordErr error
recordData := GenPollingDetail(rctx, revStr, wait, domain.(string), project, labels, limit, offset, insID)
if revStr == "" {
if wait == "" {
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status, recordData)
return
}
changed, err := eventHappened(rctx, wait, &pubsub.Topic{
Expand All @@ -162,26 +168,27 @@ func returnData(rctx *restful.Context, domain interface{}, project string, label
DomainID: domain.(string),
})
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText, recordData)
return
}
if changed {
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status, recordData)
return
}
rctx.WriteHeader(http.StatusNotModified)
_, recordErr = service.RecordService.RecordSuccess(rctx.Ctx, recordData, http.StatusNotModified, "", "")
} else {
revised, err := isRevised(rctx.Ctx, revStr, domain.(string))
if err != nil {
if err == ErrInvalidRev {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText, recordData)
return
}
WriteErrResponse(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText, recordData)
return
}
if revised {
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status, recordData)
return
} else if wait != "" {
changed, err := eventHappened(rctx, wait, &pubsub.Topic{
Expand All @@ -191,21 +198,47 @@ func returnData(rctx *restful.Context, domain interface{}, project string, label
DomainID: domain.(string),
})
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
WriteErrResponseWithRecord(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText, recordData)
return
}
if changed {
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status)
queryAndResponse(rctx, domain, project, "", labels, limit, offset, status, recordData)
return
}
rctx.WriteHeader(http.StatusNotModified)
return
_, recordErr = service.RecordService.RecordSuccess(rctx.Ctx, recordData, http.StatusNotModified, "", "")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

太多地方需要考虑调用，不如用defer+一个func，虽然损失性能，但是优雅易维护

} else {
rctx.WriteHeader(http.StatusNotModified)
_, recordErr = service.RecordService.RecordSuccess(rctx.Ctx, recordData, http.StatusNotModified, "", "")
}
if recordErr != nil {
openlogging.Error("record detail error error:" + recordErr.Error())
}
}
}

//GenPollingDetail to generate data after get or list
func GenPollingDetail(context *restful.Context, revStr, wait, domain, project string, labels map[string]string, limit, offset int64, insID string) *model.PollingDetail {
data := &model.PollingDetail{}
data.ID = uuid.NewV4().String()
data.SessionID = insID
data.Domain = domain
data.IP = iputil.ClientIP(context.Req.Request)
dataMap := map[string]interface{}{
"revStr": revStr,
"wait": wait,
"domain": domain,
"project": project,
"labels": labels,
"limit": limit,
"offset": offset,
}
data.PollingData = dataMap
data.UserAgent = context.Req.HeaderParameter("User-Agent")
data.URLPath = context.ReadRequest().Method + " " + context.ReadRequest().URL.Path
return data
}

//Search search key only by label
func (r *KVResource) Search(context *restful.Context) {
var err error
Expand Down
2 changes: 2 additions & 0 deletions server/service/mongo/init.go
View file
Open in desktop
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/apache/servicecomb-kie/server/service/mongo/history"
"github.com/apache/servicecomb-kie/server/service/mongo/kv"
"github.com/apache/servicecomb-kie/server/service/mongo/label"
"github.com/apache/servicecomb-kie/server/service/mongo/record"
"github.com/apache/servicecomb-kie/server/service/mongo/session"
"github.com/go-mesh/openlogging"
)
Expand All @@ -32,6 +33,7 @@ func init() {
service.DBInit = session.Init
service.KVService = &kv.Service{}
service.HistoryService = &history.Service{}
service.RecordService = &record.Service{}
service.RevisionService = &counter.Service{}
service.LabelService = &label.Service{}
}
10 changes: 1 addition & 9 deletions server/service/mongo/kv/kv_service.go
View file
Open in desktop
Expand Up @@ -31,12 +31,6 @@ import (
"github.com/go-mesh/openlogging"
)

//const
const (
existKvLimit = 2
existKvOffset = 0
)

//Service operate data in mongodb
type Service struct {
timeout time.Duration
Expand Down Expand Up @@ -123,9 +117,7 @@ func (s *Service) Exist(ctx context.Context, domain, key string, project string,
kvs, err := s.FindKV(ctx, domain, project,
service.WithExactLabels(),
service.WithLabels(opts.Labels),
service.WithKey(key),
service.WithLimit(existKvLimit),
service.WithOffset(existKvOffset))
service.WithKey(key))
if err != nil {
openlogging.Error(err.Error())
return nil, err
Expand Down