Skip to content

Commit

Permalink
SCB-1718 use global revision, distinguish create and update revision (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
tianxiaoliang committed Jan 13, 2020
1 parent 463852e commit 4ae2770
Show file tree
Hide file tree
Showing 23 changed files with 265 additions and 95 deletions.
1 change: 0 additions & 1 deletion build/docker/server/Dockerfile
Expand Up @@ -18,7 +18,6 @@ FROM ubuntu:19.04
RUN apt-get update && apt-get install -y net-tools
RUN mkdir -p /etc/servicecomb-kie/
RUN mkdir -p /opt/servicecomb-kie/conf
RUN export GO_CHASSIS=/opt/servicecomb-kie

ADD ./kie /opt/servicecomb-kie/
ADD ./start.sh /opt/servicecomb-kie/
Expand Down
2 changes: 1 addition & 1 deletion client/client_test.go
Expand Up @@ -110,7 +110,7 @@ var _ = Describe("Client", func() {
kvBody := model.KVRequest{}
kvBody.Key = "time"
kvBody.Value = "100s"
kvBody.ValueType = "string"
kvBody.ValueType = "text"
kvBody.Labels = make(map[string]string)
kvBody.Labels["env"] = "test"
kv, err := client2.Put(context.TODO(), kvBody, WithProject("test"))
Expand Down
24 changes: 23 additions & 1 deletion deployments/docker/db.js
Expand Up @@ -26,4 +26,26 @@ db.createUser(
}
]
}
);
);
db.createCollection("counter");
db.counter.insertOne( { name:"revision_counter",count: 1 } );
db.createCollection( "kv", {
validator: { $jsonSchema: {
bsonType: "object",
required: [ "key","domain","project","id","value","create_revision","update_revision","value_type","label_id" ],
properties: {
key: {
bsonType: "string",
description: "must be a string and is required"
},
value_type: {
enum: [ "text", "yaml", "json", "properties", "ini" ]
},
labels: {
bsonType: "object"
}
}
} }
} );
db.kv.createIndex({"id": 1}, { unique: true } );
db.kv.createIndex({key: 1, label_id: 1},{ unique: true });
2 changes: 1 addition & 1 deletion deployments/docker/docker-compose.yaml
Expand Up @@ -17,7 +17,7 @@
version: '3.1'
services:
mongo:
image: mongo:3.4
image: mongo:4.0
restart: always
ports:
- 27017:27017
Expand Down
2 changes: 2 additions & 0 deletions examples/dev/clean.sh
@@ -0,0 +1,2 @@
sudo docker rm $(sudo docker ps -a -q)
sudo docker volume prune
2 changes: 0 additions & 2 deletions examples/dev/conf/chassis.yaml
Expand Up @@ -7,8 +7,6 @@ cse:
protocols:
rest:
listenAddress: 127.0.0.1:30110
rest-consul: #consul compatible API
listenAddress: 127.0.0.1:8500
handler:
chain:
Provider:
Expand Down
50 changes: 50 additions & 0 deletions examples/dev/db.js
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

db.createUser(
{
user: "kie",
pwd: "123",
roles:[
{
role: "readWrite",
db: "kie"
}
]
}
);
db.createCollection("counter");
db.counter.insertOne( { name:"revision_counter",count: 1 } );
db.createCollection( "kv", {
validator: { $jsonSchema: {
bsonType: "object",
required: [ "key","domain","project","id","value","create_revision","update_revision","value_type","label_id" ],
properties: {
key: {
bsonType: "string",
},
value_type: {
enum: [ "text","string","yaml", "json", "properties", "ini" ]
},
labels: {
bsonType: "object"
}
}
} }
} );
db.kv.createIndex({"id": 1}, { unique: true } );
db.kv.createIndex({key: 1, label_id: 1},{ unique: true });
4 changes: 3 additions & 1 deletion examples/dev/docker-compose.yaml
Expand Up @@ -24,7 +24,9 @@ services:
environment:
MONGO_INITDB_ROOT_USERNAME: kie
MONGO_INITDB_ROOT_PASSWORD: 123

MONGO_INITDB_DATABASE: kie
volumes:
- ./db.js:/docker-entrypoint-initdb.d/db.js:ro
mongo-express:
image: mongo-express
restart: always
Expand Down
3 changes: 2 additions & 1 deletion go.mod
@@ -1,6 +1,7 @@
module github.com/apache/servicecomb-kie

require (
github.com/DataDog/zstd v1.4.4 // indirect
github.com/emicklei/go-restful v2.11.1+incompatible
github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4
github.com/go-chassis/go-archaius v1.0.0
Expand All @@ -17,7 +18,7 @@ require (
github.com/urfave/cli v1.20.0
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v1.0.0 // indirect
go.mongodb.org/mongo-driver v1.0.3
go.mongodb.org/mongo-driver v1.2.1
gopkg.in/yaml.v2 v2.2.4
)

Expand Down
7 changes: 7 additions & 0 deletions go.sum
Expand Up @@ -9,6 +9,8 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
Expand Down Expand Up @@ -125,6 +127,8 @@ github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/mdns v1.0.1 h1:XFSOubp8KWB+Jd2PDyaX5xUd5bhSP/+pTDZVDMzZJM8=
github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc=
Expand Down Expand Up @@ -182,6 +186,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down Expand Up @@ -227,6 +232,8 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.2.1 h1:ANAlYXXM5XmOdW/Nc38jOr+wS5nlk7YihT24U1imiWM=
go.mongodb.org/mongo-driver v1.2.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand Down
4 changes: 2 additions & 2 deletions pkg/common/common.go
Expand Up @@ -22,15 +22,15 @@ const (
QueryParamQ = "q"
QueryByLabelsCon = "&"
QueryParamWait = "wait"
QueryParamRev = "revision"
QueryParamMatch = "match"
QueryParamKeyID = "kv_id"
)

//http headers
const (
HeaderMatch = "X-Match"
HeaderDepth = "X-Depth"
HeaderTenant = "X-Domain"
HeaderRevision = "X-Kie-Revision"
HeaderContentType = "Content-Type"
HeaderAccept = "Accept"
)
Expand Down
22 changes: 12 additions & 10 deletions pkg/model/mongodb_doc.go → pkg/model/db_schema.go
Expand Up @@ -27,15 +27,17 @@ type LabelDoc struct {

//KVDoc is database struct to store kv
type KVDoc struct {
ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"`
LabelID string `json:"label_id,omitempty" bson:"label_id,omitempty" yaml:"label_id,omitempty"`
Key string `json:"key" yaml:"key"`
Value string `json:"value,omitempty" yaml:"value,omitempty"`
ValueType string `json:"value_type,omitempty" bson:"value_type,omitempty" yaml:"value_type,omitempty"` //ini,json,text,yaml,properties
Checker string `json:"check,omitempty" yaml:"check,omitempty"` //python script
ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"`
LabelID string `json:"label_id,omitempty" bson:"label_id,omitempty" yaml:"label_id,omitempty"`
Key string `json:"key" yaml:"key"`
Value string `json:"value,omitempty" yaml:"value,omitempty"`
ValueType string `json:"value_type,omitempty" bson:"value_type,omitempty" yaml:"value_type,omitempty"` //ini,json,text,yaml,properties
Checker string `json:"check,omitempty" yaml:"check,omitempty"` //python script
CreateRevision int64 `json:"create_revision,omitempty" bson:"create_revision," yaml:"create_revision,omitempty"`
UpdateRevision int64 `json:"update_revision,omitempty" bson:"update_revision," yaml:"update_revision,omitempty"`
Project string `json:"project,omitempty" yaml:"project,omitempty"`

Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` //redundant
Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` //redundant

Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` //redundant
Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` //redundant
Revision int `json:"revision,omitempty" bson:"revision," yaml:"revision,omitempty"`
Project string `json:"project,omitempty" yaml:"project,omitempty"`
}
12 changes: 7 additions & 5 deletions server/resource/v1/common.go
Expand Up @@ -103,11 +103,6 @@ func WriteErrResponse(context *restful.Context, status int, msg, contentType str

}

//ErrLog record error
func ErrLog(action string, kv *model.KVDoc, err error) {
openlogging.Error(fmt.Sprintf("[%s] [%v] err:%s", action, kv, err.Error()))
}

//InfoLog record info
func InfoLog(action string, kv *model.KVDoc) {
openlogging.Info(
Expand Down Expand Up @@ -157,6 +152,13 @@ func getWaitDuration(rctx *restful.Context) string {
}
return wait
}
func getRevision(rctx *restful.Context) string {
rev := rctx.ReadQueryParameter(common.QueryParamRev)
if rev == "" {
rev = "0"
}
return rev
}
func getMatchPattern(rctx *restful.Context) string {
m := rctx.ReadQueryParameter(common.QueryParamMatch)

Expand Down
6 changes: 6 additions & 0 deletions server/resource/v1/doc_struct.go
Expand Up @@ -32,6 +32,12 @@ var (
ParamType: goRestful.HeaderParameterKind,
Desc: "integer, default is 1, if you set match policy, you can set,depth to decide label number",
}
DocHeaderRevision = &restful.Parameters{
DataType: "string",
Name: common.HeaderRevision,
ParamType: goRestful.HeaderParameterKind,
Desc: "integer, current revision of kie",
}
)

//swagger doc query params
Expand Down
3 changes: 2 additions & 1 deletion server/resource/v1/kv_resource.go
Expand Up @@ -19,6 +19,7 @@
package v1

import (
"fmt"
"github.com/apache/servicecomb-kie/server/pubsub"
"net/http"
"time"
Expand Down Expand Up @@ -54,7 +55,7 @@ func (r *KVResource) Put(context *restful.Context) {
kv.Project = project
kv, err = service.KVService.CreateOrUpdate(context.Ctx, kv)
if err != nil {
ErrLog("put", kv, err)
openlogging.Error(fmt.Sprintf("put [%v] err:%s", kv, err.Error()))
WriteErrResponse(context, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
return
}
Expand Down
77 changes: 77 additions & 0 deletions server/service/mongo/counter/revision.go
@@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package counter

import (
"context"
"errors"
"github.com/apache/servicecomb-kie/server/service/mongo/session"
"github.com/go-mesh/openlogging"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)

const revision = "revision_counter"

//Service is the implementation
type Service struct {
}

//GetRevision return current revision number
func (s *Service) GetRevision(ctx context.Context) (int64, error) {
collection := session.GetDB().Collection(session.CollectionCounter)
filter := bson.M{"name": revision}
cur, err := collection.Find(ctx, filter)
if err != nil {
if err.Error() == context.DeadlineExceeded.Error() {
openlogging.Error("operation timeout")
return 0, errors.New("operation timeout")
}
return 0, err
}
defer cur.Close(ctx)
c := &Counter{}
for cur.Next(ctx) {
if err := cur.Decode(c); err != nil {
openlogging.Error("decode error: " + err.Error())
return 0, err
}
}
return c.Count, nil
}

//ApplyRevision increase revision number and return modified value
func ApplyRevision(ctx context.Context) (int64, error) {
collection := session.GetDB().Collection(session.CollectionCounter)
filter := bson.M{"name": revision}
sr := collection.FindOneAndUpdate(ctx, filter,
bson.D{
{"$inc", bson.D{
{"count", 1},
}}}, options.FindOneAndUpdate().SetReturnDocument(options.After))
if sr.Err() != nil {
return 0, sr.Err()
}
c := &Counter{}
err := sr.Decode(c)
if err != nil {
openlogging.Error("decode error: " + err.Error())
return 0, err
}
return c.Count, nil
}

0 comments on commit 4ae2770

Please sign in to comment.