SCB-1613 a new API for user want to list key values by labels (#53)

.travis.yml

@@ -58,4 +58,4 @@ jobs:
         - cd $HOME/gopath/src/github.com/apache/servicecomb-kie
         - go get github.com/mattn/goveralls
         - go get golang.org/x/tools/cmd/cover
-        - bash scripts/travis/unit_test.sh
+        - bash scripts/travis/unit_test.sh && $HOME/gopath/bin/goveralls -coverprofile=coverage.txt -service=travis-ci

README.md

@@ -1,5 +1,8 @@
-# Apache-ServiceComb-Kie [![Build Status](https://travis-ci.org/apache/servicecomb-kie.svg?branch=master)](https://travis-ci.org/apache/servicecomb-kie?branch=master) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
+# Apache-ServiceComb-Kie 
 
+[![Build Status](https://travis-ci.org/apache/servicecomb-kie.svg?branch=master)](https://travis-ci.org/apache/servicecomb-kie?branch=master) 
+[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
+[![Coverage Status](https://coveralls.io/repos/github/apache/servicecomb-kie/badge.svg?branch=master)](https://coveralls.io/github/apache/servicecomb-kie?branch=master)
 A service for configuration management in distributed system.
 
 ## Conceptions

client/adaptor/kie_client.go

@@ -60,15 +60,15 @@ func (c *Client) PullConfigs(labels ...map[string]string) (map[string]interface{
 	var err error
 	var configurationsValue []*model.KVResponse
 	if len(labels) != 0 {
-		configurationsValue, err = c.KieClient.SearchByLabels(context.TODO(), client.WithGetProject("default"), client.WithLabels(labels...))
+		configurationsValue, err = c.KieClient.Search(context.TODO(), client.WithGetProject("default"), client.WithLabels(labels...))
 	} else {
-		configurationsValue, err = c.KieClient.SearchByLabels(context.TODO(), client.WithGetProject("default"), client.WithLabels(c.opts.Labels))
+		configurationsValue, err = c.KieClient.Search(context.TODO(), client.WithGetProject("default"), client.WithLabels(c.opts.Labels))
 	}
 	if err != nil {
 		openlogging.GetLogger().Errorf("Error in Querying the Response from Kie %s %#v", err.Error(), labels)
 		return nil, err
 	}
-	openlogging.GetLogger().Debugf("KieClient SearchByLabels. %#v", labels)
+	openlogging.GetLogger().Debugf("KieClient Search. %#v", labels)
 	//Parse config result.
 	for _, docRes := range configurationsValue {
 		for _, docInfo := range docRes.Data {

client/adaptor/kie_client_test.go

@@ -138,7 +138,7 @@ func TestKieClient_DeleteConfigs(t *testing.T) {
 		config.LabelService: "test",
 	})
 	//assert.Equal(t, resp.StatusCode, 404)
-	assert.Equal(t, err.Error(), "delete 1 failed,http status [200 OK], body [[{\"label\":null,\"data\":null}]]")
+	assert.Equal(t, "delete 1 failed,http status [200 OK], body [[{\"data\":null}]]", err.Error())
 	// Shutdown the helper server gracefully
 	if err := helper.Shutdown(context.Background()); err != nil {
 		panic(err)

client/client.go

@@ -158,8 +158,8 @@ func (c *Client) Get(ctx context.Context, key string, opts ...GetOption) ([]*mod
 	return kvs, nil
 }
 
-//SearchByLabels get value by lables
-func (c *Client) SearchByLabels(ctx context.Context, opts ...GetOption) ([]*model.KVResponse, error) {
+//Search get value by labels
+func (c *Client) Search(ctx context.Context, opts ...GetOption) ([]*model.KVResponse, error) {
 	options := GetOptions{}
 	for _, o := range opts {
 		o(&options)

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/go-chassis/go-archaius v0.24.0
 	github.com/go-chassis/go-chassis v1.7.6
 	github.com/go-chassis/go-chassis-config v0.15.0
-	github.com/go-chassis/go-restful-swagger20 v1.0.2-0.20191118130439-7eec0f2639f6
+	github.com/go-chassis/go-restful-swagger20 v1.0.2-0.20191118130439-7eec0f2639f6 // indirect
 	github.com/go-chassis/paas-lager v1.0.2-0.20190328010332-cf506050ddb2
 	github.com/go-mesh/openlogging v1.0.1
 	github.com/golang/snappy v0.0.1 // indirect

pkg/model/kv.go

@@ -28,7 +28,10 @@ type KVRequest struct {
 
 //KVResponse represents the key value list
 type KVResponse struct {
-	LabelDoc *LabelDocResponse `json:"label"`
+	LabelDoc *LabelDocResponse `json:"label,omitempty"`
+	PageNum  int               `json:"num,omitempty"`
+	Size     int               `json:"size,omitempty"`
+	Total    int               `json:"total,omitempty"`
 	Data     []*KVDoc          `json:"data"`
 }
 

server/resource/v1/doc_struct.go

@@ -75,16 +75,19 @@ var (
 		DataType:  "string",
 		Name:      "key",
 		ParamType: goRestful.PathParameterKind,
+		Required:  true,
 	}
 	DocPathProject = &restful.Parameters{
 		DataType:  "string",
 		Name:      "project",
 		ParamType: goRestful.PathParameterKind,
+		Required:  true,
 	}
 	DocPathLabelID = &restful.Parameters{
 		DataType:  "string",
 		Name:      "label_id",
 		ParamType: goRestful.PathParameterKind,
+		Required:  true,
 	}
 )
 

server/resource/v1/kv_resource.go

@@ -20,6 +20,7 @@ package v1
 
 import (
 	"net/http"
+	"strconv"
 
 	"github.com/apache/servicecomb-kie/pkg/common"
 	"github.com/apache/servicecomb-kie/pkg/model"
@@ -39,10 +40,6 @@ func (r *KVResource) Put(context *restful.Context) {
 	var err error
 	key := context.ReadPathParameter("key")
 	project := context.ReadPathParameter("project")
-	if project == "" {
-		WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
-		return
-	}
 	kv := new(model.KVDoc)
 	if err = readRequest(context, kv); err != nil {
 		WriteErrResponse(context, http.StatusBadRequest, err.Error(), common.ContentTypeText)
@@ -78,10 +75,6 @@ func (r *KVResource) GetByKey(context *restful.Context) {
 		return
 	}
 	project := context.ReadPathParameter("project")
-	if project == "" {
-		WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
-		return
-	}
 	values := context.ReadRequest().URL.Query()
 	labels := make(map[string]string, len(values))
 	for k, v := range values {
@@ -118,25 +111,81 @@ func (r *KVResource) GetByKey(context *restful.Context) {
 
 }
 
-//SearchByLabels search key only by label
-func (r *KVResource) SearchByLabels(context *restful.Context) {
+//List TODO pagination
+func (r *KVResource) List(rctx *restful.Context) {
+	project := rctx.ReadPathParameter("project")
+	domain := ReadDomain(rctx)
+	if domain == nil {
+		WriteErrResponse(rctx, http.StatusInternalServerError, MsgDomainMustNotBeEmpty, common.ContentTypeText)
+		return
+	}
+	var limit int64 = 20
+	var offset int64 = 0
+	labels := make(map[string]string, 0)
+	var err error
+	for k, v := range rctx.ReadRequest().URL.Query() {
+		if k == "limit" {
+			limit, err = strconv.ParseInt(v[0], 10, 64)
+			if err != nil {
+				WriteErrResponse(rctx, http.StatusBadRequest, "invalid limit number", common.ContentTypeText)
+			}
+			if limit < 1 || limit > 50 {
+				WriteErrResponse(rctx, http.StatusBadRequest, "invalid limit number", common.ContentTypeText)
+			}
+			continue
+		}
+		if k == "offset" {
+			offset, err = strconv.ParseInt(v[0], 10, 64)
+			if err != nil {
+				WriteErrResponse(rctx, http.StatusBadRequest, "invalid offset number", common.ContentTypeText)
+			}
+			if offset < 1 {
+				WriteErrResponse(rctx, http.StatusBadRequest, "invalid offset number", common.ContentTypeText)
+			}
+			continue
+		}
+		labels[k] = v[0]
+	}
+	result, err := service.KVService.List(rctx.Ctx, domain.(string), project, "", labels, int(limit), int(offset))
+	if err != nil {
+		openlogging.Error("can not find by labels", openlogging.WithTags(openlogging.Tags{
+			"err": err.Error(),
+		}))
+		WriteErrResponse(rctx, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
+		return
+	}
+	err = writeResponse(rctx, result)
+	if err != nil {
+		openlogging.Error(err.Error())
+	}
+}
+
+//Search search key only by label
+func (r *KVResource) Search(context *restful.Context) {
 	var err error
 	labelCombinations, err := ReadLabelCombinations(context.ReadRestfulRequest())
 	if err != nil {
 		WriteErrResponse(context, http.StatusBadRequest, err.Error(), common.ContentTypeText)
 		return
 	}
 	project := context.ReadPathParameter("project")
-	if project == "" {
-		WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
-		return
-	}
 	domain := ReadDomain(context)
 	if domain == nil {
 		WriteErrResponse(context, http.StatusInternalServerError, MsgDomainMustNotBeEmpty, common.ContentTypeText)
 		return
 	}
 	var kvs []*model.KVResponse
+	if labelCombinations == nil {
+		result, err := service.KVService.FindKV(context.Ctx, domain.(string), project)
+		if err != nil {
+			openlogging.Error("can not find by labels", openlogging.WithTags(openlogging.Tags{
+				"err": err.Error(),
+			}))
+			WriteErrResponse(context, http.StatusInternalServerError, err.Error(), common.ContentTypeText)
+			return
+		}
+		kvs = append(kvs, result...)
+	}
 	for _, labels := range labelCombinations {
 		openlogging.Debug("find by combination", openlogging.WithTags(openlogging.Tags{
 			"q": labels,
@@ -171,10 +220,6 @@ func (r *KVResource) SearchByLabels(context *restful.Context) {
 //Delete deletes key by ids
 func (r *KVResource) Delete(context *restful.Context) {
 	project := context.ReadPathParameter("project")
-	if project == "" {
-		WriteErrResponse(context, http.StatusForbidden, "project must not be empty", common.ContentTypeText)
-		return
-	}
 	domain := ReadDomain(context)
 	if domain == nil {
 		WriteErrResponse(context, http.StatusInternalServerError, MsgDomainMustNotBeEmpty, common.ContentTypeText)
@@ -240,7 +285,7 @@ func (r *KVResource) URLPatterns() []restful.Route {
 		}, {
 			Method:       http.MethodGet,
 			Path:         "/v1/{project}/kie/kv",
-			ResourceFunc: r.SearchByLabels,
+			ResourceFunc: r.Search,
 			FuncDesc:     "search key values by labels combination",
 			Parameters: []*restful.Parameters{
 				DocPathProject, DocQueryCombination,
@@ -254,11 +299,27 @@ func (r *KVResource) URLPatterns() []restful.Route {
 			},
 			Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
 			Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+		}, {
+			Method:       http.MethodGet,
+			Path:         "/v1/{project}/kie/kv:list",
+			ResourceFunc: r.List,
+			FuncDesc:     "list key values by labels and key",
+			Parameters: []*restful.Parameters{
+				DocPathProject, DocQueryLabelParameters,
+			},
+			Returns: []*restful.Returns{
+				{
+					Code:  http.StatusOK,
+					Model: model.KVResponse{},
+				},
+			},
+			Consumes: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
+			Produces: []string{goRestful.MIME_JSON, common.ContentTypeYaml},
 		}, {
 			Method:       http.MethodDelete,
 			Path:         "/v1/{project}/kie/kv",
 			ResourceFunc: r.Delete,
-			FuncDesc:     "delete key by kvID and labelID. if you want better performance, you need to give labelID",
+			FuncDesc:     "delete key by kvID and labelID. Want better performance, give labelID",
 			Parameters: []*restful.Parameters{
 				DocPathProject,
 				DocQueryKVIDParameters,

server/resource/v1/kv_resource_test.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"github.com/apache/servicecomb-kie/server/service"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httptest"
 
@@ -43,13 +44,12 @@ var _ = Describe("v1 kv resource", func() {
 	config.Configurations = &config.Config{
 		DB: config.DB{},
 	}
-
+	config.Configurations.DB.URI = "mongodb://kie:123@127.0.0.1:27017"
+	err := service.DBInit()
+	if err != nil {
+		panic(err)
+	}
 	Describe("put kv", func() {
-		config.Configurations.DB.URI = "mongodb://kie:123@127.0.0.1:27017"
-		err := service.DBInit()
-		It("should not return err", func() {
-			Expect(err).Should(BeNil())
-		})
 		Context("valid param", func() {
 			kv := &model.KVDoc{
 				Key:    "timeout",
@@ -86,4 +86,34 @@ var _ = Describe("v1 kv resource", func() {
 			})
 		})
 	})
+	Describe("list kv", func() {
+		Context("with no label", func() {
+			r, _ := http.NewRequest("GET", "/v1/test/kie/kv:list", nil)
+			noopH := &noop.NoopAuthHandler{}
+			chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
+			r.Header.Set("Content-Type", "application/json")
+			kvr := &v1.KVResource{}
+			c, err := restfultest.New(kvr, chain)
+			It("should not return error", func() {
+				Expect(err).Should(BeNil())
+			})
+			resp := httptest.NewRecorder()
+			c.ServeHTTP(resp, r)
+
+			body, err := ioutil.ReadAll(resp.Body)
+			It("should not return err", func() {
+				Expect(err).Should(BeNil())
+			})
+			log.Println(string(body))
+			result := &model.KVResponse{}
+			err = json.Unmarshal(body, result)
+			It("should not return err", func() {
+				Expect(err).Should(BeNil())
+			})
+
+			It("should longer than 1", func() {
+				Expect(len(result.Data)).NotTo(Equal(0))
+			})
+		})
+	})
 })

server/service/mongo/kv/kv_service.go

@@ -192,6 +192,30 @@ func (s *Service) Delete(kvID string, labelID string, domain string, project str
 	return nil
 }
 
+//List get kv list by key and criteria
+func (s *Service) List(ctx context.Context, domain, project, key string, labels map[string]string, limit, offset int) (*model.KVResponse, error) {
+	opts := service.NewDefaultFindOpts()
+	opts.Labels = labels
+	opts.Key = key
+	cur, err := findKV(ctx, domain, project, opts)
+	if err != nil {
+		return nil, err
+	}
+	defer cur.Close(ctx)
+	result := &model.KVResponse{}
+	for cur.Next(ctx) {
+		curKV := &model.KVDoc{}
+
+		if err := cur.Decode(curKV); err != nil {
+			openlogging.Error("decode to KVs error: " + err.Error())
+			return nil, err
+		}
+		clearPart(curKV)
+		result.Data = append(result.Data, curKV)
+	}
+	return result, nil
+}
+
 //FindKV get kvs by key, labels
 //because labels has a a lot of combination,
 //you can use WithDepth(0) to return only one kv which's labels exactly match the criteria
@@ -251,7 +275,7 @@ func (s *Service) FindKV(ctx context.Context, domain string, project string, opt
 		for _, labelGroup = range kvResp {
 			if reflect.DeepEqual(labelGroup.LabelDoc.Labels, curKV.Labels) {
 				groupExist = true
-				clearKV(curKV)
+				clearAll(curKV)
 				labelGroup.Data = append(labelGroup.Data, curKV)
 				break
 			}
@@ -265,7 +289,7 @@ func (s *Service) FindKV(ctx context.Context, domain string, project string, opt
 				},
 				Data: []*model.KVDoc{curKV},
 			}
-			clearKV(curKV)
+			clearAll(curKV)
 			openlogging.Debug("add new label group")
 			kvResp = append(kvResp, labelGroup)
 		}