Mongo (#5)

* mongodb as policy store * use mongodb cloud instance for testing * Update ci.yml * recolve review comments
teramoby · Jul 1, 2020 · b00ce42 · b00ce42
1 parent 63bc475
commit b00ce42
Show file tree

Hide file tree

Showing 403 changed files with 86,596 additions and 33 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -7,6 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     name: build-test
     steps:
+    # resolve https://github.com/actions/virtual-environments/issues/798
+    - name: bypass ubuntu DNS issue
+      run: |
+       sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
     - uses: actions/checkout@v2
       name: Checkout code
       with:

diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Gopkg.toml b/Gopkg.toml
@@ -90,3 +90,7 @@ ignored = ["github.com/teramoby/speedle-plus/samples*"]
 [[override]]
   name = "github.com/coreos/bbolt"
   revision = "48ea1b39c25fc1bab3506fbc712ecbaa842c4d2d"
+
+[[constraint]]
+  name = "go.mongodb.org/mongo-driver"
+  version = "1.3.4"
diff --git a/Makefile b/Makefile
@@ -45,31 +45,39 @@ speedleUnitTests:
 	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/eval 
 	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/store/file
 	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/store/etcd
+	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/store/mongodb
 	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/cmd/spctl/pdl
 	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/suid
 	go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/assertion
 	go clean -testcache
 	STORE_TYPE=etcd go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/eval
+	go clean -testcache
+	STORE_TYPE=mongodb go test ${TEST_OPTS} github.com/teramoby/speedle-plus/pkg/eval
 
 testSpeedleRest:
 	pkg/svcs/pmsrest/run_file_test.sh
 	pkg/svcs/pmsrest/run_etcd_test.sh
+	pkg/svcs/pmsrest/run_mongodb_test.sh
 
 testSpeedleGRpc:
 	pkg/svcs/pmsgrpc/run_file_test.sh
 	pkg/svcs/pmsgrpc/run_etcd_test.sh
+	pkg/svcs/pmsgrpc/run_mongodb_test.sh
 
 testSpeedleRestADSCheck:
 	pkg/svcs/adsrest/run_file_test.sh
 	pkg/svcs/adsrest/run_etcd_test.sh
+	pkg/svcs/adsrest/run_mongodb_test.sh
 
 testSpeedleGRpcADSCheck:
 	pkg/svcs/adsgrpc/run_file_test.sh
 	pkg/svcs/adsgrpc/run_etcd_test.sh
+	pkg/svcs/adsgrpc/run_mongodb_test.sh
 
 testSpctl:
 	cmd/spctl/command/run_file_test.sh
 	cmd/spctl/command/run_etcd_test.sh
+	cmd/spctl/command/run_mongodb_test.sh
 
 testSpeedleTls:
 	pkg/svcs/pmsrest/tls_test.sh

diff --git a/api/pms/types.go b/api/pms/types.go
@@ -10,24 +10,24 @@ type Permission struct {
 }
 
 type Function struct {
-	Name           string            `json:"name"`
-	Description    string            `json:"description,omitempty"`
-	FuncURL        string            `json:"funcURL"`                  //used by speedle/sphinx ADS
-	LocalFuncURL   string            `json:"localFuncURL,omitempty"`   //used by sphinx runtime proxy to get better performance
-	CA             string            `json:"ca,omitempty"`             //security related configurations
-	ResultCachable bool              `json:"resultCachable,omitempty"` //false by default
-	ResultTTL      int64             `json:"resultTTL,omitempty"`      // TTL of function result in second
-	Metadata       map[string]string `json:"metadata,omitempty"`
+	Name           string            `json:"name" bson:"_id"`
+	Description    string            `json:"description,omitempty" bson:"description,omitempty"`
+	FuncURL        string            `json:"funcURL" bson:"funcurl"`                                   //used by speedle/sphinx ADS
+	LocalFuncURL   string            `json:"localFuncURL,omitempty"  bson:"localfuncurl"`              //used by sphinx runtime proxy to get better performance
+	CA             string            `json:"ca,omitempty" bson:"ca,omitempty"`                         //security related configurations
+	ResultCachable bool              `json:"resultCachable,omitempty" bson:"resultcachable,omitempty"` //false by default
+	ResultTTL      int64             `json:"resultTTL,omitempty" bson:"resultttl,omitempty"`           // TTL of function result in second
+	Metadata       map[string]string `json:"metadata,omitempty" bson:"metadata,omitempty"`
 }
 
 type Policy struct {
-	ID          string            `json:"id"`
-	Name        string            `json:"name"`
-	Effect      string            `json:"effect,omitempty"`
-	Permissions []*Permission     `json:"permissions,omitempty"`
-	Principals  [][]string        `json:"principals,omitempty"`
-	Condition   string            `json:"condition,omitempty"`
-	Metadata    map[string]string `json:"metadata,omitempty"`
+	ID          string            `json:"id" bson:"_id"`
+	Name        string            `json:"name" bson:"name,omitempty"`
+	Effect      string            `json:"effect,omitempty" bson:"effect,omitempty"`
+	Permissions []*Permission     `json:"permissions,omitempty" bson:"permissions,omitempty"`
+	Principals  [][]string        `json:"principals,omitempty" bson:"principals,omitempty"`
+	Condition   string            `json:"condition,omitempty" bson:"condition,omitempty"`
+	Metadata    map[string]string `json:"metadata,omitempty" bson:"metadata,omitempty"`
 }
 
 const (
@@ -41,23 +41,23 @@ const (
 )
 
 type RolePolicy struct {
-	ID                  string            `json:"id"`
-	Name                string            `json:"name"`
-	Effect              string            `json:"effect,omitempty"`
-	Roles               []string          `json:"roles,omitempty"`
-	Principals          []string          `json:"principals,omitempty"`
-	Resources           []string          `json:"resources,omitempty"`
-	ResourceExpressions []string          `json:"resourceExpressions,omitempty"`
-	Condition           string            `json:"condition,omitempty"`
-	Metadata            map[string]string `json:"metadata,omitempty"`
+	ID                  string            `json:"id" bson:"_id"`
+	Name                string            `json:"name" bson:"name,omitempty"`
+	Effect              string            `json:"effect,omitempty" bson:"effect,omitempty"`
+	Roles               []string          `json:"roles,omitempty" bson:"roles,omitempty"`
+	Principals          []string          `json:"principals,omitempty" bson:"principals,omitempty"`
+	Resources           []string          `json:"resources,omitempty" bson:"resources,omitempty"`
+	ResourceExpressions []string          `json:"resourceExpressions,omitempty" bson:"resourceexpressions,omitempty"`
+	Condition           string            `json:"condition,omitempty" bson:"condition,omitempty"`
+	Metadata            map[string]string `json:"metadata,omitempty" bson:"metadata,omitempty"`
 }
 
 type Service struct {
-	Name         string            `json:"name" binding:"required"`
-	Type         string            `json:"type,omitempty"`
-	Policies     []*Policy         `json:"policies,omitempty"`
-	RolePolicies []*RolePolicy     `json:"rolePolicies,omitempty"`
-	Metadata     map[string]string `json:"metadata,omitempty"`
+	Name         string            `json:"name" binding:"required"  bson:"_id"`
+	Type         string            `json:"type,omitempty" bson:"type,omitempty"`
+	Policies     []*Policy         `json:"policies,omitempty" bson:"policies,omitempty"`
+	RolePolicies []*RolePolicy     `json:"rolePolicies,omitempty" bson:"rolepolicies,omitempty"`
+	Metadata     map[string]string `json:"metadata,omitempty" bson:"metadata,omitempty"`
 }
 
 const GlobalService = "global"

diff --git a/cmd/spctl/command/run_mongodb_test.sh b/cmd/spctl/command/run_mongodb_test.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+source ${GOPATH}/src/github.com/teramoby/speedle-plus/setTestEnv.sh
+go clean -testcache
+
+#Reconfig spctl
+${GOPATH}/bin/spctl config pms-endpoint http://localhost:6733/policy-mgmt/v1/
+
+startPMS mongodb --config-file ${GOPATH}/src/github.com/teramoby/speedle-plus/pkg/svcs/pmsrest/config_mongodb.json
+sleep 5
+${GOPATH}/bin/spctl delete service --all
+go test ${TEST_OPTS} github.com/teramoby/speedle-plus/cmd/spctl/command -run=TestMats
+
diff --git a/cmd/speedle-ads/stores.go b/cmd/speedle-ads/stores.go
@@ -9,4 +9,5 @@ package main
 import (
 	_ "github.com/teramoby/speedle-plus/pkg/store/etcd"
 	_ "github.com/teramoby/speedle-plus/pkg/store/file"
+	_ "github.com/teramoby/speedle-plus/pkg/store/mongodb"
 )
diff --git a/cmd/speedle-pms/stores.go b/cmd/speedle-pms/stores.go
@@ -9,4 +9,5 @@ package main
 import (
 	_ "github.com/teramoby/speedle-plus/pkg/store/etcd"
 	_ "github.com/teramoby/speedle-plus/pkg/store/file"
+	_ "github.com/teramoby/speedle-plus/pkg/store/mongodb"
 )