MB-53439 MB-50997 scope UDF definitions stored in _system scope, open…

… system:functions

Change-Id: Idaad439a674c92744a065dfd6103b95228d5aed2
Reviewed-on: https://review.couchbase.org/c/query/+/179023
Reviewed-by: Sitaram Vemulapalli <sitaram.vemulapalli@couchbase.com>
Tested-by: Marco Greco <marco.greco@couchbase.com>

algebra/path.go

@@ -258,13 +258,21 @@ func (path *Path) ProtectedString() string {
 }
 
 func (path *Path) string(forceBackticks bool, isContext bool) string {
+	return pathFromParts(forceBackticks, isContext, path.elements...)
+}
+
+func PathFromParts(elements ...string) string {
+	return pathFromParts(false, false, elements...)
+}
+
+func pathFromParts(forceBackticks bool, isContext bool, elements ...string) string {
 	acc := ""
-	lastIndex := len(path.elements) - 1
+	lastIndex := len(elements) - 1
 	if isContext {
 		lastIndex--
 	}
 	for i := 0; i <= lastIndex; i++ {
-		s := path.elements[i]
+		s := elements[i]
 
 		// The first element, i.e. the namespace, may be an empty string.
 		// That means we can omit it, and the separator after it.
@@ -388,6 +396,14 @@ func ValidateQueryContext(queryContext string) errors.Error {
 	return nil
 }
 
+func PartsFromPath(path string) int {
+	res, parts := validatePathOrContext(path)
+	if res != "" {
+		return -1
+	}
+	return parts
+}
+
 func validatePathOrContext(queryContext string) (string, int) {
 	hasNamespace := false
 	parts := 0

datastore/couchbase/collections.go

@@ -24,7 +24,7 @@ import (
 	"github.com/couchbase/query/datastore"
 	"github.com/couchbase/query/errors"
 	"github.com/couchbase/query/functions"
-	functionsStorage "github.com/couchbase/query/functions/metakv"
+	functionsStorage "github.com/couchbase/query/functions/storage"
 	"github.com/couchbase/query/logging"
 	"github.com/couchbase/query/value"
 )

datastore/system/system.go

@@ -94,6 +94,7 @@ func (s *store) PrivilegesFromPath(fullname string, keyspace string, privilege a
 		case KEYSPACE_NAME_INDEXES:
 		case KEYSPACE_NAME_ALL_INDEXES:
 		case KEYSPACE_NAME_MY_USER_INFO:
+		case KEYSPACE_NAME_FUNCTIONS:
 
 		// system read for everything else
 		default:

datastore/system/system_keyspace_functions.go

@@ -9,10 +9,12 @@
 package system
 
 import (
+	"github.com/couchbase/query/algebra"
+	"github.com/couchbase/query/auth"
 	"github.com/couchbase/query/datastore"
 	"github.com/couchbase/query/errors"
 	"github.com/couchbase/query/expression"
-	functions "github.com/couchbase/query/functions/metakv"
+	functionsStorage "github.com/couchbase/query/functions/storage"
 	"github.com/couchbase/query/timestamp"
 	"github.com/couchbase/query/value"
 )
@@ -38,12 +40,57 @@ func (b *functionsKeyspace) Name() string {
 }
 
 func (b *functionsKeyspace) Count(context datastore.QueryContext) (int64, errors.Error) {
-	count, err := functions.Count()
-	if err == nil {
-		return count, nil
-	} else {
-		return 0, errors.NewMetaKVError("Count", err)
+	var count int64
+
+	internal, external := hasGlobalFunctionsAccess(context)
+	lastScope := ""
+	hasScopeInternal := false
+	hasScopeExternal := false
+	isAdmin := datastore.IsAdmin(context.Credentials())
+	if internal || external {
+		err := functionsStorage.Foreach("", func(path string, v value.Value) error {
+			if !isAdmin {
+				i, err := functionsStorage.IsInternal(v)
+				if err != nil {
+					return err
+				}
+				if (i && !internal) || (!i && !external) {
+					return nil
+				}
+			}
+			count++
+			return nil
+		})
+		if err != nil {
+			return 0, errors.NewStorageAccessError("count", err)
+		}
+	}
+	buckets := datastore.GetDatastore().GetUserBuckets(context.Credentials())
+	for _, b := range buckets {
+		err := functionsStorage.Foreach(b, func(path string, v value.Value) error {
+			if !isAdmin {
+				parts := algebra.ParsePath(path)
+				scope := parts[1] + "." + parts[2]
+				if scope != lastScope {
+					hasScopeInternal, hasScopeExternal = hasScopeFunctionsAccess(path, context)
+					lastScope = scope
+				}
+				i, err := functionsStorage.IsInternal(v)
+				if err != nil {
+					return err
+				}
+				if (i && !hasScopeInternal) || (!i && !hasScopeExternal) {
+					return nil
+				}
+			}
+			count++
+			return nil
+		})
+		if err != nil {
+			return 0, errors.NewStorageAccessError("count", err)
+		}
 	}
+	return count, nil
 }
 
 func (b *functionsKeyspace) Size(context datastore.QueryContext) (int64, errors.Error) {
@@ -60,6 +107,12 @@ func (b *functionsKeyspace) Indexers() ([]datastore.Indexer, errors.Error) {
 
 func (b *functionsKeyspace) Fetch(keys []string, keysMap map[string]value.AnnotatedValue,
 	context datastore.QueryContext, subPaths []string) (errs errors.Errors) {
+
+	internal, external := hasGlobalFunctionsAccess(context)
+	lastScope := ""
+	hasScopeInternal := false
+	hasScopeExternal := false
+	isAdmin := datastore.IsAdmin(context.Credentials())
 	for _, k := range keys {
 		item, e := b.fetchOne(k)
 		if e != nil {
@@ -71,6 +124,42 @@ func (b *functionsKeyspace) Fetch(keys []string, keysMap map[string]value.Annota
 		}
 
 		if item != nil {
+			if !isAdmin {
+				parts := algebra.ParsePath(k)
+				switch len(parts) {
+				case 2:
+					i, err := functionsStorage.IsInternal(item)
+					if err != nil {
+						if errs == nil {
+							errs = make([]errors.Error, 0, 1)
+						}
+						errs = append(errs, errors.NewStorageAccessError("Fetch", err))
+						continue
+					}
+					if (i && !internal) || (!i && !external) {
+						continue
+					}
+				case 4:
+					scope := parts[1] + "." + parts[2]
+					if scope != lastScope {
+						hasScopeInternal, hasScopeExternal = hasScopeFunctionsAccess(k, context)
+						lastScope = scope
+					}
+					i, err := functionsStorage.IsInternal(item)
+					if err != nil {
+						if errs == nil {
+							errs = make([]errors.Error, 0, 1)
+						}
+						errs = append(errs, errors.NewStorageAccessError("Fetch", err))
+						continue
+					}
+					if (i && !hasScopeInternal) || (!i && !hasScopeExternal) {
+						continue
+					}
+				default:
+				}
+			}
+
 			item.NewMeta()["keyspace"] = b.fullName
 			item.SetId(k)
 		}
@@ -81,21 +170,16 @@ func (b *functionsKeyspace) Fetch(keys []string, keysMap map[string]value.Annota
 }
 
 func (b *functionsKeyspace) fetchOne(key string) (value.AnnotatedValue, errors.Error) {
-	body, err := functions.Get(key)
+	body, err := functionsStorage.Get(key)
 
 	// get does not return is not found, but nil, nil instead
 	if err == nil && body == nil {
 		return nil, errors.NewSystemDatastoreError(nil, "Key Not Found "+key)
 	}
 	if err != nil {
-		return nil, errors.NewMetaKVError("Fetch", err)
+		return nil, errors.NewStorageAccessError("Fetch", err)
 	}
-	return value.NewAnnotatedValue(value.NewParsedValue(body, false)), nil
-}
-
-// dodgy, but the not found error is not exported in metakv
-func isNotFoundError(err error) bool {
-	return err != nil && err.Error() == "Not found"
+	return value.NewAnnotatedValue(body), nil
 }
 
 func newFunctionsKeyspace(p *namespace) (*functionsKeyspace, errors.Error) {
@@ -165,16 +249,71 @@ func (pi *functionsIndex) Scan(requestId string, span *datastore.Span, distinct
 	pi.ScanEntries(requestId, limit, cons, vector, conn)
 }
 
+func hasGlobalFunctionsAccess(context datastore.QueryContext) (bool, bool) {
+	privs1 := auth.NewPrivileges()
+	privs1.Add("", auth.PRIV_QUERY_MANAGE_FUNCTIONS, auth.PRIV_PROPS_NONE)
+	privs2 := auth.NewPrivileges()
+	privs2.Add("", auth.PRIV_QUERY_EXECUTE_FUNCTIONS, auth.PRIV_PROPS_NONE)
+	err1 := datastore.GetDatastore().Authorize(privs1, context.Credentials())
+	err2 := datastore.GetDatastore().Authorize(privs2, context.Credentials())
+	internal := err1 == nil || err2 == nil
+
+	privs1 = auth.NewPrivileges()
+	privs1.Add("", auth.PRIV_QUERY_MANAGE_FUNCTIONS_EXTERNAL, auth.PRIV_PROPS_NONE)
+	privs2 = auth.NewPrivileges()
+	privs2.Add("", auth.PRIV_QUERY_EXECUTE_FUNCTIONS_EXTERNAL, auth.PRIV_PROPS_NONE)
+	err1 = datastore.GetDatastore().Authorize(privs1, context.Credentials())
+	err2 = datastore.GetDatastore().Authorize(privs2, context.Credentials())
+	external := err1 == nil || err2 == nil
+	return internal, external
+}
+
+func hasScopeFunctionsAccess(path string, context datastore.QueryContext) (bool, bool) {
+	privs1 := auth.NewPrivileges()
+	privs1.Add(path, auth.PRIV_QUERY_MANAGE_SCOPE_FUNCTIONS, auth.PRIV_PROPS_NONE)
+	privs2 := auth.NewPrivileges()
+	privs2.Add(path, auth.PRIV_QUERY_EXECUTE_SCOPE_FUNCTIONS, auth.PRIV_PROPS_NONE)
+	err1 := datastore.GetDatastore().Authorize(privs1, context.Credentials())
+	err2 := datastore.GetDatastore().Authorize(privs2, context.Credentials())
+	internal := err1 == nil || err2 == nil
+
+	privs1 = auth.NewPrivileges()
+	privs1.Add(path, auth.PRIV_QUERY_MANAGE_SCOPE_FUNCTIONS_EXTERNAL, auth.PRIV_PROPS_NONE)
+	privs2 = auth.NewPrivileges()
+	privs2.Add(path, auth.PRIV_QUERY_EXECUTE_SCOPE_FUNCTIONS_EXTERNAL, auth.PRIV_PROPS_NONE)
+	err1 = datastore.GetDatastore().Authorize(privs1, context.Credentials())
+	err2 = datastore.GetDatastore().Authorize(privs2, context.Credentials())
+	external := err1 == nil || err2 == nil
+	return internal, external
+}
+
 func (pi *functionsIndex) ScanEntries(requestId string, limit int64, cons datastore.ScanConsistency,
 	vector timestamp.Vector, conn *datastore.IndexConnection) {
 	defer conn.Sender().Close()
 
-	err := functions.Foreach(func(path string, value []byte) error {
-		entry := datastore.IndexEntry{PrimaryKey: path}
-		sendSystemKey(conn, &entry)
-		return nil
-	})
-	if err != nil {
-		conn.Error(errors.NewMetaKVIndexError(err))
+	context := conn.QueryContext()
+	internal, external := hasGlobalFunctionsAccess(context)
+	if internal || external {
+		err := functionsStorage.Scan("", func(path string) error {
+			entry := datastore.IndexEntry{PrimaryKey: path}
+			sendSystemKey(conn, &entry)
+			return nil
+		})
+		if err != nil {
+			conn.Error(errors.NewStorageAccessError("scan", err))
+			return
+		}
+	}
+	buckets := datastore.GetDatastore().GetUserBuckets(context.Credentials())
+	for _, b := range buckets {
+		err := functionsStorage.Scan(b, func(path string) error {
+			entry := datastore.IndexEntry{PrimaryKey: path}
+			sendSystemKey(conn, &entry)
+			return nil
+		})
+		if err != nil {
+			conn.Error(errors.NewStorageAccessError("scan", err))
+			return
+		}
 	}
 }

errors/codes.go

@@ -288,7 +288,7 @@ const (
 	E_INTERNAL_FUNCTION                       ErrorCode = 10103
 	E_ARGUMENTS_MISMATCH                      ErrorCode = 10104
 	E_INVALID_FUNCTION_NAME                   ErrorCode = 10105
-	E_METAKV                                  ErrorCode = 10106
+	E_FUNCTIONS_STORAGE                       ErrorCode = 10106
 	E_FUNCTION_ENCODING                       ErrorCode = 10107
 	E_FUNCTIONS_DISABLED                      ErrorCode = 10108
 	E_FUNCTION_EXECUTION                      ErrorCode = 10109

errors/function.go

@@ -67,22 +67,24 @@ func NewInvalidFunctionNameError(name string, e error) Error {
 }
 
 func NewMetaKVError(where string, what error) Error {
-	return &err{level: EXCEPTION, ICode: E_METAKV, IKey: "function.storage.error", ICause: what,
+	c := make(map[string]interface{})
+	c["cause"] = where
+	return &err{level: EXCEPTION, ICode: E_FUNCTIONS_STORAGE, IKey: "function.storage.error", ICause: what, cause: c,
 		InternalMsg:    fmt.Sprintf("Could not access function definition for %v because %v", where, what),
 		InternalCaller: CallerN(1)}
 }
 
 // same number and key as above, not an error
 func NewMetaKVChangeCounterError(what error) Error {
-	return &err{level: EXCEPTION, ICode: E_METAKV, IKey: "function.storage.error", ICause: what,
+	return &err{level: EXCEPTION, ICode: E_FUNCTIONS_STORAGE, IKey: "function.storage.error", ICause: what,
 		InternalMsg:    fmt.Sprintf("Could not access functions change counter because %v", what),
 		InternalCaller: CallerN(1)}
 }
 
 // same number and key as above, not an error
-func NewMetaKVIndexError(what error) Error {
-	return &err{level: EXCEPTION, ICode: E_METAKV, IKey: "function.storage.error", ICause: what,
-		InternalMsg:    fmt.Sprintf("Could not access functions definitions because %v", what),
+func NewStorageAccessError(where string, what error) Error {
+	return &err{level: EXCEPTION, ICode: E_FUNCTIONS_STORAGE, IKey: "function.storage.error", ICause: what,
+		InternalMsg:    fmt.Sprintf("Could not access functions definitions during %v because %v", where, what),
 		InternalCaller: CallerN(1)}
 }
 

functions/constructor/constructor.go

@@ -19,7 +19,9 @@ import (
 	"github.com/couchbase/query/functions/golang"
 	"github.com/couchbase/query/functions/inline"
 	"github.com/couchbase/query/functions/javascript"
-	storage "github.com/couchbase/query/functions/metakv"
+	metaStorage "github.com/couchbase/query/functions/metakv"
+	systemStorage "github.com/couchbase/query/functions/system"
+	"github.com/couchbase/query/tenant"
 	"github.com/gorilla/mux"
 )
 
@@ -29,7 +31,8 @@ func Init(mux *mux.Router, threads int) {
 	functionsBridge.NewGolangBody = golang.NewGolangBody
 	functionsBridge.NewJavascriptBody = javascript.NewJavascriptBody
 	authorize.Init()
-	storage.Init()
+	metaStorage.Init()
+	systemStorage.Init()
 	golang.Init()
 	inline.Init()
 	javascript.Init(mux, threads)
@@ -53,11 +56,15 @@ func newGlobalFunction(elem []string, namespace string, queryContext string) (fu
 	}
 	switch len(elem) {
 	case 1:
-		return storage.NewGlobalFunction(namespace, elem[0])
+		return metaStorage.NewGlobalFunction(namespace, elem[0])
 	case 2:
-		return storage.NewGlobalFunction(ns, elem[1])
+		return metaStorage.NewGlobalFunction(ns, elem[1])
 	case 4:
-		return storage.NewScopeFunction(ns, elem[1], elem[2], elem[3])
+		if tenant.IsServerless() {
+			return systemStorage.NewScopeFunction(ns, elem[1], elem[2], elem[3])
+		} else {
+			return metaStorage.NewScopeFunction(ns, elem[1], elem[2], elem[3])
+		}
 	default:
 		return nil, errors.NewInvalidFunctionNameError(elem[len(elem)-1], fmt.Errorf("invalid function path"))
 	}