-
Notifications
You must be signed in to change notification settings - Fork 38
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[#23] Support SQL-like queries for placement
JSON format is rather verbose an inconvenient to be edited by hand. This commit implements SQL-like language for representing placement policy. Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
- Loading branch information
Showing
7 changed files
with
531 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Package policy provides facilities for creating policy from SQL-like language. | ||
| // eBNF grammar is provided in `grammar.ebnf` for illustration. | ||
| // | ||
| // Current limitations: | ||
| // 1. Grouping filter expressions in parenthesis is not supported right now. | ||
| // Requiring this will make query too verbose, making it optional makes | ||
| // our grammar not LL(1). This can be supported in future. | ||
| // 2. Filters must be defined before they are used. | ||
| // This requirement may be relaxed in future. | ||
| // | ||
| // Example query: | ||
| // REP 1 in SPB | ||
| // REP 2 in Americas | ||
| // CBF 4 | ||
| // SELECT 1 Node IN City FROM SPBSSD AS SPB | ||
| // SELECT 2 Node IN SAME City FROM Americas AS Americas | ||
| // FILTER SSD EQ true AS IsSSD | ||
| // FILTER @IsSSD AND Country eq "RU" AND City eq "St.Petersburg" AS SPBSSD | ||
| // FILTER 'Continent' == 'North America' OR Continent == 'South America' AS Americas | ||
| package policy |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| Policy ::= | ||
| RepStmt, [RepStmt], | ||
| CbtStmt?, | ||
| [SelectStmt], | ||
| [FilterStmt], | ||
| ; | ||
| RepStmt ::= | ||
| 'REP', Number1, (* number of object replicas *) | ||
| ('AS', Ident)? (* optional selector name *) | ||
| ; | ||
| CbtStmt ::= 'CBF', Number1 (* container backup factor *) | ||
| ; | ||
| SelectStmt ::= | ||
| 'SELECT', Number1, (* number of nodes to select without container backup factor *) | ||
| 'IN', Clause?, Ident, (* bucket name *) | ||
| FROM, (Ident | '*'), (* filter reference or whole netmap *) | ||
| ('AS', Ident)? (* optional selector name *) | ||
| ; | ||
| Clause ::= | ||
| 'SAME' (* nodes from the same bucket *) | ||
| | 'DISTINCT' (* nodes from distinct buckets *) | ||
| ; | ||
| FilterStmt ::= | ||
| 'FILTER', AndChain, ['OR', AndChain], | ||
| 'AS', Ident (* obligatory filter name *) | ||
| ; | ||
| AndChain ::= | ||
| Expr, ['AND', Expr] | ||
| ; | ||
| Expr ::= | ||
| '@' Ident (* filter reference *) | ||
| | Ident, Op, Value (* attribute filter *) | ||
| ; | ||
| Op ::= 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE' | ||
| ; | ||
| Value ::= Ident | Number | String | ||
| ; | ||
| Number1 ::= Digit1 [Digit]; | ||
| Number ::= Digit [Digit]; | ||
| Digit1 ::= '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ; | ||
| Digit ::= '0' | Digit1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package policy | ||
|
|
||
| import ( | ||
| "github.com/alecthomas/participle" | ||
| ) | ||
|
|
||
| var parser *participle.Parser | ||
|
|
||
| func init() { | ||
| p, err := participle.Build(&query{}) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| parser = p | ||
| } | ||
|
|
||
| type query struct { | ||
| Replicas []*replicaStmt `@@+` | ||
| CBF uint32 `("CBF" @Int)?` | ||
| Selectors []*selectorStmt `@@*` | ||
| Filters []*filterStmt `@@*` | ||
| } | ||
|
|
||
| type replicaStmt struct { | ||
| Count int `"REP" @Int` | ||
| Selector string `("IN" @Ident)?` | ||
| } | ||
|
|
||
| type selectorStmt struct { | ||
| Count uint32 `"SELECT" @Int` | ||
| Clause string `"IN" @("SAME" | "DISTINCT")?` | ||
| Bucket string `@Ident` | ||
| Filter string `"FROM" @(Ident | "*")` | ||
| Name string `("AS" @Ident)?` | ||
| } | ||
|
|
||
| type filterStmt struct { | ||
| Value *orChain `"FILTER" @@` | ||
| Name string `"AS" @Ident` | ||
| } | ||
|
|
||
| type filterOrExpr struct { | ||
| Reference string `"@"@Ident` | ||
| Expr *simpleExpr `| @@` | ||
| } | ||
|
|
||
| type orChain struct { | ||
| Clauses []*andChain `@@ ("OR" @@)*` | ||
| } | ||
|
|
||
| type andChain struct { | ||
| Clauses []*filterOrExpr `@@ ("AND" @@)*` | ||
| } | ||
|
|
||
| type simpleExpr struct { | ||
| Key string `@Ident` | ||
| // We don't use literals here to improve error messages. | ||
| Op string `@Ident` | ||
| Value string `@(Ident | String | Int)` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| package policy | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| sdknm "github.com/nspcc-dev/neofs-api-go/pkg/netmap" | ||
| "github.com/nspcc-dev/neofs-api-go/v2/netmap" | ||
| ) | ||
|
|
||
| var ( | ||
| ErrInvalidNumber = errors.New("policy: expected positive integer") | ||
| ErrUnknownOp = errors.New("policy: unknown operation") | ||
| ErrUnknownFilter = errors.New("policy: filter not found") | ||
| ErrUnknownSelector = errors.New("policy: selector not found") | ||
| ) | ||
|
|
||
| func parse(s string) (*query, error) { | ||
| q := new(query) | ||
| err := parser.Parse(strings.NewReader(s), q) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return q, nil | ||
| } | ||
|
|
||
| // Parse parses s into a placement policy. | ||
| func Parse(s string) (*netmap.PlacementPolicy, error) { | ||
| q, err := parse(s) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| seenFilters := map[string]bool{} | ||
| fs := make([]*netmap.Filter, 0, len(q.Filters)) | ||
| for _, qf := range q.Filters { | ||
| f, err := filterFromOrChain(qf.Value, seenFilters) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| f.SetName(qf.Name) | ||
| fs = append(fs, f) | ||
| seenFilters[qf.Name] = true | ||
| } | ||
|
|
||
| seenSelectors := map[string]bool{} | ||
| ss := make([]*netmap.Selector, 0, len(q.Selectors)) | ||
| for _, qs := range q.Selectors { | ||
| if qs.Filter != sdknm.MainFilterName && !seenFilters[qs.Filter] { | ||
| return nil, fmt.Errorf("%w: '%s'", ErrUnknownFilter, qs.Filter) | ||
| } | ||
| s := new(netmap.Selector) | ||
| switch qs.Clause { | ||
| case "SAME": | ||
| s.SetClause(netmap.Same) | ||
| case "DISTINCT": | ||
| s.SetClause(netmap.Distinct) | ||
| default: | ||
| s.SetClause(netmap.UnspecifiedClause) | ||
| } | ||
| s.SetName(qs.Name) | ||
| seenSelectors[qs.Name] = true | ||
| s.SetFilter(qs.Filter) | ||
| s.SetAttribute(qs.Bucket) | ||
| if qs.Count == 0 { | ||
| return nil, fmt.Errorf("%w: SELECT", ErrInvalidNumber) | ||
| } | ||
| s.SetCount(qs.Count) | ||
| ss = append(ss, s) | ||
| } | ||
|
|
||
| rs := make([]*netmap.Replica, 0, len(q.Replicas)) | ||
| for _, qr := range q.Replicas { | ||
| r := new(netmap.Replica) | ||
| if qr.Selector != "" { | ||
| if !seenSelectors[qr.Selector] { | ||
| return nil, fmt.Errorf("%w: '%s'", ErrUnknownSelector, qr.Selector) | ||
| } | ||
| r.SetSelector(qr.Selector) | ||
| } | ||
| if qr.Count == 0 { | ||
| return nil, fmt.Errorf("%w: REP", ErrInvalidNumber) | ||
| } | ||
| r.SetCount(uint32(qr.Count)) | ||
| rs = append(rs, r) | ||
| } | ||
|
|
||
| p := new(netmap.PlacementPolicy) | ||
| p.SetFilters(fs) | ||
| p.SetSelectors(ss) | ||
| p.SetReplicas(rs) | ||
| p.SetContainerBackupFactor(q.CBF) | ||
| return p, nil | ||
| } | ||
|
|
||
| func filterFromOrChain(expr *orChain, seen map[string]bool) (*netmap.Filter, error) { | ||
| var fs []*netmap.Filter | ||
| for _, ac := range expr.Clauses { | ||
| f, err := filterFromAndChain(ac, seen) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| fs = append(fs, f) | ||
| } | ||
| if len(fs) == 1 { | ||
| return fs[0], nil | ||
| } | ||
|
|
||
| f := new(netmap.Filter) | ||
| f.SetOp(netmap.OR) | ||
| f.SetFilters(fs) | ||
| return f, nil | ||
| } | ||
|
|
||
| func filterFromAndChain(expr *andChain, seen map[string]bool) (*netmap.Filter, error) { | ||
| var fs []*netmap.Filter | ||
| for _, fe := range expr.Clauses { | ||
| var f *netmap.Filter | ||
| var err error | ||
| if fe.Expr != nil { | ||
| f, err = filterFromSimpleExpr(fe.Expr, seen) | ||
| } else { | ||
| f = new(netmap.Filter) | ||
| f.SetName(fe.Reference) | ||
| } | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| fs = append(fs, f) | ||
| } | ||
| if len(fs) == 1 { | ||
| return fs[0], nil | ||
| } | ||
|
|
||
| f := new(netmap.Filter) | ||
| f.SetOp(netmap.AND) | ||
| f.SetFilters(fs) | ||
| return f, nil | ||
| } | ||
|
|
||
| func filterFromSimpleExpr(se *simpleExpr, seen map[string]bool) (*netmap.Filter, error) { | ||
| f := new(netmap.Filter) | ||
| f.SetKey(se.Key) | ||
| switch se.Op { | ||
| case "EQ": | ||
| f.SetOp(netmap.EQ) | ||
| case "NE": | ||
| f.SetOp(netmap.NE) | ||
| case "GE": | ||
| f.SetOp(netmap.GE) | ||
| case "GT": | ||
| f.SetOp(netmap.GT) | ||
| case "LE": | ||
| f.SetOp(netmap.LE) | ||
| case "LT": | ||
| f.SetOp(netmap.LT) | ||
| default: | ||
| return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, se.Op) | ||
| } | ||
| f.SetValue(se.Value) | ||
| return f, nil | ||
| } |
Oops, something went wrong.