diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 9026314313..1bd9f08394 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -2,7 +2,7 @@ name: Nightly on: workflow_dispatch: {} # Allow for manual triggers schedule: - - cron: '0 8 * * *' # Daily, at 8:00 UTC + - cron: '0 8 * * 0-4' # Sun-Thu, at 8:00 UTC jobs: diff --git a/ast/annotations.go b/ast/annotations.go index 1134cc87c4..fe5e321997 100644 --- a/ast/annotations.go +++ b/ast/annotations.go @@ -11,6 +11,7 @@ import ( "sort" "strings" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/internal/deepcopy" "github.com/open-policy-agent/opa/util" ) @@ -39,7 +40,7 @@ type ( comments []*Comment node Node - jsonOptions JSONOptions + jsonOptions astJSON.Options } // SchemaAnnotation contains a schema declaration for the document identified by the path. @@ -76,7 +77,7 @@ type ( Annotations *Annotations `json:"annotations,omitempty"` Location *Location `json:"location,omitempty"` // The location of the node the annotations are applied to - jsonOptions JSONOptions + jsonOptions astJSON.Options node Node // The node the annotations are applied to } @@ -180,8 +181,11 @@ func (a *Annotations) GetTargetPath() Ref { } } -func (a *Annotations) setJSONOptions(opts JSONOptions) { +func (a *Annotations) setJSONOptions(opts astJSON.Options) { a.jsonOptions = opts + if a.Location != nil { + a.Location.JSONOptions = opts + } } func (a *Annotations) MarshalJSON() ([]byte, error) { diff --git a/ast/compile.go b/ast/compile.go index 28392cade0..b577d9e47d 100644 --- a/ast/compile.go +++ b/ast/compile.go @@ -7,7 +7,6 @@ package ast import ( "fmt" "io" - "os" "sort" "strconv" "strings" @@ -146,7 +145,6 @@ type Compiler struct { keepModules bool // whether to keep the unprocessed, parse modules (below) parsedModules map[string]*Module // parsed, but otherwise unprocessed modules, kept track of when keepModules is true useTypeCheckAnnotations bool // whether to provide annotated information (schemas) to the type checker - generalRuleRefsEnabled bool } // CompilerStage defines the interface for stages in the compiler. @@ -335,8 +333,6 @@ func NewCompiler() *Compiler { {"BuildComprehensionIndices", "compile_stage_rebuild_comprehension_indices", c.buildComprehensionIndices}, } - _, c.generalRuleRefsEnabled = os.LookupEnv("EXPERIMENTAL_GENERAL_RULE_REFS") - return c } @@ -1005,50 +1001,8 @@ func (c *Compiler) checkRuleConflicts() { // data.p.q[r][s] { r := input.r; s := input.s } // data.p[q].r.s { q := input.q } - if c.generalRuleRefsEnabled { - if r.Ref().IsGround() && len(node.Children) > 0 { - conflicts = node.flattenChildren() - } - } else { // TODO: Remove when general rule refs are enabled by default. - if r.Head.RuleKind() == SingleValue && len(node.Children) > 0 { - if len(ref) > 1 && !ref[len(ref)-1].IsGround() { // p.q[x] and p.q.s.t => check grandchildren - for _, c := range node.Children { - grandchildrenFound := false - - if len(c.Values) > 0 { - childRules := extractRules(c.Values) - for _, childRule := range childRules { - childRef := childRule.Ref() - if childRule.Head.RuleKind() == SingleValue && !childRef[len(childRef)-1].IsGround() { - // The child is a partial object rule, so it's effectively "generating" grandchildren. - grandchildrenFound = true - break - } - } - } - - if len(c.Children) > 0 { - grandchildrenFound = true - } - - if grandchildrenFound { - conflicts = node.flattenChildren() - break - } - } - } else { // p.q.s and p.q.s.t => any children are in conflict - conflicts = node.flattenChildren() - } - } - - // Multi-value rules may not have any other rules in their extent; e.g.: - // - // data.p[v] { v := ... } - // data.p.q := 42 # In direct conflict with data.p[v], which is constructing a set and cannot have values assigned to a sub-path. - - if r.Head.RuleKind() == MultiValue && len(node.Children) > 0 { - conflicts = node.flattenChildren() - } + if r.Ref().IsGround() && len(node.Children) > 0 { + conflicts = node.flattenChildren() } if r.Head.RuleKind() == SingleValue && r.Head.Ref().IsGround() { @@ -1811,18 +1765,6 @@ func (c *Compiler) rewriteRuleHeadRefs() { } for i := 1; i < len(ref); i++ { - // NOTE: Unless enabled via the EXPERIMENTAL_GENERAL_RULE_REFS env var, non-string values in the refs are forbidden - // except for the last position, e.g. - // OK: p.q.r[s] - // NOT OK: p[q].r.s - // TODO: Remove when general rule refs are enabled by default. - if !c.generalRuleRefsEnabled && i != len(ref)-1 { // last - if _, ok := ref[i].Value.(String); !ok { - c.err(NewError(TypeErr, rule.Loc(), "rule head must only contain string terms (except for last): %v", ref[i])) - continue - } - } - // Rewrite so that any non-scalar elements that in the last position of // the rule are vars: // p.q.r[y.z] { ... } => p.q.r[__local0__] { __local0__ = y.z } @@ -2385,8 +2327,9 @@ func (c *Compiler) rewriteLocalVarsInRule(rule *Rule, unusedArgs VarSet, argsSta // Rewrite assignments in body. used := NewVarSet() - last := rule.Head.Ref()[len(rule.Head.Ref())-1] - used.Update(last.Vars()) + for _, t := range rule.Head.Ref()[1:] { + used.Update(t.Vars()) + } if rule.Head.Key != nil { used.Update(rule.Head.Key.Vars()) diff --git a/ast/compile_test.go b/ast/compile_test.go index 2dcdf75ab8..2fb3532494 100644 --- a/ast/compile_test.go +++ b/ast/compile_test.go @@ -467,8 +467,6 @@ func toRef(s string) Ref { } func TestCompilerCheckRuleHeadRefs(t *testing.T) { - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - tests := []struct { note string modules []*Module @@ -607,64 +605,6 @@ func TestCompilerCheckRuleHeadRefs(t *testing.T) { } } -// TODO: Remove when general rule refs are enabled by default. -func TestCompilerCheckRuleHeadRefsWithGeneralRuleRefsDisabled(t *testing.T) { - - tests := []struct { - note string - modules []*Module - expected *Rule - err string - }{ - { - note: "ref contains var", - modules: modules( - `package x - p.q[i].r = 1 { i := 10 }`, - ), - err: "rego_type_error: rule head must only contain string terms (except for last): i", - }, - { - note: "invalid: ref in ref", - modules: modules( - `package x - p.q[arr[0]].r { i := 10 }`, - ), - err: "rego_type_error: rule head must only contain string terms (except for last): arr[0]", - }, - { - note: "invalid: non-string in ref (not last position)", - modules: modules( - `package x - p.q[10].r { true }`, - ), - err: "rego_type_error: rule head must only contain string terms (except for last): 10", - }, - } - - for _, tc := range tests { - t.Run(tc.note, func(t *testing.T) { - mods := make(map[string]*Module, len(tc.modules)) - for i, m := range tc.modules { - mods[fmt.Sprint(i)] = m - } - c := NewCompiler() - c.Modules = mods - compileStages(c, c.rewriteRuleHeadRefs) - if tc.err != "" { - assertCompilerErrorStrings(t, c, []string{tc.err}) - } else { - if len(c.Errors) > 0 { - t.Fatalf("expected no errors, got %v", c.Errors) - } - if tc.expected != nil { - assertRulesEqual(t, tc.expected, mods["0"].Rules[0]) - } - } - }) - } -} - func TestRuleTreeWithDotsInHeads(t *testing.T) { // TODO(sr): multi-val with var key in ref @@ -1940,8 +1880,6 @@ func TestCompilerCheckRuleConflictsDefaultFunction(t *testing.T) { } func TestCompilerCheckRuleConflictsDotsInRuleHeads(t *testing.T) { - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - tests := []struct { note string modules []*Module @@ -2187,70 +2125,6 @@ func TestCompilerCheckRuleConflictsDotsInRuleHeads(t *testing.T) { } } -// TODO: Remove when general rule refs are enabled by default. -func TestGeneralRuleRefsDisabled(t *testing.T) { - // EXPERIMENTAL_GENERAL_RULE_REFS env var not set - - tests := []struct { - note string - modules []*Module - err string - }{ - { - note: "single-value with other rule overlap, unknown key", - modules: modules( - `package pkg - p.q[r] = x { r = input.key; x = input.foo } - p.q.r.s = x { true } - `), - err: "rego_type_error: rule data.pkg.p.q[r] conflicts with [data.pkg.p.q.r.s]", - }, - { - note: "single-value with other rule overlap, unknown ref var and key", - modules: modules( - `package pkg - p.q[r][s] = x { r = input.key1; s = input.key2; x = input.foo } - p.q.r.s.t = x { true } - `), - err: "rego_type_error: rule head must only contain string terms (except for last): r", - }, - { - note: "single-value partial object with other partial object rule overlap, unknown keys (regression test for #5855; invalidated by multi-var refs)", - modules: modules( - `package pkg - p[r] := x { r = input.key; x = input.bar } - p.q[r] := x { r = input.key; x = input.bar } - `), - err: "rego_type_error: rule data.pkg.p[r] conflicts with [data.pkg.p.q[r]]", - }, - { - note: "single-value partial object with other partial object (implicit 'true' value) rule overlap, unknown keys", - modules: modules( - `package pkg - p[r] := x { r = input.key; x = input.bar } - p.q[r] { r = input.key } - `), - err: "rego_type_error: rule data.pkg.p[r] conflicts with [data.pkg.p.q[r]]", - }, - } - for _, tc := range tests { - t.Run(tc.note, func(t *testing.T) { - mods := make(map[string]*Module, len(tc.modules)) - for i, m := range tc.modules { - mods[fmt.Sprint(i)] = m - } - c := NewCompiler() - c.Modules = mods - compileStages(c, c.checkRuleConflicts) - if tc.err != "" { - assertCompilerErrorStrings(t, c, []string{tc.err}) - } else { - assertCompilerErrorStrings(t, c, []string{}) - } - }) - } -} - func TestCompilerCheckUndefinedFuncs(t *testing.T) { module := ` @@ -7278,6 +7152,29 @@ func TestCompilerCheckUnusedAssignedVar(t *testing.T) { &Error{Message: "assigned var y unused"}, }, }, + { + note: "general ref in rule head", + module: `package test + p[q].r[s] := 1 { + q := "foo" + s := "bar" + t := "baz" + } + `, + expectedErrors: Errors{ + &Error{Message: "assigned var t unused"}, + }, + }, + { + note: "general ref in rule head (no errors)", + module: `package test + p[q].r[s] := 1 { + q := "foo" + s := "bar" + } + `, + expectedErrors: Errors{}, + }, } makeTestRunner := func(tc testCase, strict bool) func(t *testing.T) { diff --git a/ast/json/json.go b/ast/json/json.go new file mode 100644 index 0000000000..2a72a9dbd6 --- /dev/null +++ b/ast/json/json.go @@ -0,0 +1,33 @@ +package json + +// Options defines the options for JSON operations, +// currently only marshaling can be configured +type Options struct { + MarshalOptions MarshalOptions +} + +// MarshalOptions defines the options for JSON marshaling, +// currently only toggling the marshaling of location information is supported +type MarshalOptions struct { + // IncludeLocation toggles the marshaling of location information + IncludeLocation NodeToggle + // IncludeLocationText additionally/optionally includes the text of the location + IncludeLocationText bool +} + +// NodeToggle is a generic struct to allow the toggling of +// settings for different ast node types +type NodeToggle struct { + Term bool + Package bool + Comment bool + Import bool + Rule bool + Head bool + Expr bool + SomeDecl bool + Every bool + With bool + Annotations bool + AnnotationsRef bool +} diff --git a/ast/location/location.go b/ast/location/location.go index 9ef1e6dfd4..1ac4d15282 100644 --- a/ast/location/location.go +++ b/ast/location/location.go @@ -3,8 +3,11 @@ package location import ( "bytes" + "encoding/json" "errors" "fmt" + + astJSON "github.com/open-policy-agent/opa/ast/json" ) // Location records a position in source code @@ -14,6 +17,9 @@ type Location struct { Row int `json:"row"` // The line in the source. Col int `json:"col"` // The column in the row. Offset int `json:"-"` // The byte offset for the location in the source. + + // JSONOptions specifies options for marshaling and unmarshaling of locations + JSONOptions astJSON.Options } // NewLocation returns a new Location object. @@ -87,3 +93,23 @@ func (loc *Location) Compare(other *Location) int { } return 0 } + +func (loc *Location) MarshalJSON() ([]byte, error) { + // structs are used here to preserve the field ordering of the original Location struct + data := struct { + File string `json:"file"` + Row int `json:"row"` + Col int `json:"col"` + Text []byte `json:"text,omitempty"` + }{ + File: loc.File, + Row: loc.Row, + Col: loc.Col, + } + + if loc.JSONOptions.MarshalOptions.IncludeLocationText { + data.Text = loc.Text + } + + return json.Marshal(data) +} diff --git a/ast/location/location_test.go b/ast/location/location_test.go index a6e54c62d1..f61fec5edd 100644 --- a/ast/location/location_test.go +++ b/ast/location/location_test.go @@ -1,8 +1,10 @@ package location import ( + "encoding/json" "testing" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/util" ) @@ -85,3 +87,46 @@ func TestLocationCompare(t *testing.T) { } } } + +func TestLocationMarshal(t *testing.T) { + testCases := map[string]struct { + loc *Location + exp string + }{ + "default json options": { + loc: &Location{ + Text: []byte("text"), + File: "file", + Row: 1, + Col: 1, + }, + exp: `{"file":"file","row":1,"col":1}`, + }, + "including text": { + loc: &Location{ + Text: []byte("text"), + File: "file", + Row: 1, + Col: 1, + JSONOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocationText: true, + }, + }, + }, + exp: `{"file":"file","row":1,"col":1,"text":"dGV4dA=="}`, + }, + } + + for id, tc := range testCases { + t.Run(id, func(t *testing.T) { + bs, err := json.Marshal(tc.loc) + if err != nil { + t.Fatal(err) + } + if string(bs) != tc.exp { + t.Fatalf("Expected %v but got %v", tc.exp, string(bs)) + } + }) + } +} diff --git a/ast/marshal.go b/ast/marshal.go index 891945db8b..53fb112044 100644 --- a/ast/marshal.go +++ b/ast/marshal.go @@ -1,7 +1,11 @@ package ast +import ( + astJSON "github.com/open-policy-agent/opa/ast/json" +) + // customJSON is an interface that can be implemented by AST nodes that // allows the parser to set options for JSON operations on that node. type customJSON interface { - setJSONOptions(JSONOptions) + setJSONOptions(astJSON.Options) } diff --git a/ast/marshal_test.go b/ast/marshal_test.go index 38b5216279..d6410102f6 100644 --- a/ast/marshal_test.go +++ b/ast/marshal_test.go @@ -4,9 +4,79 @@ import ( "encoding/json" "testing" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/util" ) +func TestGeneric_MarshalWithLocationJSONOptions(t *testing.T) { + testCases := map[string]struct { + Term *Term + ExpectedJSON string + }{ + "base case, no location options set": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + }(), + ExpectedJSON: `{"type":"string","value":"example"}`, + }, + "location included, location text excluded": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ + Term: true, + }, + IncludeLocationText: false, + }, + }, + } + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"type":"string","value":"example"}`, + }, + "location included, location text also included": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + t := &Term{ + Value: v, + Location: NewLocation([]byte("things"), "example.rego", 1, 2), + } + t.setJSONOptions( + astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ + Term: true, + }, + IncludeLocationText: true, + }, + }, + ) + return t + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2,"text":"dGhpbmdz"},"type":"string","value":"example"}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Term) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + func TestTerm_MarshalJSON(t *testing.T) { testCases := map[string]struct { Term *Term @@ -28,9 +98,9 @@ func TestTerm_MarshalJSON(t *testing.T) { return &Term{ Value: v, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: false, }, }, @@ -45,9 +115,9 @@ func TestTerm_MarshalJSON(t *testing.T) { return &Term{ Value: v, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: true, }, }, @@ -132,9 +202,9 @@ func TestPackage_MarshalJSON(t *testing.T) { Package: &Package{ Path: EmptyRef(), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Package: false, }, }, @@ -146,9 +216,9 @@ func TestPackage_MarshalJSON(t *testing.T) { Package: &Package{ Path: EmptyRef(), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Package: true, }, }, @@ -188,9 +258,9 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Comment: false, // ignored }, }, @@ -202,9 +272,9 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Comment: true, // ignored }, }, @@ -253,9 +323,9 @@ func TestImport_MarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Import: false, }, }, @@ -274,9 +344,9 @@ func TestImport_MarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Import: true, }, }, @@ -327,9 +397,9 @@ func TestRule_MarshalJSON(t *testing.T) { "location excluded": { Rule: func() *Rule { r := rule.Copy() - r.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + r.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Rule: false, }, }, @@ -341,9 +411,9 @@ func TestRule_MarshalJSON(t *testing.T) { "location included": { Rule: func() *Rule { r := rule.Copy() - r.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + r.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Rule: true, }, }, @@ -394,9 +464,9 @@ func TestHead_MarshalJSON(t *testing.T) { "location excluded": { Head: func() *Head { h := head.Copy() - h.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + h.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Head: false, }, }, @@ -409,9 +479,9 @@ func TestHead_MarshalJSON(t *testing.T) { "location included": { Head: func() *Head { h := head.Copy() - h.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + h.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Head: true, }, }, @@ -462,9 +532,9 @@ func TestExpr_MarshalJSON(t *testing.T) { "location excluded": { Expr: func() *Expr { e := expr.Copy() - e.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + e.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Expr: false, }, }, @@ -477,9 +547,9 @@ func TestExpr_MarshalJSON(t *testing.T) { "location included": { Expr: func() *Expr { e := expr.Copy() - e.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + e.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Expr: true, }, }, @@ -581,7 +651,7 @@ func TestSomeDecl_MarshalJSON(t *testing.T) { SomeDecl: &SomeDecl{ Symbols: []*Term{term}, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{SomeDecl: false}}}, + jsonOptions: astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{SomeDecl: false}}}, }, ExpectedJSON: `{"symbols":[{"type":"string","value":"example"}]}`, }, @@ -589,7 +659,7 @@ func TestSomeDecl_MarshalJSON(t *testing.T) { SomeDecl: &SomeDecl{ Symbols: []*Term{term}, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{SomeDecl: true}}}, + jsonOptions: astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{SomeDecl: true}}}, }, ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"symbols":[{"type":"string","value":"example"}]}`, }, @@ -643,7 +713,7 @@ allow { "location excluded": { Every: func() *Every { e := every.Copy() - e.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{Every: false}}} + e.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{Every: false}}} return e }(), ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, @@ -651,7 +721,7 @@ allow { "location included": { Every: func() *Every { e := every.Copy() - e.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{Every: true}}} + e.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{Every: true}}} return e }(), ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"location":{"file":"example.rego","row":7,"col":2},"value":{"type":"var","value":"e"}}`, @@ -701,7 +771,7 @@ b { "location excluded": { With: func() *With { w := with.Copy() - w.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{With: false}}} + w.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{With: false}}} return w }(), ExpectedJSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, @@ -709,7 +779,7 @@ b { "location included": { With: func() *With { w := with.Copy() - w.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{With: true}}} + w.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{With: true}}} return w }(), ExpectedJSON: `{"location":{"file":"example.rego","row":7,"col":4},"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, @@ -761,9 +831,9 @@ func TestAnnotations_MarshalJSON(t *testing.T) { }, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{Annotations: false}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{Annotations: false}, }, }, }, @@ -781,9 +851,9 @@ func TestAnnotations_MarshalJSON(t *testing.T) { }, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{Annotations: true}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{Annotations: true}, }, }, }, @@ -824,9 +894,9 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { Path: []*Term{}, Annotations: &Annotations{}, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{AnnotationsRef: false}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{AnnotationsRef: false}, }, }, }, @@ -838,9 +908,9 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { Annotations: &Annotations{}, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{AnnotationsRef: true}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{AnnotationsRef: true}, }, }, }, @@ -916,9 +986,9 @@ package test p = 1`, options: ParserOptions{ ProcessAnnotation: true, - JSONOptions: &JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + JSONOptions: &astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: true, Package: true, Comment: true, @@ -937,8 +1007,8 @@ p = 1`, }, expected: []string{ `{"annotations":{"authors":[{"name":"pkg"}],"custom":{"pkg":"pkg"},"description":"pkg","location":{"file":"","row":1,"col":1},"organizations":["pkg"],"related_resources":[{"ref":"https://pkg"}],"schemas":[{"path":[{"type":"var","value":"input"},{"type":"string","value":"foo"}],"definition":{"type":"boolean"}}],"scope":"package","title":"pkg"},"location":{"file":"","row":14,"col":1},"path":[{"location":{"file":"","row":14,"col":9},"type":"var","value":"data"},{"location":{"file":"","row":14,"col":9},"type":"string","value":"test"}]}`, - `{"annotations":{"authors":[{"name":"doc"}],"custom":{"doc":"doc"},"description":"doc","location":{"file":"","row":16,"col":1},"organizations":["doc"],"related_resources":[{"ref":"https://doc"}],"schemas":[{"path":[{"type":"var","value":"input"},{"type":"string","value":"bar"}],"definition":{"type":"integer"}}],"scope":"document","title":"doc"},"location":{"file":"","row":44,"col":1},"path":[{"location":{"file":"","row":14,"col":9},"type":"var","value":"data"},{"location":{"file":"","row":14,"col":9},"type":"string","value":"test"},{"type":"string","value":"p"}]}`, - `{"annotations":{"authors":[{"name":"rule"}],"custom":{"rule":"rule"},"description":"rule","location":{"file":"","row":31,"col":1},"organizations":["rule"],"related_resources":[{"ref":"https://rule"}],"schemas":[{"path":[{"type":"var","value":"input"},{"type":"string","value":"baz"}],"definition":{"type":"string"}}],"scope":"rule","title":"rule"},"location":{"file":"","row":44,"col":1},"path":[{"location":{"file":"","row":14,"col":9},"type":"var","value":"data"},{"location":{"file":"","row":14,"col":9},"type":"string","value":"test"},{"type":"string","value":"p"}]}`, + `{"annotations":{"authors":[{"name":"doc"}],"custom":{"doc":"doc"},"description":"doc","location":{"file":"","row":16,"col":1},"organizations":["doc"],"related_resources":[{"ref":"https://doc"}],"schemas":[{"path":[{"type":"var","value":"input"},{"type":"string","value":"bar"}],"definition":{"type":"integer"}}],"scope":"document","title":"doc"},"location":{"file":"","row":44,"col":1},"path":[{"location":{"file":"","row":14,"col":9},"type":"var","value":"data"},{"location":{"file":"","row":14,"col":9},"type":"string","value":"test"},{"location":{"file":"","row":44,"col":1},"type":"string","value":"p"}]}`, + `{"annotations":{"authors":[{"name":"rule"}],"custom":{"rule":"rule"},"description":"rule","location":{"file":"","row":31,"col":1},"organizations":["rule"],"related_resources":[{"ref":"https://rule"}],"schemas":[{"path":[{"type":"var","value":"input"},{"type":"string","value":"baz"}],"definition":{"type":"string"}}],"scope":"rule","title":"rule"},"location":{"file":"","row":44,"col":1},"path":[{"location":{"file":"","row":14,"col":9},"type":"var","value":"data"},{"location":{"file":"","row":14,"col":9},"type":"string","value":"test"},{"location":{"file":"","row":44,"col":1},"type":"string","value":"p"}]}`, }, }, } diff --git a/ast/parser.go b/ast/parser.go index 540d7b80c1..7cbc0b2892 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -11,7 +11,6 @@ import ( "io" "math/big" "net/url" - "os" "regexp" "sort" "strconv" @@ -22,6 +21,7 @@ import ( "github.com/open-policy-agent/opa/ast/internal/scanner" "github.com/open-policy-agent/opa/ast/internal/tokens" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/ast/location" ) @@ -97,43 +97,13 @@ func (e *parsedTermCacheItem) String() string { // ParserOptions defines the options for parsing Rego statements. type ParserOptions struct { - Capabilities *Capabilities - ProcessAnnotation bool - AllFutureKeywords bool - FutureKeywords []string - SkipRules bool - JSONOptions *JSONOptions - unreleasedKeywords bool // TODO(sr): cleanup - generalRuleRefsEnabled bool -} - -// JSONOptions defines the options for JSON operations, -// currently only marshaling can be configured -type JSONOptions struct { - MarshalOptions JSONMarshalOptions -} - -// JSONMarshalOptions defines the options for JSON marshaling, -// currently only toggling the marshaling of location information is supported -type JSONMarshalOptions struct { - IncludeLocation NodeToggle -} - -// NodeToggle is a generic struct to allow the toggling of -// settings for different ast node types -type NodeToggle struct { - Term bool - Package bool - Comment bool - Import bool - Rule bool - Head bool - Expr bool - SomeDecl bool - Every bool - With bool - Annotations bool - AnnotationsRef bool + Capabilities *Capabilities + ProcessAnnotation bool + AllFutureKeywords bool + FutureKeywords []string + SkipRules bool + JSONOptions *astJSON.Options + unreleasedKeywords bool // TODO(sr): cleanup } // NewParser creates and initializes a Parser. @@ -142,7 +112,6 @@ func NewParser() *Parser { s: &state{}, po: ParserOptions{}, } - _, p.po.generalRuleRefsEnabled = os.LookupEnv("EXPERIMENTAL_GENERAL_RULE_REFS") return p } @@ -210,9 +179,9 @@ func (p *Parser) WithSkipRules(skip bool) *Parser { return p } -// WithJSONOptions sets the JSONOptions which will be set on nodes to configure +// WithJSONOptions sets the Options which will be set on nodes to configure // their JSON marshaling behavior. -func (p *Parser) WithJSONOptions(jsonOptions *JSONOptions) *Parser { +func (p *Parser) WithJSONOptions(jsonOptions *astJSON.Options) *Parser { p.po.JSONOptions = jsonOptions return p } @@ -627,11 +596,6 @@ func (p *Parser) parseRules() []*Rule { return []*Rule{&rule} } - if !p.po.generalRuleRefsEnabled && usesContains && !rule.Head.Reference.IsGround() { - p.error(p.s.Loc(), "multi-value rules need ground refs") - return nil - } - // back-compat with `p[x] { ... }`` hasIf := p.s.tok == tokens.If @@ -795,43 +759,36 @@ func (p *Parser) parseElse(head *Head) *Rule { } hasIf := p.s.tok == tokens.If + hasLBrace := p.s.tok == tokens.LBrace - if hasIf { - p.scan() - s := p.save() - if expr := p.parseLiteral(); expr != nil { - // NOTE(sr): set literals are never false or undefined, so parsing this as - // p if false else if { true } - // ^^^^^^^^ set of one element, `true` - // isn't valid. - isSetLiteral := false - if t, ok := expr.Terms.(*Term); ok { - _, isSetLiteral = t.Value.(Set) - } - // expr.Term is []*Term or Every - if !isSetLiteral { - rule.Body.Append(expr) - setLocRecursive(rule.Body, rule.Location) - return &rule - } - } - p.restore(s) - } - - if p.s.tok != tokens.LBrace { + if !hasIf && !hasLBrace { rule.Body = NewBody(NewExpr(BooleanTerm(true))) setLocRecursive(rule.Body, rule.Location) return &rule } - p.scan() + if hasIf { + p.scan() + } - if rule.Body = p.parseBody(tokens.RBrace); rule.Body == nil { + if p.s.tok == tokens.LBrace { + p.scan() + if rule.Body = p.parseBody(tokens.RBrace); rule.Body == nil { + return nil + } + p.scan() + } else if p.s.tok != tokens.EOF { + expr := p.parseLiteral() + if expr == nil { + return nil + } + rule.Body.Append(expr) + setLocRecursive(rule.Body, rule.Location) + } else { + p.illegal("rule body expected") return nil } - p.scan() - if p.s.tok == tokens.Else { if rule.Else = p.parseElse(head); rule.Else == nil { return nil @@ -841,7 +798,6 @@ func (p *Parser) parseElse(head *Head) *Rule { } func (p *Parser) parseHead(defaultRule bool) (*Head, bool) { - head := &Head{} loc := p.s.Loc() defer func() { @@ -864,7 +820,9 @@ func (p *Parser) parseHead(defaultRule bool) (*Head, bool) { switch x := ref.Value.(type) { case Var: - head = NewHead(x) + // Modify the code to add the location to the head ref + // and set the head ref's jsonOptions. + head = VarHead(x, ref.Location, p.po.JSONOptions) case Ref: head = RefHead(x) case Call: diff --git a/ast/parser_ext.go b/ast/parser_ext.go index 99430a8fc5..9226a9059a 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -16,6 +16,8 @@ import ( "fmt" "strings" "unicode" + + astJSON "github.com/open-policy-agent/opa/ast/json" ) // MustParseBody returns a parsed body. @@ -244,7 +246,9 @@ func ParseCompleteDocRuleFromEqExpr(module *Module, lhs, rhs *Term) (*Rule, erro var head *Head if v, ok := lhs.Value.(Var); ok { - head = NewHead(v) + // Modify the code to add the location to the head ref + // and set the head ref's jsonOptions. + head = VarHead(v, lhs.Location, &lhs.jsonOptions) } else if r, ok := lhs.Value.(Ref); ok { // groundness ? if _, ok := r[0].Value.(Var); !ok { return nil, fmt.Errorf("invalid rule head: %v", r) @@ -350,7 +354,9 @@ func ParsePartialSetDocRuleFromTerm(module *Module, term *Term) (*Rule, error) { if !ok { return nil, fmt.Errorf("%vs cannot be used for rule head", TypeName(term.Value)) } - head = NewHead(v) + // Modify the code to add the location to the head ref + // and set the head ref's jsonOptions. + head = VarHead(v, ref[0].Location, &ref[0].jsonOptions) head.Key = ref[1] } head.Location = term.Location @@ -709,7 +715,7 @@ func setRuleModule(rule *Rule, module *Module) { } } -func setJSONOptions(x interface{}, jsonOptions *JSONOptions) { +func setJSONOptions(x interface{}, jsonOptions *astJSON.Options) { vis := NewGenericVisitor(func(x interface{}) bool { if x, ok := x.(customJSON); ok { x.setJSONOptions(*jsonOptions) diff --git a/ast/parser_test.go b/ast/parser_test.go index b4cb9951a9..7a2af2ae05 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/open-policy-agent/opa/ast/internal/tokens" + astJSON "github.com/open-policy-agent/opa/ast/json" ) const ( @@ -3315,9 +3316,9 @@ func TestRuleFromBodyJSONOptions(t *testing.T) { } parserOpts := ParserOptions{ProcessAnnotation: true} - parserOpts.JSONOptions = &JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + parserOpts.JSONOptions = &astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: true, Package: true, Comment: true, @@ -5136,33 +5137,33 @@ func assertParseModuleJSONOptions(t *testing.T, msg string, input string, opts . rule := m.Rules[0] if rule.Head.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected rule Head JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Head.jsonOptions) + t.Fatalf("Error on test \"%s\": expected rule Head Options\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Head.jsonOptions) } if rule.Body[0].jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected rule Body JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Body[0].jsonOptions) + t.Fatalf("Error on test \"%s\": expected rule Body Options\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Body[0].jsonOptions) } switch terms := rule.Body[0].Terms.(type) { case []*Term: for _, term := range terms { if term.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, term.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, term.jsonOptions) } } case *SomeDecl: if terms.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) } case *Every: if terms.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) } case *Term: if terms.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) } } if rule.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected rule JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.jsonOptions) + t.Fatalf("Error on test \"%s\": expected rule Options\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.jsonOptions) } } diff --git a/ast/policy.go b/ast/policy.go index 2822b82d87..fedc738ff2 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -12,6 +12,7 @@ import ( "strings" "time" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/util" ) @@ -153,7 +154,7 @@ type ( Text []byte Location *Location - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Package represents the namespace of the documents produced @@ -162,7 +163,7 @@ type ( Path Ref `json:"path"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Import represents a dependency on a document outside of the policy @@ -172,7 +173,7 @@ type ( Alias Var `json:"alias,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Rule represents a rule as defined in the language. Rules define the @@ -190,7 +191,7 @@ type ( // on the rule (e.g., printing, comparison, visiting, etc.) Module *Module `json:"-"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Head represents the head of a rule. @@ -203,7 +204,7 @@ type ( Assign bool `json:"assign,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Args represents zero or more arguments to a rule. @@ -222,7 +223,7 @@ type ( Negated bool `json:"negated,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // SomeDecl represents a variable declaration statement. The symbols are variables. @@ -230,7 +231,7 @@ type ( Symbols []*Term `json:"symbols"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } Every struct { @@ -240,7 +241,7 @@ type ( Body Body `json:"body"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // With represents a modifier on an expression. @@ -249,7 +250,7 @@ type ( Value *Term `json:"value"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } ) @@ -428,10 +429,13 @@ func (c *Comment) Equal(other *Comment) bool { return c.Location.Equal(other.Location) && bytes.Equal(c.Text, other.Text) } -func (c *Comment) setJSONOptions(opts JSONOptions) { +func (c *Comment) setJSONOptions(opts astJSON.Options) { // Note: this is not used for location since Comments use default JSON marshaling // behavior with struct field names in JSON. c.jsonOptions = opts + if c.Location != nil { + c.Location.JSONOptions = opts + } } // Compare returns an integer indicating whether pkg is less than, equal to, @@ -478,8 +482,11 @@ func (pkg *Package) String() string { return fmt.Sprintf("package %v", path) } -func (pkg *Package) setJSONOptions(opts JSONOptions) { +func (pkg *Package) setJSONOptions(opts astJSON.Options) { pkg.jsonOptions = opts + if pkg.Location != nil { + pkg.Location.JSONOptions = opts + } } func (pkg *Package) MarshalJSON() ([]byte, error) { @@ -588,8 +595,11 @@ func (imp *Import) String() string { return strings.Join(buf, " ") } -func (imp *Import) setJSONOptions(opts JSONOptions) { +func (imp *Import) setJSONOptions(opts astJSON.Options) { imp.jsonOptions = opts + if imp.Location != nil { + imp.Location.JSONOptions = opts + } } func (imp *Import) MarshalJSON() ([]byte, error) { @@ -699,8 +709,11 @@ func (rule *Rule) String() string { return strings.Join(buf, " ") } -func (rule *Rule) setJSONOptions(opts JSONOptions) { +func (rule *Rule) setJSONOptions(opts astJSON.Options) { rule.jsonOptions = opts + if rule.Location != nil { + rule.Location.JSONOptions = opts + } } func (rule *Rule) MarshalJSON() ([]byte, error) { @@ -769,6 +782,17 @@ func NewHead(name Var, args ...*Term) *Head { return head } +// VarHead creates a head object, initializes its Name, Location, and Options, +// and returns the new head. +func VarHead(name Var, location *Location, jsonOpts *astJSON.Options) *Head { + h := NewHead(name) + h.Reference[0].Location = location + if jsonOpts != nil { + h.Reference[0].setJSONOptions(*jsonOpts) + } + return h +} + // RefHead returns a new Head object with the passed Ref. If args are provided, // the first will be used for the value. func RefHead(ref Ref, args ...*Term) *Head { @@ -915,8 +939,11 @@ func (head *Head) String() string { return buf.String() } -func (head *Head) setJSONOptions(opts JSONOptions) { +func (head *Head) setJSONOptions(opts astJSON.Options) { head.jsonOptions = opts + if head.Location != nil { + head.Location.JSONOptions = opts + } } func (head *Head) MarshalJSON() ([]byte, error) { @@ -1465,8 +1492,11 @@ func (expr *Expr) String() string { return strings.Join(buf, " ") } -func (expr *Expr) setJSONOptions(opts JSONOptions) { +func (expr *Expr) setJSONOptions(opts astJSON.Options) { expr.jsonOptions = opts + if expr.Location != nil { + expr.Location.JSONOptions = opts + } } func (expr *Expr) MarshalJSON() ([]byte, error) { @@ -1561,8 +1591,11 @@ func (d *SomeDecl) Hash() int { return termSliceHash(d.Symbols) } -func (d *SomeDecl) setJSONOptions(opts JSONOptions) { +func (d *SomeDecl) setJSONOptions(opts astJSON.Options) { d.jsonOptions = opts + if d.Location != nil { + d.Location.JSONOptions = opts + } } func (d *SomeDecl) MarshalJSON() ([]byte, error) { @@ -1635,8 +1668,11 @@ func (q *Every) KeyValueVars() VarSet { return vis.vars } -func (q *Every) setJSONOptions(opts JSONOptions) { +func (q *Every) setJSONOptions(opts astJSON.Options) { q.jsonOptions = opts + if q.Location != nil { + q.Location.JSONOptions = opts + } } func (q *Every) MarshalJSON() ([]byte, error) { @@ -1714,8 +1750,11 @@ func (w *With) SetLoc(loc *Location) { w.Location = loc } -func (w *With) setJSONOptions(opts JSONOptions) { +func (w *With) setJSONOptions(opts astJSON.Options) { w.jsonOptions = opts + if w.Location != nil { + w.Location.JSONOptions = opts + } } func (w *With) MarshalJSON() ([]byte, error) { diff --git a/ast/term.go b/ast/term.go index e89366aa74..9ef8ae7474 100644 --- a/ast/term.go +++ b/ast/term.go @@ -22,6 +22,7 @@ import ( "github.com/OneOfOne/xxhash" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/ast/location" "github.com/open-policy-agent/opa/util" ) @@ -294,7 +295,7 @@ type Term struct { Value Value `json:"value"` // the value of the Term as represented in Go Location *Location `json:"location,omitempty"` // the location of the Term in the source - jsonOptions JSONOptions + jsonOptions astJSON.Options } // NewTerm returns a new Term object. @@ -419,8 +420,11 @@ func (term *Term) IsGround() bool { return term.Value.IsGround() } -func (term *Term) setJSONOptions(opts JSONOptions) { +func (term *Term) setJSONOptions(opts astJSON.Options) { term.jsonOptions = opts + if term.Location != nil { + term.Location.JSONOptions = opts + } } // MarshalJSON returns the JSON encoding of the term. @@ -1028,6 +1032,20 @@ func (ref Ref) ConstantPrefix() Ref { return ref[:i] } +func (ref Ref) StringPrefix() Ref { + r := ref.Copy() + + for i := 1; i < len(ref); i++ { + switch r[i].Value.(type) { + case String: // pass + default: // cut off + return r[:i] + } + } + + return r +} + // GroundPrefix returns the ground portion of the ref starting from the head. By // definition, the head of the reference is always ground. func (ref Ref) GroundPrefix() Ref { diff --git a/bundle/bundle.go b/bundle/bundle.go index 5567a2ae59..66c8ce37c6 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/format" "github.com/open-policy-agent/opa/internal/file/archive" "github.com/open-policy-agent/opa/internal/merge" @@ -391,7 +392,7 @@ type Reader struct { verificationConfig *VerificationConfig skipVerify bool processAnnotations bool - jsonOptions *ast.JSONOptions + jsonOptions *astJSON.Options capabilities *ast.Capabilities files map[string]FileInfo // files in the bundle signature payload sizeLimitBytes int64 @@ -463,7 +464,7 @@ func (r *Reader) WithCapabilities(caps *ast.Capabilities) *Reader { } // WithJSONOptions sets the JSONOptions to use when parsing policy files -func (r *Reader) WithJSONOptions(opts *ast.JSONOptions) *Reader { +func (r *Reader) WithJSONOptions(opts *astJSON.Options) *Reader { r.jsonOptions = opts return r } diff --git a/cmd/eval_test.go b/cmd/eval_test.go index 8050fb614d..3e84f31701 100755 --- a/cmd/eval_test.go +++ b/cmd/eval_test.go @@ -1653,3 +1653,230 @@ func TestBundleWithStrictFlag(t *testing.T) { } } + +func TestIfElseIfElseNoBrace(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else := 1 if false + else := 2`, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + defined, err := eval([]string{"data.bug.p"}, params, &buf) + if !defined || err != nil { + t.Fatalf("Unexpected undefined or error: %v", err) + } + }) +} + +func TestIfElseIfElseBrace(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else := 1 if { false } + else := 2`, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + defined, err := eval([]string{"data.bug.p"}, params, &buf) + if !defined || err != nil { + t.Fatalf("Unexpected undefined or error: %v", err) + } + }) +} + +func TestIfElse(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else := 1 `, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + defined, err := eval([]string{"data.bug.p"}, params, &buf) + if !defined || err != nil { + t.Fatalf("Unexpected undefined or error: %v", err) + } + }) +} + +func TestElseNoIf(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else = x { + x=2 + } `, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + defined, err := eval([]string{"data.bug.p"}, params, &buf) + if !defined || err != nil { + t.Fatalf("Unexpected undefined or error: %v", err) + } + }) +} + +func TestElseIf(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else := x if { + x=2 + } `, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + defined, err := eval([]string{"data.bug.p"}, params, &buf) + if !defined || err != nil { + t.Fatalf("Unexpected undefined or error: %v", err) + } + }) +} + +func TestElseIfElse(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else := x if { + x=2 + 1==2 + } else =x { + x=3 + }`, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + defined, err := eval([]string{"data.bug.p"}, params, &buf) + if !defined || err != nil { + t.Fatalf("Unexpected undefined or error: %v", err) + } + }) +} + +func TestUnexpectedElseIfElseErr(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + p if false + else := x if { + x=2 + 1==2 + } else + x=3 + `, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + _, err := eval([]string{"data.bug.p"}, params, &buf) + + // Check if there was an error + if err == nil { + t.Fatalf("expected an error, but got nil") + } + + // Check the error message + errorMessage := err.Error() + expectedErrorMessage := "rego_parse_error: unexpected ident token: expected else value term or rule body" + if !strings.Contains(errorMessage, expectedErrorMessage) { + t.Fatalf("expected error message to contain '%s', but got '%s'", expectedErrorMessage, errorMessage) + } + }) +} + +func TestUnexpectedElseIfErr(t *testing.T) { + files := map[string]string{ + "bug.rego": `package bug + import future.keywords.if + q := 1 if false + else := 2 if + `, + } + + test.WithTempFS(files, func(path string) { + + params := newEvalCommandParams() + params.optimizationLevel = 1 + params.dataPaths = newrepeatedStringFlag([]string{path}) + params.entrypoints = newrepeatedStringFlag([]string{"bug/p"}) + + var buf bytes.Buffer + + _, err := eval([]string{"data.bug.p"}, params, &buf) + + // Check if there was an error + if err == nil { + t.Fatalf("expected an error, but got nil") + } + + // Check the error message + errorMessage := err.Error() + expectedErrorMessage := "rego_parse_error: unexpected eof token: rule body expected" + if !strings.Contains(errorMessage, expectedErrorMessage) { + t.Fatalf("expected error message to contain '%s', but got '%s'", expectedErrorMessage, errorMessage) + } + }) +} diff --git a/cmd/parse.go b/cmd/parse.go index ef529c8ea7..700794d550 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" pr "github.com/open-policy-agent/opa/internal/presentation" "github.com/open-policy-agent/opa/loader" "github.com/open-policy-agent/opa/util" @@ -72,9 +73,10 @@ func parse(args []string, params *parseParams, stdout io.Writer, stderr io.Write parserOpts := ast.ParserOptions{ProcessAnnotation: true} if exposeLocation { - parserOpts.JSONOptions = &ast.JSONOptions{ - MarshalOptions: ast.JSONMarshalOptions{ - IncludeLocation: ast.NodeToggle{ + parserOpts.JSONOptions = &astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocationText: true, + IncludeLocation: astJSON.NodeToggle{ Term: true, Package: true, Comment: true, diff --git a/cmd/parse_test.go b/cmd/parse_test.go index a5f62009d5..b5371fbce4 100644 --- a/cmd/parse_test.go +++ b/cmd/parse_test.go @@ -129,9 +129,9 @@ func TestParseJSONOutputWithLocations(t *testing.T) { files := map[string]string{ "x.rego": `package x - - p = 1 - `, + +p = 1 +`, } errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), @@ -149,14 +149,16 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 1 + "col": 1, + "text": "cGFja2FnZQ==" }, "path": [ { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "var", "value": "data" @@ -165,7 +167,8 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "string", "value": "x" @@ -180,13 +183,15 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 5, + "text": "MQ==" }, "terms": { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 5, + "text": "MQ==" }, "type": "boolean", "value": true @@ -199,13 +204,20 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 5, + "text": "MQ==" }, "type": "number", "value": 1 }, "ref": [ { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 1, + "text": "cA==" + }, "type": "var", "value": "p" } @@ -213,36 +225,276 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "cCA9IDE=" } }, "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "cCA9IDE=" } } ] } `, "TEMPDIR", tempDirPath, -1) + gotLines := strings.Split(string(stdout), "\n") + wantLines := strings.Split(expectedOutput, "\n") + min := len(gotLines) + if len(wantLines) < min { + min = len(wantLines) + } + + for i := 0; i < min; i++ { + if gotLines[i] != wantLines[i] { + t.Fatalf("Expected line %d to be\n%v\n, got\n%v", i, wantLines[i], gotLines[i]) + } + } + + if len(gotLines) != len(wantLines) { + t.Fatalf("Expected %d lines, got %d", len(wantLines), len(gotLines)) + } +} + +func TestParseRefsJSONOutput(t *testing.T) { + + files := map[string]string{ + "x.rego": `package x + + a.b.c := true + `, + } + errc, stdout, stderr, _ := testParse(t, files, &parseParams{ + format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), + }) + if errc != 0 { + t.Fatalf("Expected exit code 0, got %v", errc) + } + if len(stderr) > 0 { + t.Fatalf("Expected no stderr output, got:\n%s\n", string(stderr)) + } + + expectedOutput := `{ + "package": { + "path": [ + { + "type": "var", + "value": "data" + }, + { + "type": "string", + "value": "x" + } + ] + }, + "rules": [ + { + "body": [ + { + "index": 0, + "terms": { + "type": "boolean", + "value": true + } + } + ], + "head": { + "value": { + "type": "boolean", + "value": true + }, + "assign": true, + "ref": [ + { + "type": "var", + "value": "a" + }, + { + "type": "string", + "value": "b" + }, + { + "type": "string", + "value": "c" + } + ] + } + } + ] +} +` + if got, want := string(stdout), expectedOutput; got != want { t.Fatalf("Expected output\n%v\n, got\n%v", want, got) } } -func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { +func TestParseRefsJSONOutputWithLocations(t *testing.T) { files := map[string]string{ "x.rego": `package x - default allow = false - allow = true { - input.method == "GET" - input.path = ["getUser", user] - input.user == user +a.b.c := true +`, + } + errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ + format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), + jsonInclude: "locations", + }) + if errc != 0 { + t.Fatalf("Expected exit code 0, got %v", errc) + } + if len(stderr) > 0 { + t.Fatalf("Expected no stderr output, got:\n%s\n", string(stderr)) + } + + expectedOutput := strings.Replace(`{ + "package": { + "location": { + "file": "TEMPDIR/x.rego", + "row": 1, + "col": 1, + "text": "cGFja2FnZQ==" + }, + "path": [ + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 1, + "col": 9, + "text": "eA==" + }, + "type": "var", + "value": "data" + }, + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 1, + "col": 9, + "text": "eA==" + }, + "type": "string", + "value": "x" + } + ] + }, + "rules": [ + { + "body": [ + { + "index": 0, + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 10, + "text": "dHJ1ZQ==" + }, + "terms": { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 10, + "text": "dHJ1ZQ==" + }, + "type": "boolean", + "value": true + } + } + ], + "head": { + "value": { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 10, + "text": "dHJ1ZQ==" + }, + "type": "boolean", + "value": true + }, + "assign": true, + "ref": [ + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 1, + "text": "YQ==" + }, + "type": "var", + "value": "a" + }, + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 3, + "text": "Yg==" + }, + "type": "string", + "value": "b" + }, + { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 5, + "text": "Yw==" + }, + "type": "string", + "value": "c" + } + ], + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 1, + "text": "YS5iLmMgOj0gdHJ1ZQ==" + } + }, + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 1, + "text": "YS5iLmMgOj0gdHJ1ZQ==" + } } - `, + ] +} +`, "TEMPDIR", tempDirPath, -1) + + gotLines := strings.Split(string(stdout), "\n") + wantLines := strings.Split(expectedOutput, "\n") + min := len(gotLines) + if len(wantLines) < min { + min = len(wantLines) + } + + for i := 0; i < min; i++ { + if gotLines[i] != wantLines[i] { + t.Fatalf("Expected line %d to be\n%v\n, got\n%v", i, wantLines[i], gotLines[i]) + } + } + + if len(gotLines) != len(wantLines) { + t.Fatalf("Expected %d lines, got %d", len(wantLines), len(gotLines)) + } +} +func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { + + files := map[string]string{ + "x.rego": `package x + +default allow = false +allow = true { + input.method == "GET" + input.path = ["getUser", user] + input.user == user +} +`, } errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), @@ -260,14 +512,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 1 + "col": 1, + "text": "cGFja2FnZQ==" }, "path": [ { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "var", "value": "data" @@ -276,7 +530,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "string", "value": "x" @@ -291,13 +546,15 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "ZGVmYXVsdA==" }, "terms": { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "ZGVmYXVsdA==" }, "type": "boolean", "value": true @@ -311,13 +568,20 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 19 + "col": 17, + "text": "ZmFsc2U=" }, "type": "boolean", "value": false }, "ref": [ { + "location": { + "file": "TEMPDIR/x.rego", + "row": 3, + "col": 9, + "text": "YWxsb3c=" + }, "type": "var", "value": "allow" } @@ -325,13 +589,15 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 11 + "col": 9, + "text": "YWxsb3cgPSBmYWxzZQ==" } }, "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "ZGVmYXVsdA==" } }, { @@ -341,14 +607,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 7 + "col": 3, + "text": "aW5wdXQubWV0aG9kID09ICJHRVQi" }, "terms": [ { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 20 + "col": 16, + "text": "PT0=" }, "type": "ref", "value": [ @@ -356,7 +624,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 20 + "col": 16, + "text": "PT0=" }, "type": "var", "value": "equal" @@ -367,7 +636,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 7 + "col": 3, + "text": "aW5wdXQubWV0aG9k" }, "type": "ref", "value": [ @@ -375,7 +645,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 7 + "col": 3, + "text": "aW5wdXQ=" }, "type": "var", "value": "input" @@ -384,7 +655,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 13 + "col": 9, + "text": "bWV0aG9k" }, "type": "string", "value": "method" @@ -395,7 +667,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 23 + "col": 19, + "text": "IkdFVCI=" }, "type": "string", "value": "GET" @@ -407,14 +680,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 7 + "col": 3, + "text": "aW5wdXQucGF0aCA9IFsiZ2V0VXNlciIsIHVzZXJd" }, "terms": [ { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 18 + "col": 14, + "text": "PQ==" }, "type": "ref", "value": [ @@ -422,7 +697,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 18 + "col": 14, + "text": "PQ==" }, "type": "var", "value": "eq" @@ -433,7 +709,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 7 + "col": 3, + "text": "aW5wdXQucGF0aA==" }, "type": "ref", "value": [ @@ -441,7 +718,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 7 + "col": 3, + "text": "aW5wdXQ=" }, "type": "var", "value": "input" @@ -450,7 +728,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 13 + "col": 9, + "text": "cGF0aA==" }, "type": "string", "value": "path" @@ -461,7 +740,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 20 + "col": 16, + "text": "WyJnZXRVc2VyIiwgdXNlcl0=" }, "type": "array", "value": [ @@ -469,7 +749,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 21 + "col": 17, + "text": "ImdldFVzZXIi" }, "type": "string", "value": "getUser" @@ -478,7 +759,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 32 + "col": 28, + "text": "dXNlcg==" }, "type": "var", "value": "user" @@ -492,14 +774,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 7 + "col": 3, + "text": "aW5wdXQudXNlciA9PSB1c2Vy" }, "terms": [ { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 18 + "col": 14, + "text": "PT0=" }, "type": "ref", "value": [ @@ -507,7 +791,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 18 + "col": 14, + "text": "PT0=" }, "type": "var", "value": "equal" @@ -518,7 +803,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 7 + "col": 3, + "text": "aW5wdXQudXNlcg==" }, "type": "ref", "value": [ @@ -526,7 +812,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 7 + "col": 3, + "text": "aW5wdXQ=" }, "type": "var", "value": "input" @@ -535,7 +822,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 13 + "col": 9, + "text": "dXNlcg==" }, "type": "string", "value": "user" @@ -546,7 +834,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 21 + "col": 17, + "text": "dXNlcg==" }, "type": "var", "value": "user" @@ -560,13 +849,20 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 13 + "col": 9, + "text": "dHJ1ZQ==" }, "type": "boolean", "value": true }, "ref": [ { + "location": { + "file": "TEMPDIR/x.rego", + "row": 4, + "col": 1, + "text": "YWxsb3c=" + }, "type": "var", "value": "allow" } @@ -574,21 +870,36 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 5 + "col": 1, + "text": "YWxsb3cgPSB0cnVl" } }, "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 5 + "col": 1, + "text": "YWxsb3cgPSB0cnVlIHsKICBpbnB1dC5tZXRob2QgPT0gIkdFVCIKICBpbnB1dC5wYXRoID0gWyJnZXRVc2VyIiwgdXNlcl0KICBpbnB1dC51c2VyID09IHVzZXIKfQ==" } } ] } `, "TEMPDIR", tempDirPath, -1) - if got, want := string(stdout), expectedOutput; got != want { - t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + gotLines := strings.Split(string(stdout), "\n") + wantLines := strings.Split(expectedOutput, "\n") + min := len(gotLines) + if len(wantLines) < min { + min = len(wantLines) + } + + for i := 0; i < min; i++ { + if gotLines[i] != wantLines[i] { + t.Fatalf("Expected line %d to be\n%v\n, got\n%v", i, wantLines[i], gotLines[i]) + } + } + + if len(gotLines) != len(wantLines) { + t.Fatalf("Expected %d lines, got %d", len(wantLines), len(gotLines)) } } diff --git a/compile/compile_bench_test.go b/compile/compile_bench_test.go index a26773e98f..079f4d41ad 100644 --- a/compile/compile_bench_test.go +++ b/compile/compile_bench_test.go @@ -3,6 +3,7 @@ package compile import ( "context" "fmt" + "io/fs" "strings" "testing" @@ -22,11 +23,11 @@ func BenchmarkCompileDynamicPolicy(b *testing.B) { for _, n := range numPolicies { testcase := generateDynamicPolicyBenchmarkData(n) - b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { - test.WithTempFS(testcase, func(root string) { - b.ResetTimer() - + test.WithTestFS(testcase, true, func(root string, fileSys fs.FS) { + b.ResetTimer() + b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { compiler := New(). + WithFS(fileSys). WithPaths(root) err := compiler.Build(context.Background()) diff --git a/docs/content/_index.md b/docs/content/_index.md index 6cce6e1a7f..362d7c5bef 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -60,7 +60,7 @@ There are three kinds of components in the system: * Networks connect servers and can be public or private. Public networks are connected to the Internet. * Ports attach servers to networks. -All of the servers, networks, and ports are provisioned by a script. The script +All the servers, networks, and ports are provisioned by a script. The script receives a JSON representation of the system as input: ```live:example:input @@ -724,6 +724,11 @@ shell_accessible ```live:example/logical_or/partial_set:output ``` +{{< info >}} +💡 there's a [blog post](https://www.styra.com/blog/how-to-express-or-in-rego/) that goes into much more detail +on this topic showing different methods express OR in idiomatic Rego for different use cases. +{{< /info >}} + ### Putting It Together diff --git a/docs/website/content/integrations/enterprise-contract.md b/docs/website/content/integrations/enterprise-contract.md new file mode 100644 index 0000000000..804d1deb59 --- /dev/null +++ b/docs/website/content/integrations/enterprise-contract.md @@ -0,0 +1,22 @@ +--- +title: Enterprise Contract +software: +- enterprise-contract +inventors: +- enterprise-contract +labels: + category: security + layer: application +code: +- https://github.com/enterprise-contract/ec-cli +blogs: +- https://enterprisecontract.dev/posts/ +docs_features: + go-integration: + note: | + The [Enterprise Contract](https://enterprisecontract.dev/) uses the OPA go + library to process rego policies when validating the signatures and + attestations of container images and other software artifacts. +--- +Securely verify supply chain artifacts, and enforce policies about how they were built and tested, +in a manageable, scalable, and declarative way. diff --git a/docs/website/content/integrations/expressing-or.md b/docs/website/content/integrations/expressing-or.md new file mode 100644 index 0000000000..edd8b1b2a3 --- /dev/null +++ b/docs/website/content/integrations/expressing-or.md @@ -0,0 +1,19 @@ +--- +title: Express OR in Rego +subtitle: Idiomatic Rego Examples +labels: + category: learning + layer: web +inventors: +- styra +blogs: +- http://styra.com/blog/how-to-express-or-in-rego +docs_features: + learning-rego: + note: | + This [learning material](http://styra.com/blog/how-to-express-or-in-rego) + is a great way to learn how to migrate logic from other languages to Rego. +--- + +This [learning material](http://styra.com/blog/how-to-express-or-in-rego) +is a great way to learn how to migrate logic from other languages to Rego. diff --git a/docs/website/content/integrations/regocpp.md b/docs/website/content/integrations/regocpp.md new file mode 100644 index 0000000000..9fcf9c5ef8 --- /dev/null +++ b/docs/website/content/integrations/regocpp.md @@ -0,0 +1,13 @@ +--- +title: regocpp +software: +- regopy +- regorust +inventors: +- microsoft +tutorials: +- https://microsoft.github.io/rego-cpp/ +code: +- https://github.com/microsoft/rego-cpp +--- +The rego-cpp project is a Rego compiler and runtime implemented in C++. As such, it provides the ability to integrate Rego natively into a wider range of languages. We currrently support C, C++, Rust, and Python, and are largely compatible with v0.55.0 of the language. \ No newline at end of file diff --git a/docs/website/content/organizations/enterprise-contract.md b/docs/website/content/organizations/enterprise-contract.md new file mode 100644 index 0000000000..684bde7b93 --- /dev/null +++ b/docs/website/content/organizations/enterprise-contract.md @@ -0,0 +1,4 @@ +--- +link: https://enterprisecontract.dev/ +title: Enterprise Contract +--- diff --git a/docs/website/content/softwares/enterprise-contract.md b/docs/website/content/softwares/enterprise-contract.md new file mode 100644 index 0000000000..0fce1f4ed9 --- /dev/null +++ b/docs/website/content/softwares/enterprise-contract.md @@ -0,0 +1,4 @@ +--- +link: https://github.com/enterprise-contract/ +title: Enterprise Contract +--- diff --git a/docs/website/content/softwares/regopy.md b/docs/website/content/softwares/regopy.md new file mode 100644 index 0000000000..17c2f9168b --- /dev/null +++ b/docs/website/content/softwares/regopy.md @@ -0,0 +1,4 @@ +--- +link: https://pypi.org/project/regopy/ +title: regopy +--- diff --git a/docs/website/content/softwares/regorust.md b/docs/website/content/softwares/regorust.md new file mode 100644 index 0000000000..72e4b6a0db --- /dev/null +++ b/docs/website/content/softwares/regorust.md @@ -0,0 +1,4 @@ +--- +link: https://crates.io/crates/regorust +title: regorust +--- diff --git a/docs/website/layouts/support/section.html.html b/docs/website/layouts/support/section.html.html index d7d87e3b19..f3b24b1e8a 100644 --- a/docs/website/layouts/support/section.html.html +++ b/docs/website/layouts/support/section.html.html @@ -17,27 +17,31 @@

-
+
- Styra is the original creator of Open Policy - Agent and provides support, professional services, training, and + + Styra + is the original creator of Open Policy Agent and provides + support, professional services, training, and enterprise products.
-
+
- Policy-as-Code Laboratories provides strategic planning - and integration consulting for OPA and Rego across the PaC ecosystem - (Cloud, Kubernetes, Open Shift and legacy platforms) + + Policy-as-Code Laboratories + provides strategic planning + and integration consulting for OPA and Rego across the PaC ecosystem + (Cloud, Kubernetes, Open Shift and legacy platforms)
diff --git a/docs/website/static/img/logos/integrations/enterprise-contract.svg b/docs/website/static/img/logos/integrations/enterprise-contract.svg new file mode 100644 index 0000000000..a01843cc52 --- /dev/null +++ b/docs/website/static/img/logos/integrations/enterprise-contract.svg @@ -0,0 +1,173 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/website/static/img/logos/integrations/expressing-or.png b/docs/website/static/img/logos/integrations/expressing-or.png new file mode 100644 index 0000000000..0e06cd1536 Binary files /dev/null and b/docs/website/static/img/logos/integrations/expressing-or.png differ diff --git a/docs/website/static/img/logos/integrations/regocpp.svg b/docs/website/static/img/logos/integrations/regocpp.svg new file mode 100644 index 0000000000..4b90785e9d --- /dev/null +++ b/docs/website/static/img/logos/integrations/regocpp.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/format/format_test.go b/format/format_test.go index 0c545fb557..8d363210ff 100644 --- a/format/format_test.go +++ b/format/format_test.go @@ -78,8 +78,6 @@ func TestFormatSourceError(t *testing.T) { } func TestFormatSource(t *testing.T) { - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - regoFiles, err := filepath.Glob("testfiles/*.rego") if err != nil { panic(err) diff --git a/go.mod b/go.mod index 84ac226eb4..c0811e8fba 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/agnivade/levenshtein v1.1.1 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 - github.com/containerd/containerd v1.7.4 + github.com/containerd/containerd v1.7.6 github.com/dgraph-io/badger/v3 v3.2103.5 github.com/fortytw2/leaktest v1.3.0 github.com/foxcpp/go-mockdns v1.0.0 @@ -40,7 +40,7 @@ require ( go.uber.org/automaxprocs v1.5.3 golang.org/x/net v0.15.0 golang.org/x/time v0.3.0 - google.golang.org/grpc v1.58.0 + google.golang.org/grpc v1.58.2 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v2 v2.4.0 oras.land/oras-go/v2 v2.3.0 @@ -49,6 +49,7 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect diff --git a/go.sum b/go.sum index 525687a33d..0d579bc1d6 100644 --- a/go.sum +++ b/go.sum @@ -36,7 +36,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= +github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= +github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -72,9 +73,9 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/containerd v1.7.4 h1:Q5lwCrO44ahHhO65rXthXkfJUG5W78LXwK9gTt8XFfU= -github.com/containerd/containerd v1.7.4/go.mod h1:gq7JDNtCrI1Zlcc572a9tvP1f1Ja8VBxiB9J00apAtU= -github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 h1:a5jOuoZHKBi2oH9JsfNqrrPpHhmrYU0NAte3M/EPudw= +github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= +github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= +github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -604,8 +605,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= -google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/bundle/inspect/inspect.go b/internal/bundle/inspect/inspect.go index f6d61d54e3..da2dca40d9 100644 --- a/internal/bundle/inspect/inspect.go +++ b/internal/bundle/inspect/inspect.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/bundle" initload "github.com/open-policy-agent/opa/internal/runtime/init" "github.com/open-policy-agent/opa/loader" @@ -32,9 +33,9 @@ func File(path string, includeAnnotations bool) (*Info, error) { b, err := loader.NewFileLoader(). WithSkipBundleVerification(true). WithProcessAnnotation(true). // Always process annotations, for enriching namespace listing - WithJSONOptions(&ast.JSONOptions{ - MarshalOptions: ast.JSONMarshalOptions{ - IncludeLocation: ast.NodeToggle{ + WithJSONOptions(&json.Options{ + MarshalOptions: json.MarshalOptions{ + IncludeLocation: json.NodeToggle{ // Annotation location data is only included if includeAnnotations is set AnnotationsRef: includeAnnotations, }, diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 9c3fd7332d..61addb2efd 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -25,6 +25,8 @@ type QuerySet struct { } type planiter func() error +type planLocalIter func(ir.Local) error +type stmtFactory func(ir.Local) ir.Stmt // Planner implements a query planner for Rego queries. type Planner struct { @@ -147,32 +149,31 @@ func (p *Planner) buildFunctrie() error { } for _, rule := range module.Rules { - r := rule.Ref() - switch r[len(r)-1].Value.(type) { - case ast.String: // pass - default: // cut off - r = r[:len(r)-1] - } + r := rule.Ref().StringPrefix() val := p.rules.LookupOrInsert(r) + + val.rules = val.DescendantRules() val.rules = append(val.rules, rule) + val.children = nil } } return nil } func (p *Planner) planRules(rules []*ast.Rule) (string, error) { + // We know the rules with closer to the root (shorter static path) are ordered first. pathRef := rules[0].Ref() // figure out what our rules' collective name/path is: // if we're planning both p.q.r and p.q[s], we'll name // the function p.q (for the mapping table) - // TODO(sr): this has to change when allowing `p[v].q.r[w]` ref rules - // including the mapping lookup structure and lookup functions pieces := len(pathRef) for i := range rules { r := rules[i].Ref() - if _, ok := r[len(r)-1].Value.(ast.String); !ok { - pieces = len(r) - 1 + for j, t := range r { + if _, ok := t.Value.(ast.String); !ok && j > 0 && j < pieces { + pieces = j + } } } // control if p.a = 1 is to return 1 directly; or insert 1 under key "a" into an object @@ -236,7 +237,11 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) { fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeObjectStmt{Target: fn.Return})) } case ast.MultiValue: - fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeSetStmt{Target: fn.Return})) + if buildObject { + fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeObjectStmt{Target: fn.Return})) + } else { + fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeSetStmt{Target: fn.Return})) + } } // For complete document rules, allocate one local variable for output @@ -252,6 +257,12 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) { var defaultRule *ast.Rule var ruleLoc *location.Location + // We sort rules by ref length, to ensure that when merged, we can detect conflicts when one + // rule attempts to override values (deep and shallow) defined by another rule. + sort.Slice(rules, func(i, j int) bool { + return len(rules[i].Ref()) > len(rules[j].Ref()) + }) + // Generate function blocks for rules. for i := range rules { @@ -320,18 +331,19 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) { switch rule.Head.RuleKind() { case ast.SingleValue: if buildObject { - ref := rule.Head.Ref() - last := ref[len(ref)-1] - return p.planTerm(last, func() error { - key := p.ltarget - return p.planTerm(rule.Head.Value, func() error { - value := p.ltarget - p.appendStmt(&ir.ObjectInsertOnceStmt{ - Object: fn.Return, - Key: key, - Value: value, + ref := rule.Ref() + return p.planTerm(rule.Head.Value, func() error { + value := p.ltarget + return p.planNestedObjects(fn.Return, ref[pieces:len(ref)-1], func(obj ir.Local) error { + return p.planTerm(ref[len(ref)-1], func() error { + key := p.ltarget + p.appendStmt(&ir.ObjectInsertOnceStmt{ + Object: obj, + Key: key, + Value: value, + }) + return nil }) - return nil }) }) } @@ -343,6 +355,28 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) { return nil }) case ast.MultiValue: + if buildObject { + ref := rule.Ref() + // we drop the trailing set key from the ref + return p.planNestedObjects(fn.Return, ref[pieces:len(ref)-1], func(obj ir.Local) error { + // Last term on rule ref is the key an which the set is assigned in the deepest nested object + return p.planTerm(ref[len(ref)-1], func() error { + key := p.ltarget + return p.planTerm(rule.Head.Key, func() error { + value := p.ltarget + factory := func(v ir.Local) ir.Stmt { return &ir.MakeSetStmt{Target: v} } + return p.planDotOr(obj, key, factory, func(set ir.Local) error { + p.appendStmt(&ir.SetAddStmt{ + Set: set, + Value: value, + }) + p.appendStmt(&ir.ObjectInsertStmt{Key: key, Value: op(set), Object: obj}) + return nil + }) + }) + }) + }) + } return p.planTerm(rule.Head.Key, func() error { p.appendStmt(&ir.SetAddStmt{ Set: fn.Return, @@ -422,6 +456,59 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) { return fn.Name, nil } +func (p *Planner) planDotOr(obj ir.Local, key ir.Operand, or stmtFactory, iter planLocalIter) error { + // We're constructing the following plan: + // + // | block a + // | | block b + // | | | dot &{Source:Local Key:{Value:Local} Target:Local} + // | | | break 1 + // | | or &{Target:Local} + // | | *ir.ObjectInsertOnceStmt &{Key:{Value:Local} Value:{Value:Local} Object:Local} + + prev := p.curr + dotBlock := &ir.Block{} + p.curr = dotBlock + + val := p.newLocal() + p.appendStmt(&ir.DotStmt{ + Source: op(obj), + Key: key, + Target: val, + }) + p.appendStmt(&ir.BreakStmt{Index: 1}) + + outerBlock := &ir.Block{ + Stmts: []ir.Stmt{ + &ir.BlockStmt{Blocks: []*ir.Block{dotBlock}}, // FIXME: Set Location + or(val), + &ir.ObjectInsertOnceStmt{Key: key, Value: op(val), Object: obj}, + }, + } + + p.curr = prev + p.appendStmt(&ir.BlockStmt{Blocks: []*ir.Block{outerBlock}}) + return iter(val) +} + +func (p *Planner) planNestedObjects(obj ir.Local, ref ast.Ref, iter planLocalIter) error { + if len(ref) == 0 { + //return fmt.Errorf("nested object construction didn't create object") + return iter(obj) + } + + t := ref[0] + + return p.planTerm(t, func() error { + key := p.ltarget + + factory := func(v ir.Local) ir.Stmt { return &ir.MakeObjectStmt{Target: v} } + return p.planDotOr(obj, key, factory, func(childObj ir.Local) error { + return p.planNestedObjects(childObj, ref[1:], iter) + }) + }) +} + func (p *Planner) planFuncParams(params []ir.Local, args ast.Args, idx int, iter planiter) error { if idx >= len(args) { return iter() diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index cb22d67448..a6ff37bba3 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -16,7 +16,6 @@ import ( ) func TestPlannerHelloWorld(t *testing.T) { - // NOTE(tsandall): These tests are not meant to give comprehensive coverage // of the planner. Currently we have a suite of end-to-end tests in the // test/wasm/ directory that are specified in YAML, compiled into Wasm, and @@ -151,6 +150,46 @@ func TestPlannerHelloWorld(t *testing.T) { p[v] = 2 { v := "b" } `}, }, + { + note: "partial object (ref-head) with var", + queries: []string{`data.test.p.q = x`}, + modules: []string{` + package test + p.q.r["a"] = 1 + p.q[v] = 2 { v := "b" } + `}, + }, + { + note: "partial object (ref-head) with var (shallow query)", + queries: []string{`data.test.p = x`}, + modules: []string{` + package test + p.q["a"] = 1 + p.q[v] = 2 { v := "b" } + p.r["c"] = 3 + p.r[v] = 4 { v := "d" } + `}, + }, + { + note: "partial object (ref-head) with var (multiple)", + queries: []string{`data.test.p.q = x`}, + modules: []string{` + package test + p.q["a"] = 1 + p.q[v] = x { l1 := ["b", "c", "d"]; l2 := ["foo", "bar"]; l3 := [2, 3]; v := l1[_]; x := l2[_]; z := l3[_] } + `}, + }, + { + note: "partial object (general ref-head) with var", + queries: []string{`data.test.p.q = x`}, + modules: []string{` + package test + p.q["a"] = 1 + p.q.b.s.baz = 2 + p.q.b.s.foo.c = 3 + p.q[r].s[t].u = v { x := ["foo", "bar"]; r := "b"; t := x[v]} + `}, + }, { note: "every", queries: []string{`data.test.p`}, diff --git a/internal/planner/rules.go b/internal/planner/rules.go index 06fa6d158d..f5d6f3fc6c 100644 --- a/internal/planner/rules.go +++ b/internal/planner/rules.go @@ -98,6 +98,7 @@ func (t *ruletrie) Rules() []*ast.Rule { // // and we're retrieving a.b, we want Rules() to include the rule body // of a.b.c. + // FIXME: We need to go deeper than just immediate children (?) for _, rs := range t.children { if r := rs[len(rs)-1].rules; r != nil { rules = append(rules, r...) @@ -157,13 +158,46 @@ func (t *ruletrie) Lookup(key ast.Ref) *ruletrie { return node } +func (t *ruletrie) LookupShallowest(key ast.Ref) *ruletrie { + node := t + for _, elem := range key { + node = node.Get(elem.Value) + if node == nil { + return nil + } + if len(node.rules) > 0 { + return node + } + } + return node +} + +// TODO: Collapse rules with overlapping extent to same node(?) func (t *ruletrie) LookupOrInsert(key ast.Ref) *ruletrie { - if val := t.Lookup(key); val != nil { + if val := t.LookupShallowest(key); val != nil { + return val } return t.Insert(key) } +func (t *ruletrie) DescendantRules() []*ast.Rule { + if len(t.children) == 0 { + return t.rules + } + + rules := make([]*ast.Rule, len(t.rules), len(t.rules)+len(t.children)) // could be too little + copy(rules, t.rules) + + for _, cs := range t.children { + for _, c := range cs { + rules = append(rules, c.DescendantRules()...) + } + } + + return rules +} + func (t *ruletrie) ChildrenCount() int { return len(t.children) } diff --git a/internal/wasm/sdk/test/e2e/exceptions.yaml b/internal/wasm/sdk/test/e2e/exceptions.yaml index 951a438bd4..664e29b20f 100644 --- a/internal/wasm/sdk/test/e2e/exceptions.yaml +++ b/internal/wasm/sdk/test/e2e/exceptions.yaml @@ -2,14 +2,4 @@ "data/toplevel integer": "https://github.com/open-policy-agent/opa/issues/3711" "data/nested integer": "https://github.com/open-policy-agent/opa/issues/3711" "withkeyword/function: indirect call, arity 1, replacement is value that needs eval (array comprehension)": "https://github.com/open-policy-agent/opa/issues/5311" -"withkeyword/builtin: indirect call, arity 1, replacement is value that needs eval (array comprehension)": "https://github.com/open-policy-agent/opa/issues/5311" -"refheads/general, single var": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, multiple vars": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, deep query": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, overlapping rule, no conflict": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, overlapping rule, conflict": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, set leaf": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, set leaf, deep query": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, input var": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, external non-ground var": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" -"refheads/general, multiple result-set entries": "Tests with arbitrary vars in rule refs (general refs) are not supported by the planner yet" \ No newline at end of file +"withkeyword/builtin: indirect call, arity 1, replacement is value that needs eval (array comprehension)": "https://github.com/open-policy-agent/opa/issues/5311" \ No newline at end of file diff --git a/internal/wasm/sdk/test/e2e/external_test.go b/internal/wasm/sdk/test/e2e/external_test.go index bdd98526f0..487ee5ae70 100644 --- a/internal/wasm/sdk/test/e2e/external_test.go +++ b/internal/wasm/sdk/test/e2e/external_test.go @@ -65,6 +65,10 @@ func TestWasmE2E(t *testing.T) { t.SkipNow() } + for k, v := range tc.Env { + t.Setenv(k, v) + } + opts := []func(*rego.Rego){ rego.Query(tc.Query), } diff --git a/loader/loader.go b/loader/loader.go index df45259df4..4583f21b44 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -18,6 +18,7 @@ import ( "sigs.k8s.io/yaml" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/bundle" fileurl "github.com/open-policy-agent/opa/internal/file/url" "github.com/open-policy-agent/opa/internal/merge" @@ -100,7 +101,7 @@ type FileLoader interface { WithSkipBundleVerification(bool) FileLoader WithProcessAnnotation(bool) FileLoader WithCapabilities(*ast.Capabilities) FileLoader - WithJSONOptions(*ast.JSONOptions) FileLoader + WithJSONOptions(*astJSON.Options) FileLoader } // NewFileLoader returns a new FileLoader instance. @@ -175,7 +176,7 @@ func (fl *fileLoader) WithCapabilities(caps *ast.Capabilities) FileLoader { } // WithJSONOptions sets the JSONOptions for use when parsing files -func (fl *fileLoader) WithJSONOptions(opts *ast.JSONOptions) FileLoader { +func (fl *fileLoader) WithJSONOptions(opts *astJSON.Options) FileLoader { fl.opts.JSONOptions = opts return fl } diff --git a/loader/loader_test.go b/loader/loader_test.go index 64c7cf9b90..323ba96b30 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/bundle" "github.com/open-policy-agent/opa/util" "github.com/open-policy-agent/opa/util/test" @@ -655,9 +656,9 @@ func TestLoadWithJSONOptions(t *testing.T) { } // load the file with JSON options set to include location data - loaded, err := NewFileLoader().WithFS(fsys).WithJSONOptions(&ast.JSONOptions{ - MarshalOptions: ast.JSONMarshalOptions{ - IncludeLocation: ast.NodeToggle{ + loaded, err := NewFileLoader().WithFS(fsys).WithJSONOptions(&astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Package: true, }, }, diff --git a/plugins/rest/auth.go b/plugins/rest/auth.go index 567900e8b4..f69496f08b 100644 --- a/plugins/rest/auth.go +++ b/plugins/rest/auth.go @@ -721,7 +721,43 @@ func (acs *awsCredentialServiceChain) addService(service awsCredentialService) { acs.awsCredentialServices = append(acs.awsCredentialServices, service) } +type awsCredentialCheckErrors []*awsCredentialCheckError + +func (e awsCredentialCheckErrors) Error() string { + + if len(e) == 0 { + return "no error(s)" + } + + if len(e) == 1 { + return fmt.Sprintf("1 error occurred: %v", e[0].Error()) + } + + s := make([]string, len(e)) + for i, err := range e { + s[i] = err.Error() + } + + return fmt.Sprintf("%d errors occurred:\n%s", len(e), strings.Join(s, "\n")) +} + +type awsCredentialCheckError struct { + message string +} + +func newAWSCredentialError(message string) *awsCredentialCheckError { + return &awsCredentialCheckError{ + message: message, + } +} + +func (e *awsCredentialCheckError) Error() string { + return e.message +} + func (acs *awsCredentialServiceChain) credentials(ctx context.Context) (aws.Credentials, error) { + var errs awsCredentialCheckErrors + for _, service := range acs.awsCredentialServices { credential, err := service.credentials(ctx) if err != nil { @@ -731,6 +767,7 @@ func (acs *awsCredentialServiceChain) credentials(ctx context.Context) (aws.Cred return aws.Credentials{}, err } + errs = append(errs, newAWSCredentialError(err.Error())) continue } @@ -738,7 +775,7 @@ func (acs *awsCredentialServiceChain) credentials(ctx context.Context) (aws.Cred return credential, nil } - return aws.Credentials{}, errors.New("all AWS credential providers failed") + return aws.Credentials{}, fmt.Errorf("all AWS credential providers failed: %v", errs) } func (ap *awsSigningAuthPlugin) awsCredentialService() awsCredentialService { diff --git a/plugins/rest/rest_test.go b/plugins/rest/rest_test.go index a6c5f96a8b..26210cfe1a 100644 --- a/plugins/rest/rest_test.go +++ b/plugins/rest/rest_test.go @@ -1789,6 +1789,7 @@ func TestAWSCredentialServiceChain(t *testing.T) { input string wantErr bool env map[string]string + errMsg string }{ { name: "Fallback to Environment Credential", @@ -1826,6 +1827,7 @@ func TestAWSCredentialServiceChain(t *testing.T) { } }`, wantErr: true, + errMsg: "all AWS credential providers failed: 4 errors occurred", env: map[string]string{}, }, } @@ -1859,10 +1861,19 @@ func TestAWSCredentialServiceChain(t *testing.T) { awsPlugin.logger = client.logger err = awsPlugin.Prepare(req) - if err != nil && !tc.wantErr { - t.Fatalf("Unexpected error: %v", err) - } else if err == nil && tc.wantErr { - t.Fatalf("Expected error for input %v", tc.input) + + if tc.wantErr { + if err == nil { + t.Fatalf("Expected error for input %v", tc.input) + } + + if !strings.Contains(err.Error(), tc.errMsg) { + t.Fatalf("Expected error message %v but got %v", tc.errMsg, err.Error()) + } + } else { + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } } }) } diff --git a/test/cases/testdata/partialobjectdoc/test-partialobjectdoc-ref.yaml b/test/cases/testdata/partialobjectdoc/test-partialobjectdoc-ref.yaml new file mode 100644 index 0000000000..0fee305bbf --- /dev/null +++ b/test/cases/testdata/partialobjectdoc/test-partialobjectdoc-ref.yaml @@ -0,0 +1,22 @@ +cases: +- note: partialobjectdoc/ref + modules: + - | + package generated + + p.q[k] = v { + k := ["foo", "bar"][v] + } + + p.baz := 2 + + q { + x := "bar" + y := "q" + p[y][x] == 1 + } + + query: data.generated.q = x + want_result: + - x: + true diff --git a/test/cases/testdata/refheads/test-generic-refs.yaml b/test/cases/testdata/refheads/test-generic-refs.yaml index 74fb221f6e..7fe16bbe0e 100644 --- a/test/cases/testdata/refheads/test-generic-refs.yaml +++ b/test/cases/testdata/refheads/test-generic-refs.yaml @@ -1,7 +1,5 @@ cases: - note: 'refheads/general, single var' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -17,8 +15,6 @@ cases: c: r: 2 - note: 'refheads/general, multiple vars' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -34,8 +30,6 @@ cases: c: 2: true - note: 'refheads/general, deep query' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -46,8 +40,6 @@ cases: - x: 1: true - note: 'refheads/general, overlapping rule, no conflict' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -63,20 +55,117 @@ cases: r: 1 c: r: 2 + - note: 'refheads/general, overlapping rule, different dynamic depths, no conflict' + modules: + - | + package test + + p[q].r.s := i { q := ["a", "b", "c"][i] } + p.d.r := 3 + p.e := 4 + query: data.test.p = x + want_result: + - x: + a: + r: + s: 0 + b: + r: + s: 1 + c: + r: + s: 2 + d: + r: 3 + e: 4 + - note: 'refheads/general, self-conflict' + modules: + - | + package test + + p[q].r.s := i { q := ["a", "b", "c", "b"][i] } + query: data.test.p = x + want_error_code: eval_conflict_error - note: 'refheads/general, overlapping rule, conflict' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test - p[q].r := i { q := ["a", "b", "c"][i] } + p[q].r.s := i { q := ["a", "b", "c"][i] } p.a.r := 42 query: data.test.p = x - want_error: eval_conflict_error + want_error_code: eval_conflict_error + - note: 'refheads/general, overlapping rule, deep override inside other rule object value, conflict' + modules: + - | + package test + + p[q].r := v { + q := "q"; + v := { + "s": { + "t" : 1 + } + } + } + + p.q.r.s.t := 42 + query: data.test.p = x + want_error_code: eval_conflict_error + - note: 'refheads/general, overlapping rule, deep injection into other rule object value, conflict' + modules: + - | + package test + + p[q].r := v { + q := "q"; + v := { + "s": { + "t" : 1 + } + } + } + + p.q.r.s.u := 42 + query: data.test.p = x + want_error_code: eval_conflict_error + - note: 'refheads/general, set leaf (shallow ref)' + modules: + - | + package test + import future.keywords + + p[q] contains r { + x := ["a", "b", "c"] + q := x[_] + r := x[_] + q != r + } + + p.b contains "foo" + query: data.test.p = x + want_result: + - x: + a: [ "b", "c" ] + b: [ "a", "c", "foo" ] + c: [ "a", "b" ] + - note: 'refheads/general, set leaf (other rule defines dynamic ref portion)' + modules: + - | + package test + import future.keywords + + p.q contains r { + r := ["a", "b", "c"][_] + } + + p[q] := r { q := "foo"; r := "bar" } + query: data.test.p = x + want_result: + - x: + q: [ "a", "b", "c" ] + foo: bar - note: 'refheads/general, set leaf' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -100,8 +189,6 @@ cases: c: r: [ "a", "b" ] - note: 'refheads/general, set leaf, deep query' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -119,8 +206,6 @@ cases: want_result: - x: "c" - note: 'refheads/general, input var' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -134,8 +219,6 @@ cases: bar: r: "foo" - note: 'refheads/general, external non-ground var' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test @@ -158,8 +241,6 @@ cases: bar: 1 baz: 2 - note: 'refheads/general, multiple result-set entries' - env: - EXPERIMENTAL_GENERAL_RULE_REFS: "true" modules: - | package test diff --git a/test/cases/testdata/refheads/test-refs-as-rule-heads.yaml b/test/cases/testdata/refheads/test-refs-as-rule-heads.yaml index f8d00b5a7e..ebe69d0027 100644 --- a/test/cases/testdata/refheads/test-refs-as-rule-heads.yaml +++ b/test/cases/testdata/refheads/test-refs-as-rule-heads.yaml @@ -25,6 +25,16 @@ cases: q: r: 1 s: 2 +- note: 'refheads/single-value, with var, conflict' + modules: + - | + package test + + p.q.r = 1 + p.q[s] = 2 { s := "r" } + query: data.test.p.q = x + want_error_code: eval_conflict_error + want_error: object keys must be unique - modules: - | package test diff --git a/topdown/eval_test.go b/topdown/eval_test.go index bd24f987fe..9b4e565ff9 100644 --- a/topdown/eval_test.go +++ b/topdown/eval_test.go @@ -247,9 +247,6 @@ func TestContainsNestedRefOrCall(t *testing.T) { } func TestTopdownVirtualCache(t *testing.T) { - // TODO: break out into separate tests - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - ctx := context.Background() store := inmem.New() @@ -604,8 +601,6 @@ func TestTopdownVirtualCache(t *testing.T) { } func TestPartialRule(t *testing.T) { - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - ctx := context.Background() store := inmem.New() @@ -1451,27 +1446,3 @@ func TestPartialRule(t *testing.T) { }) } } - -// TODO: Remove when general rule refs are enabled by default. -func TestGeneralRuleRefsFeatureFlag(t *testing.T) { - module := ast.MustParseModule(`package test - p[q].r { q := "q" }`) - mods := map[string]*ast.Module{ - "": module, - } - c := ast.NewCompiler() - c.Compile(mods) - - if !strings.Contains(c.Errors.Error(), "rego_type_error: rule head must only contain string terms (except for last)") { - t.Fatal("Expected error but got:", c.Errors) - } - - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - - c = ast.NewCompiler() - c.Compile(mods) - - if c.Errors != nil { - t.Fatal("Unexpected error:", c.Errors) - } -} diff --git a/topdown/topdown_partial_test.go b/topdown/topdown_partial_test.go index 126c7273a4..617800f58b 100644 --- a/topdown/topdown_partial_test.go +++ b/topdown/topdown_partial_test.go @@ -18,9 +18,6 @@ import ( ) func TestTopDownPartialEval(t *testing.T) { - // TODO: break out into separate tests - t.Setenv("EXPERIMENTAL_GENERAL_RULE_REFS", "true") - tests := []struct { note string unknowns []string diff --git a/types/types.go b/types/types.go index 96d5140d7c..a4b87cd554 100644 --- a/types/types.go +++ b/types/types.go @@ -214,7 +214,7 @@ func (t *Array) String() string { for _, tpe := range t.static { buf = append(buf, Sprint(tpe)) } - var repr = prefix + repr := prefix if len(buf) > 0 { repr += "<" + strings.Join(buf, ", ") + ">" } @@ -355,7 +355,7 @@ func (t *Object) String() string { for _, p := range t.static { buf = append(buf, fmt.Sprintf("%v: %v", p.Key, Sprint(p.Value))) } - var repr = prefix + repr := prefix if len(buf) > 0 { repr += "<" + strings.Join(buf, ", ") + ">" } @@ -412,7 +412,6 @@ func (t *Object) toMap() map[string]interface{} { // Select returns the type of the named property. func (t *Object) Select(name interface{}) Type { - pos := sort.Search(len(t.static), func(x int) bool { return util.Compare(t.static[x].Key, name) >= 0 }) @@ -566,22 +565,73 @@ func (t Any) Merge(other Type) Any { } // Union returns a new Any type that is the union of the two Any types. +// Note(philipc): The two Any slices MUST be sorted before running Union, +// or else this method will fail to merge the two slices correctly. func (t Any) Union(other Any) Any { - if len(t) == 0 { + lenT := len(t) + lenOther := len(other) + // Return the more general (blank) Any type if present. + if lenT == 0 { return t } - if len(other) == 0 { + if lenOther == 0 { return other } - cpy := make(Any, len(t)) - copy(cpy, t) - for i := range other { - if !cpy.Contains(other[i]) { - cpy = append(cpy, other[i]) + // Prealloc the output list. + maxLen := lenT + if lenT < lenOther { + maxLen = lenOther + } + merged := make(Any, 0, maxLen) + // Note(philipc): Create a merged slice, doing the minimum number of + // comparisons along the way. We treat this as a problem of merging two + // sorted lists that might have duplicates. This specifically saves us + // from cases where one list might be *much* longer than the other. + // Algorithm: + // Assume: + // - List A + // - List B + // - List Output + // - Idx_a, Idx_b + // Procedure: + // - While Idx_a < len(A) and Idx_b < len(B) + // - Compare head(A) and head(B) + // - Cases: + // - A < B: Append head(A) to Output, advance Idx_a + // - A == B: Append head(A) to Output, advance Idx_a, Idx_b + // - A > B: Append head(B) to Output, advance Idx_b + // - Return output + idxA := 0 + idxB := 0 + for idxA < lenT || idxB < lenOther { + // Early-exit cases: + if idxA == lenT { + // Ran out of elements in t. Copy over what's left from other. + merged = append(merged, other[idxB:]...) + break + } else if idxB == lenOther { + // Ran out of elements in other. Copy over what's left from t. + merged = append(merged, t[idxA:]...) + break + } + // Normal selection of next element to merge: + switch Compare(t[idxA], other[idxB]) { + // A < B: + case -1: + merged = append(merged, t[idxA]) + idxA++ + // A == B: + case 0: + merged = append(merged, t[idxA]) + idxA++ + idxB++ + // A > B: + case 1: + merged = append(merged, other[idxB]) + idxB++ } } - sort.Sort(typeSlice(cpy)) - return cpy + return merged } func (t Any) String() string { @@ -706,7 +756,6 @@ func (t *Function) MarshalJSON() ([]byte, error) { // UnmarshalJSON decodes the JSON serialized function declaration. func (t *Function) UnmarshalJSON(bs []byte) error { - tpe, err := Unmarshal(bs) if err != nil { return err diff --git a/types/types_bench_test.go b/types/types_bench_test.go index 21680a7ed1..c50c664cde 100644 --- a/types/types_bench_test.go +++ b/types/types_bench_test.go @@ -38,3 +38,57 @@ func generateType(n int) Type { } return NewObject(static, nil) } + +func generateTypeWithPrefix(n int, prefix string) Type { + static := make([]*StaticProperty, n) + for i := 0; i < n; i++ { + static[i] = NewStaticProperty(prefix+fmt.Sprint(i), S) + } + return NewObject(static, nil) +} + +func BenchmarkAnyMergeOne(b *testing.B) { + sizes := []int{100, 500, 1000, 5000, 10000} + for _, size := range sizes { + anyA := Any(make([]Type, 0, size)) + for i := 0; i < size; i++ { + tpe := generateType(i) + anyA = append(anyA, tpe) + } + tpeB := N + b.ResetTimer() + b.Run(fmt.Sprint(size), func(b *testing.B) { + result := anyA.Merge(tpeB) + if len(result) != len(anyA)+1 { + b.Fatalf("Expected length of merged result to be: %d, got: %d", len(anyA)+1, len(result)) + } + }) + } +} + +// Build up 2x Any type lists of unique and different types, then Union merge. +func BenchmarkAnyUnionAllUniqueTypes(b *testing.B) { + sizes := []int{100, 250, 500, 1000, 2500} + for _, sizeA := range sizes { + for _, sizeB := range sizes { + anyA := Any(make([]Type, 0, sizeA)) + for i := 0; i < sizeA; i++ { + tpe := generateType(i) + anyA = append(anyA, tpe) + } + anyB := Any(make([]Type, 0, sizeB)) + for i := 0; i < sizeB; i++ { + tpe := generateTypeWithPrefix(i, "B-") + anyB = append(anyB, tpe) + } + b.ResetTimer() + b.Run(fmt.Sprintf("%dx%d", sizeA, sizeB), func(b *testing.B) { + resultA2B := anyA.Union(anyB) + // Expect length to be A + B - 1, because the `object` type is present in both Any type sets. + if len(resultA2B) != (len(anyA) + len(anyB) - 1) { + b.Fatalf("Expected length of unioned result to be: %d, got: %d", len(anyA)+len(anyB), len(resultA2B)) + } + }) + } + } +} diff --git a/vendor/github.com/Microsoft/hcsshim/LICENSE b/vendor/github.com/Microsoft/hcsshim/LICENSE new file mode 100644 index 0000000000..49d21669ae --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go b/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go new file mode 100644 index 0000000000..6c435d2b64 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go @@ -0,0 +1,59 @@ +package osversion + +import ( + "fmt" + "sync" + + "golang.org/x/sys/windows" +) + +// OSVersion is a wrapper for Windows version information +// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724439(v=vs.85).aspx +type OSVersion struct { + Version uint32 + MajorVersion uint8 + MinorVersion uint8 + Build uint16 +} + +var ( + osv OSVersion + once sync.Once +) + +// Get gets the operating system version on Windows. +// The calling application must be manifested to get the correct version information. +func Get() OSVersion { + once.Do(func() { + var err error + osv = OSVersion{} + osv.Version, err = windows.GetVersion() + if err != nil { + // GetVersion never fails. + panic(err) + } + osv.MajorVersion = uint8(osv.Version & 0xFF) + osv.MinorVersion = uint8(osv.Version >> 8 & 0xFF) + osv.Build = uint16(osv.Version >> 16) + }) + return osv +} + +// Build gets the build-number on Windows +// The calling application must be manifested to get the correct version information. +func Build() uint16 { + return Get().Build +} + +// String returns the OSVersion formatted as a string. It implements the +// [fmt.Stringer] interface. +func (osv OSVersion) String() string { + return fmt.Sprintf("%d.%d.%d", osv.MajorVersion, osv.MinorVersion, osv.Build) +} + +// ToString returns the OSVersion formatted as a string. +// +// Deprecated: use [OSVersion.String]. +func (osv OSVersion) ToString() string { + return osv.String() +} diff --git a/vendor/github.com/Microsoft/hcsshim/osversion/platform_compat_windows.go b/vendor/github.com/Microsoft/hcsshim/osversion/platform_compat_windows.go new file mode 100644 index 0000000000..f8d411ad7e --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/osversion/platform_compat_windows.go @@ -0,0 +1,35 @@ +package osversion + +// List of stable ABI compliant ltsc releases +// Note: List must be sorted in ascending order +var compatLTSCReleases = []uint16{ + V21H2Server, +} + +// CheckHostAndContainerCompat checks if given host and container +// OS versions are compatible. +// It includes support for stable ABI compliant versions as well. +// Every release after WS 2022 will support the previous ltsc +// container image. Stable ABI is in preview mode for windows 11 client. +// Refer: https://learn.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility?tabs=windows-server-2022%2Cwindows-10#windows-server-host-os-compatibility +func CheckHostAndContainerCompat(host, ctr OSVersion) bool { + // check major minor versions of host and guest + if host.MajorVersion != ctr.MajorVersion || + host.MinorVersion != ctr.MinorVersion { + return false + } + + // If host is < WS 2022, exact version match is required + if host.Build < V21H2Server { + return host.Build == ctr.Build + } + + var supportedLtscRelease uint16 + for i := len(compatLTSCReleases) - 1; i >= 0; i-- { + if host.Build >= compatLTSCReleases[i] { + supportedLtscRelease = compatLTSCReleases[i] + break + } + } + return ctr.Build >= supportedLtscRelease && ctr.Build <= host.Build +} diff --git a/vendor/github.com/Microsoft/hcsshim/osversion/windowsbuilds.go b/vendor/github.com/Microsoft/hcsshim/osversion/windowsbuilds.go new file mode 100644 index 0000000000..446369591a --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/osversion/windowsbuilds.go @@ -0,0 +1,84 @@ +package osversion + +// Windows Client and Server build numbers. +// +// See: +// https://learn.microsoft.com/en-us/windows/release-health/release-information +// https://learn.microsoft.com/en-us/windows/release-health/windows-server-release-info +// https://learn.microsoft.com/en-us/windows/release-health/windows11-release-information +const ( + // RS1 (version 1607, codename "Redstone 1") corresponds to Windows Server + // 2016 (ltsc2016) and Windows 10 (Anniversary Update). + RS1 = 14393 + // V1607 (version 1607, codename "Redstone 1") is an alias for [RS1]. + V1607 = RS1 + // LTSC2016 (Windows Server 2016) is an alias for [RS1]. + LTSC2016 = RS1 + + // RS2 (version 1703, codename "Redstone 2") was a client-only update, and + // corresponds to Windows 10 (Creators Update). + RS2 = 15063 + // V1703 (version 1703, codename "Redstone 2") is an alias for [RS2]. + V1703 = RS2 + + // RS3 (version 1709, codename "Redstone 3") corresponds to Windows Server + // 1709 (Semi-Annual Channel (SAC)), and Windows 10 (Fall Creators Update). + RS3 = 16299 + // V1709 (version 1709, codename "Redstone 3") is an alias for [RS3]. + V1709 = RS3 + + // RS4 (version 1803, codename "Redstone 4") corresponds to Windows Server + // 1803 (Semi-Annual Channel (SAC)), and Windows 10 (April 2018 Update). + RS4 = 17134 + // V1803 (version 1803, codename "Redstone 4") is an alias for [RS4]. + V1803 = RS4 + + // RS5 (version 1809, codename "Redstone 5") corresponds to Windows Server + // 2019 (ltsc2019), and Windows 10 (October 2018 Update). + RS5 = 17763 + // V1809 (version 1809, codename "Redstone 5") is an alias for [RS5]. + V1809 = RS5 + // LTSC2019 (Windows Server 2019) is an alias for [RS5]. + LTSC2019 = RS5 + + // V19H1 (version 1903, codename 19H1) corresponds to Windows Server 1903 (semi-annual + // channel). + V19H1 = 18362 + // V1903 (version 1903) is an alias for [V19H1]. + V1903 = V19H1 + + // V19H2 (version 1909, codename 19H2) corresponds to Windows Server 1909 (semi-annual + // channel). + V19H2 = 18363 + // V1909 (version 1909) is an alias for [V19H2]. + V1909 = V19H2 + + // V20H1 (version 2004, codename 20H1) corresponds to Windows Server 2004 (semi-annual + // channel). + V20H1 = 19041 + // V2004 (version 2004) is an alias for [V20H1]. + V2004 = V20H1 + + // V20H2 corresponds to Windows Server 20H2 (semi-annual channel). + V20H2 = 19042 + + // V21H1 corresponds to Windows Server 21H1 (semi-annual channel). + V21H1 = 19043 + + // V21H2Win10 corresponds to Windows 10 (November 2021 Update). + V21H2Win10 = 19044 + + // V21H2Server corresponds to Windows Server 2022 (ltsc2022). + V21H2Server = 20348 + // LTSC2022 (Windows Server 2022) is an alias for [V21H2Server] + LTSC2022 = V21H2Server + + // V21H2Win11 corresponds to Windows 11 (original release). + V21H2Win11 = 22000 + + // V22H2Win10 corresponds to Windows 10 (2022 Update). + V22H2Win10 = 19045 + + // V22H2Win11 corresponds to Windows 11 (2022 Update). + V22H2Win11 = 22621 +) diff --git a/vendor/github.com/containerd/containerd/content/content.go b/vendor/github.com/containerd/containerd/content/content.go index b7230d88b6..8eb1a16924 100644 --- a/vendor/github.com/containerd/containerd/content/content.go +++ b/vendor/github.com/containerd/containerd/content/content.go @@ -87,9 +87,6 @@ type IngestManager interface { } // Info holds content specific information -// -// TODO(stevvooe): Consider a very different name for this struct. Info is way -// to general. It also reads very weird in certain context, like pluralization. type Info struct { Digest digest.Digest Size int64 @@ -111,12 +108,17 @@ type Status struct { // WalkFunc defines the callback for a blob walk. type WalkFunc func(Info) error -// Manager provides methods for inspecting, listing and removing content. -type Manager interface { +// InfoProvider provides info for content inspection. +type InfoProvider interface { // Info will return metadata about content available in the content store. // // If the content is not present, ErrNotFound will be returned. Info(ctx context.Context, dgst digest.Digest) (Info, error) +} + +// Manager provides methods for inspecting, listing and removing content. +type Manager interface { + InfoProvider // Update updates mutable information related to content. // If one or more fieldpaths are provided, only those diff --git a/vendor/github.com/containerd/containerd/log/context.go b/vendor/github.com/containerd/containerd/log/context.go index b63c602f42..20153066f3 100644 --- a/vendor/github.com/containerd/containerd/log/context.go +++ b/vendor/github.com/containerd/containerd/log/context.go @@ -14,6 +14,27 @@ limitations under the License. */ +// Package log provides types and functions related to logging, passing +// loggers through a context, and attaching context to the logger. +// +// # Transitional types +// +// This package contains various types that are aliases for types in [logrus]. +// These aliases are intended for transitioning away from hard-coding logrus +// as logging implementation. Consumers of this package are encouraged to use +// the type-aliases from this package instead of directly using their logrus +// equivalent. +// +// The intent is to replace these aliases with locally defined types and +// interfaces once all consumers are no longer directly importing logrus +// types. +// +// IMPORTANT: due to the transitional purpose of this package, it is not +// guaranteed for the full logrus API to be provided in the future. As +// outlined, these aliases are provided as a step to transition away from +// a specific implementation which, as a result, exposes the full logrus API. +// While no decisions have been made on the ultimate design and interface +// provided by this package, we do not expect carrying "less common" features. package log import ( @@ -23,98 +44,139 @@ import ( "github.com/sirupsen/logrus" ) -var ( - // G is an alias for GetLogger. - // - // We may want to define this locally to a package to get package tagged log - // messages. - G = GetLogger +// G is a shorthand for [GetLogger]. +// +// We may want to define this locally to a package to get package tagged log +// messages. +var G = GetLogger + +// L is an alias for the standard logger. +var L = &Entry{ + Logger: logrus.StandardLogger(), + // Default is three fields plus a little extra room. + Data: make(Fields, 6), +} - // L is an alias for the standard logger. - L = logrus.NewEntry(logrus.StandardLogger()) -) +type loggerKey struct{} -type ( - loggerKey struct{} +// Fields type to pass to "WithFields". +type Fields = map[string]any - // Fields type to pass to `WithFields`, alias from `logrus`. - Fields = logrus.Fields +// Entry is a logging entry. It contains all the fields passed with +// [Entry.WithFields]. It's finally logged when Trace, Debug, Info, Warn, +// Error, Fatal or Panic is called on it. These objects can be reused and +// passed around as much as you wish to avoid field duplication. +// +// Entry is a transitional type, and currently an alias for [logrus.Entry]. +type Entry = logrus.Entry - // Level is a logging level - Level = logrus.Level -) +// RFC3339NanoFixed is [time.RFC3339Nano] with nanoseconds padded using +// zeros to ensure the formatted time is always the same number of +// characters. +const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + +// Level is a logging level. +type Level = logrus.Level +// Supported log levels. const ( - // RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to - // ensure the formatted time is always the same number of characters. - RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + // TraceLevel level. Designates finer-grained informational events + // than [DebugLevel]. + TraceLevel Level = logrus.TraceLevel - // TextFormat represents the text logging format - TextFormat = "text" + // DebugLevel level. Usually only enabled when debugging. Very verbose + // logging. + DebugLevel Level = logrus.DebugLevel - // JSONFormat represents the JSON logging format - JSONFormat = "json" + // InfoLevel level. General operational entries about what's going on + // inside the application. + InfoLevel Level = logrus.InfoLevel - // TraceLevel level. - TraceLevel = logrus.TraceLevel + // WarnLevel level. Non-critical entries that deserve eyes. + WarnLevel Level = logrus.WarnLevel - // DebugLevel level. - DebugLevel = logrus.DebugLevel + // ErrorLevel level. Logs errors that should definitely be noted. + // Commonly used for hooks to send errors to an error tracking service. + ErrorLevel Level = logrus.ErrorLevel - // InfoLevel level. - InfoLevel = logrus.InfoLevel + // FatalLevel level. Logs and then calls "logger.Exit(1)". It exits + // even if the logging level is set to Panic. + FatalLevel Level = logrus.FatalLevel + + // PanicLevel level. This is the highest level of severity. Logs and + // then calls panic with the message passed to Debug, Info, ... + PanicLevel Level = logrus.PanicLevel ) -// SetLevel sets log level globally. +// SetLevel sets log level globally. It returns an error if the given +// level is not supported. +// +// level can be one of: +// +// - "trace" ([TraceLevel]) +// - "debug" ([DebugLevel]) +// - "info" ([InfoLevel]) +// - "warn" ([WarnLevel]) +// - "error" ([ErrorLevel]) +// - "fatal" ([FatalLevel]) +// - "panic" ([PanicLevel]) func SetLevel(level string) error { lvl, err := logrus.ParseLevel(level) if err != nil { return err } - logrus.SetLevel(lvl) + L.Logger.SetLevel(lvl) return nil } // GetLevel returns the current log level. func GetLevel() Level { - return logrus.GetLevel() + return L.Logger.GetLevel() } -// SetFormat sets log output format -func SetFormat(format string) error { +// OutputFormat specifies a log output format. +type OutputFormat string + +// Supported log output formats. +const ( + // TextFormat represents the text logging format. + TextFormat OutputFormat = "text" + + // JSONFormat represents the JSON logging format. + JSONFormat OutputFormat = "json" +) + +// SetFormat sets the log output format ([TextFormat] or [JSONFormat]). +func SetFormat(format OutputFormat) error { switch format { case TextFormat: - logrus.SetFormatter(&logrus.TextFormatter{ + L.Logger.SetFormatter(&logrus.TextFormatter{ TimestampFormat: RFC3339NanoFixed, FullTimestamp: true, }) + return nil case JSONFormat: - logrus.SetFormatter(&logrus.JSONFormatter{ + L.Logger.SetFormatter(&logrus.JSONFormatter{ TimestampFormat: RFC3339NanoFixed, }) + return nil default: return fmt.Errorf("unknown log format: %s", format) } - - return nil } // WithLogger returns a new context with the provided logger. Use in // combination with logger.WithField(s) for great effect. -func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context { - e := logger.WithContext(ctx) - return context.WithValue(ctx, loggerKey{}, e) +func WithLogger(ctx context.Context, logger *Entry) context.Context { + return context.WithValue(ctx, loggerKey{}, logger.WithContext(ctx)) } // GetLogger retrieves the current logger from the context. If no logger is // available, the default logger is returned. -func GetLogger(ctx context.Context) *logrus.Entry { - logger := ctx.Value(loggerKey{}) - - if logger == nil { - return L.WithContext(ctx) +func GetLogger(ctx context.Context) *Entry { + if logger := ctx.Value(loggerKey{}); logger != nil { + return logger.(*Entry) } - - return logger.(*logrus.Entry) + return L.WithContext(ctx) } diff --git a/vendor/github.com/containerd/containerd/platforms/defaults_windows.go b/vendor/github.com/containerd/containerd/platforms/defaults_windows.go index fd5756516c..d10fa9012b 100644 --- a/vendor/github.com/containerd/containerd/platforms/defaults_windows.go +++ b/vendor/github.com/containerd/containerd/platforms/defaults_windows.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "github.com/Microsoft/hcsshim/osversion" specs "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sys/windows" ) @@ -50,15 +51,36 @@ func (m windowsmatcher) Match(p specs.Platform) bool { match := m.defaultMatcher.Match(p) if match && m.OS == "windows" { - if strings.HasPrefix(p.OSVersion, m.osVersionPrefix) { + // HPC containers do not have OS version filled + if p.OSVersion == "" { return true } - return p.OSVersion == "" + + hostOsVersion := GetOsVersion(m.osVersionPrefix) + ctrOsVersion := GetOsVersion(p.OSVersion) + return osversion.CheckHostAndContainerCompat(hostOsVersion, ctrOsVersion) } return match } +func GetOsVersion(osVersionPrefix string) osversion.OSVersion { + parts := strings.Split(osVersionPrefix, ".") + if len(parts) < 3 { + return osversion.OSVersion{} + } + + majorVersion, _ := strconv.Atoi(parts[0]) + minorVersion, _ := strconv.Atoi(parts[1]) + buildNumber, _ := strconv.Atoi(parts[2]) + + return osversion.OSVersion{ + MajorVersion: uint8(majorVersion), + MinorVersion: uint8(minorVersion), + Build: uint16(buildNumber), + } +} + // Less sorts matched platforms in front of other platforms. // For matched platforms, it puts platforms with larger revision // number in front. diff --git a/vendor/github.com/containerd/containerd/platforms/platforms.go b/vendor/github.com/containerd/containerd/platforms/platforms.go index 8dcde7db7c..56613b0765 100644 --- a/vendor/github.com/containerd/containerd/platforms/platforms.go +++ b/vendor/github.com/containerd/containerd/platforms/platforms.go @@ -196,6 +196,10 @@ func Parse(specifier string) (specs.Platform, error) { p.Variant = cpuVariant() } + if p.OS == "windows" { + p.OSVersion = GetWindowsOsVersion() + } + return p, nil } @@ -218,6 +222,10 @@ func Parse(specifier string) (specs.Platform, error) { p.Variant = "" } + if p.OS == "windows" { + p.OSVersion = GetWindowsOsVersion() + } + return p, nil case 3: // we have a fully specified variant, this is rare @@ -227,6 +235,10 @@ func Parse(specifier string) (specs.Platform, error) { p.Variant = "v8" } + if p.OS == "windows" { + p.OSVersion = GetWindowsOsVersion() + } + return p, nil } diff --git a/vendor/github.com/containerd/containerd/platforms/platforms_other.go b/vendor/github.com/containerd/containerd/platforms/platforms_other.go index 03f4dcd998..59beeb3d1d 100644 --- a/vendor/github.com/containerd/containerd/platforms/platforms_other.go +++ b/vendor/github.com/containerd/containerd/platforms/platforms_other.go @@ -28,3 +28,7 @@ func newDefaultMatcher(platform specs.Platform) Matcher { Platform: Normalize(platform), } } + +func GetWindowsOsVersion() string { + return "" +} diff --git a/vendor/github.com/containerd/containerd/platforms/platforms_windows.go b/vendor/github.com/containerd/containerd/platforms/platforms_windows.go index 950e2a2ddb..733d18ddea 100644 --- a/vendor/github.com/containerd/containerd/platforms/platforms_windows.go +++ b/vendor/github.com/containerd/containerd/platforms/platforms_windows.go @@ -17,7 +17,10 @@ package platforms import ( + "fmt" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sys/windows" ) // NewMatcher returns a Windows matcher that will match on osVersionPrefix if @@ -32,3 +35,8 @@ func newDefaultMatcher(platform specs.Platform) Matcher { }, } } + +func GetWindowsOsVersion() string { + major, minor, build := windows.RtlGetNtVersionNumbers() + return fmt.Sprintf("%d.%d.%d", major, minor, build) +} diff --git a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index 96110a1887..3d6c0182f1 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -673,7 +673,7 @@ func requestFields(req *http.Request) log.Fields { } } - return log.Fields(fields) + return fields } func responseFields(resp *http.Response) log.Fields { @@ -691,7 +691,7 @@ func responseFields(resp *http.Response) log.Fields { } } - return log.Fields(fields) + return fields } // IsLocalhost checks if the registry host is local. diff --git a/vendor/github.com/containerd/containerd/remotes/handlers.go b/vendor/github.com/containerd/containerd/remotes/handlers.go index 31de555164..0ff39179c2 100644 --- a/vendor/github.com/containerd/containerd/remotes/handlers.go +++ b/vendor/github.com/containerd/containerd/remotes/handlers.go @@ -204,8 +204,9 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc // Base handlers can be provided which will be called before any push specific // handlers. // -// If the passed in content.Provider is also a content.Manager then this will -// also annotate the distribution sources in the manager. +// If the passed in content.Provider is also a content.InfoProvider (such as +// content.Manager) then this will also annotate the distribution sources using +// labels prefixed with "containerd.io/distribution.source". func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Provider, limiter *semaphore.Weighted, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error { var m sync.Mutex @@ -234,7 +235,7 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, st platformFilterhandler := images.FilterPlatforms(images.ChildrenHandler(store), platform) var handler images.Handler - if m, ok := store.(content.Manager); ok { + if m, ok := store.(content.InfoProvider); ok { annotateHandler := annotateDistributionSourceHandler(platformFilterhandler, m) handler = images.Handlers(annotateHandler, filterHandler, pushHandler) } else { @@ -344,14 +345,15 @@ func FilterManifestByPlatformHandler(f images.HandlerFunc, m platforms.Matcher) // annotateDistributionSourceHandler add distribution source label into // annotation of config or blob descriptor. -func annotateDistributionSourceHandler(f images.HandlerFunc, manager content.Manager) images.HandlerFunc { +func annotateDistributionSourceHandler(f images.HandlerFunc, provider content.InfoProvider) images.HandlerFunc { return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { children, err := f(ctx, desc) if err != nil { return nil, err } - // only add distribution source for the config or blob data descriptor + // Distribution source is only used for config or blob but may be inherited from + // a manifest or manifest list switch desc.MediaType { case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: @@ -359,12 +361,28 @@ func annotateDistributionSourceHandler(f images.HandlerFunc, manager content.Man return children, nil } + // parentInfo can be used to inherit info for non-existent blobs + var parentInfo *content.Info + for i := range children { child := children[i] - info, err := manager.Info(ctx, child.Digest) + info, err := provider.Info(ctx, child.Digest) if err != nil { - return nil, err + if !errdefs.IsNotFound(err) { + return nil, err + } + if parentInfo == nil { + pi, err := provider.Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + parentInfo = &pi + } + // Blob may not exist locally, annotate with parent labels for cross repo + // mount or fetch. Parent sources may apply to all children since most + // registries enforce that children exist before the manifests. + info = *parentInfo } for k, v := range info.Labels { diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index b1d3be630a..350f81eace 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -23,7 +23,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.7.3+unknown" + Version = "1.7.6+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/google.golang.org/grpc/call.go b/vendor/google.golang.org/grpc/call.go index a67a3db02e..788c89c16f 100644 --- a/vendor/google.golang.org/grpc/call.go +++ b/vendor/google.golang.org/grpc/call.go @@ -27,11 +27,6 @@ import ( // // All errors returned by Invoke are compatible with the status package. func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply any, opts ...CallOption) error { - if err := cc.idlenessMgr.OnCallBegin(); err != nil { - return err - } - defer cc.idlenessMgr.OnCallEnd() - // allow interceptor to see all applicable call options, which means those // configured as defaults from dial option as well as per-call options opts = combine(cc.dopts.callOptions, opts) diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index d53d91d5d9..ff7fea1022 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -1091,8 +1091,8 @@ func (ac *addrConn) updateAddrs(addrs []resolver.Address) { ac.cancel() ac.ctx, ac.cancel = context.WithCancel(ac.cc.ctx) - // We have to defer here because GracefulClose => Close => onClose, which - // requires locking ac.mu. + // We have to defer here because GracefulClose => onClose, which requires + // locking ac.mu. if ac.transport != nil { defer ac.transport.GracefulClose() ac.transport = nil @@ -1680,16 +1680,7 @@ func (ac *addrConn) tearDown(err error) { ac.updateConnectivityState(connectivity.Shutdown, nil) ac.cancel() ac.curAddr = resolver.Address{} - if err == errConnDrain && curTr != nil { - // GracefulClose(...) may be executed multiple times when - // i) receiving multiple GoAway frames from the server; or - // ii) there are concurrent name resolver/Balancer triggered - // address removal and GoAway. - // We have to unlock and re-lock here because GracefulClose => Close => onClose, which requires locking ac.mu. - ac.mu.Unlock() - curTr.GracefulClose() - ac.mu.Lock() - } + channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ Desc: "Subchannel deleted", Severity: channelz.CtInfo, @@ -1703,6 +1694,29 @@ func (ac *addrConn) tearDown(err error) { // being deleted right away. channelz.RemoveEntry(ac.channelzID) ac.mu.Unlock() + + // We have to release the lock before the call to GracefulClose/Close here + // because both of them call onClose(), which requires locking ac.mu. + if curTr != nil { + if err == errConnDrain { + // Close the transport gracefully when the subConn is being shutdown. + // + // GracefulClose() may be executed multiple times if: + // - multiple GoAway frames are received from the server + // - there are concurrent name resolver or balancer triggered + // address removal and GoAway + curTr.GracefulClose() + } else { + // Hard close the transport when the channel is entering idle or is + // being shutdown. In the case where the channel is being shutdown, + // closing of transports is also taken care of by cancelation of cc.ctx. + // But in the case where the channel is entering idle, we need to + // explicitly close the transports here. Instead of distinguishing + // between these two cases, it is simpler to close the transport + // unconditionally here. + curTr.Close(err) + } + } } func (ac *addrConn) getState() connectivity.State { diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index 421a41f885..b14b2fbea2 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -158,11 +158,6 @@ type ClientStream interface { // If none of the above happen, a goroutine and a context will be leaked, and grpc // will not call the optionally-configured stats handler with a stats.End message. func (cc *ClientConn) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { - if err := cc.idlenessMgr.OnCallBegin(); err != nil { - return nil, err - } - defer cc.idlenessMgr.OnCallEnd() - // allow interceptor to see all applicable call options, which means those // configured as defaults from dial option as well as per-call options opts = combine(cc.dopts.callOptions, opts) @@ -179,6 +174,16 @@ func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { + // Start tracking the RPC for idleness purposes. This is where a stream is + // created for both streaming and unary RPCs, and hence is a good place to + // track active RPC count. + if err := cc.idlenessMgr.OnCallBegin(); err != nil { + return nil, err + } + // Add a calloption, to decrement the active call count, that gets executed + // when the RPC completes. + opts = append([]CallOption{OnFinish(func(error) { cc.idlenessMgr.OnCallEnd() })}, opts...) + if md, added, ok := metadata.FromOutgoingContextRaw(ctx); ok { // validate md if err := imetadata.Validate(md); err != nil { diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go index 914ce665f5..d3f5bcbfce 100644 --- a/vendor/google.golang.org/grpc/version.go +++ b/vendor/google.golang.org/grpc/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.58.0" +const Version = "1.58.2" diff --git a/vendor/modules.txt b/vendor/modules.txt index 0a29fd4759..1fa481cf42 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,6 +1,9 @@ # github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 ## explicit; go 1.20 github.com/AdaLogics/go-fuzz-headers +# github.com/Microsoft/hcsshim v0.11.0 +## explicit; go 1.18 +github.com/Microsoft/hcsshim/osversion # github.com/OneOfOne/xxhash v1.2.8 ## explicit; go 1.11 github.com/OneOfOne/xxhash @@ -32,7 +35,7 @@ github.com/cespare/xxhash # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/containerd/containerd v1.7.4 +# github.com/containerd/containerd v1.7.6 ## explicit; go 1.19 github.com/containerd/containerd/archive/compression github.com/containerd/containerd/content @@ -365,7 +368,7 @@ google.golang.org/genproto/googleapis/api/httpbody ## explicit; go 1.19 google.golang.org/genproto/googleapis/rpc/errdetails google.golang.org/genproto/googleapis/rpc/status -# google.golang.org/grpc v1.58.0 +# google.golang.org/grpc v1.58.2 ## explicit; go 1.19 google.golang.org/grpc google.golang.org/grpc/attributes