From cd0bf5b4d2d578ae56577c696659db69b7fe4d45 Mon Sep 17 00:00:00 2001 From: Charlie Egan Date: Thu, 21 Sep 2023 09:40:07 +0100 Subject: [PATCH] Add option to marshal location text (#6234) --- ast/annotations.go | 10 +- ast/json/json.go | 33 ++++ ast/location/location.go | 26 +++ ast/location/location_test.go | 45 ++++++ ast/marshal.go | 6 +- ast/marshal_test.go | 196 +++++++++++++++-------- ast/parser.go | 36 +---- ast/parser_ext.go | 4 +- ast/parser_test.go | 21 +-- ast/policy.go | 68 +++++--- ast/term.go | 8 +- bundle/bundle.go | 5 +- cmd/parse.go | 8 +- cmd/parse_test.go | 245 ++++++++++++++++++++--------- internal/bundle/inspect/inspect.go | 7 +- loader/loader.go | 5 +- loader/loader_test.go | 7 +- 17 files changed, 510 insertions(+), 220 deletions(-) create mode 100644 ast/json/json.go diff --git a/ast/annotations.go b/ast/annotations.go index 1134cc87c4..fe5e321997 100644 --- a/ast/annotations.go +++ b/ast/annotations.go @@ -11,6 +11,7 @@ import ( "sort" "strings" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/internal/deepcopy" "github.com/open-policy-agent/opa/util" ) @@ -39,7 +40,7 @@ type ( comments []*Comment node Node - jsonOptions JSONOptions + jsonOptions astJSON.Options } // SchemaAnnotation contains a schema declaration for the document identified by the path. @@ -76,7 +77,7 @@ type ( Annotations *Annotations `json:"annotations,omitempty"` Location *Location `json:"location,omitempty"` // The location of the node the annotations are applied to - jsonOptions JSONOptions + jsonOptions astJSON.Options node Node // The node the annotations are applied to } @@ -180,8 +181,11 @@ func (a *Annotations) GetTargetPath() Ref { } } -func (a *Annotations) setJSONOptions(opts JSONOptions) { +func (a *Annotations) setJSONOptions(opts astJSON.Options) { a.jsonOptions = opts + if a.Location != nil { + a.Location.JSONOptions = opts + } } func (a *Annotations) MarshalJSON() ([]byte, error) { diff --git a/ast/json/json.go b/ast/json/json.go new file mode 100644 index 0000000000..2a72a9dbd6 --- /dev/null +++ b/ast/json/json.go @@ -0,0 +1,33 @@ +package json + +// Options defines the options for JSON operations, +// currently only marshaling can be configured +type Options struct { + MarshalOptions MarshalOptions +} + +// MarshalOptions defines the options for JSON marshaling, +// currently only toggling the marshaling of location information is supported +type MarshalOptions struct { + // IncludeLocation toggles the marshaling of location information + IncludeLocation NodeToggle + // IncludeLocationText additionally/optionally includes the text of the location + IncludeLocationText bool +} + +// NodeToggle is a generic struct to allow the toggling of +// settings for different ast node types +type NodeToggle struct { + Term bool + Package bool + Comment bool + Import bool + Rule bool + Head bool + Expr bool + SomeDecl bool + Every bool + With bool + Annotations bool + AnnotationsRef bool +} diff --git a/ast/location/location.go b/ast/location/location.go index 9ef1e6dfd4..1ac4d15282 100644 --- a/ast/location/location.go +++ b/ast/location/location.go @@ -3,8 +3,11 @@ package location import ( "bytes" + "encoding/json" "errors" "fmt" + + astJSON "github.com/open-policy-agent/opa/ast/json" ) // Location records a position in source code @@ -14,6 +17,9 @@ type Location struct { Row int `json:"row"` // The line in the source. Col int `json:"col"` // The column in the row. Offset int `json:"-"` // The byte offset for the location in the source. + + // JSONOptions specifies options for marshaling and unmarshaling of locations + JSONOptions astJSON.Options } // NewLocation returns a new Location object. @@ -87,3 +93,23 @@ func (loc *Location) Compare(other *Location) int { } return 0 } + +func (loc *Location) MarshalJSON() ([]byte, error) { + // structs are used here to preserve the field ordering of the original Location struct + data := struct { + File string `json:"file"` + Row int `json:"row"` + Col int `json:"col"` + Text []byte `json:"text,omitempty"` + }{ + File: loc.File, + Row: loc.Row, + Col: loc.Col, + } + + if loc.JSONOptions.MarshalOptions.IncludeLocationText { + data.Text = loc.Text + } + + return json.Marshal(data) +} diff --git a/ast/location/location_test.go b/ast/location/location_test.go index a6e54c62d1..f61fec5edd 100644 --- a/ast/location/location_test.go +++ b/ast/location/location_test.go @@ -1,8 +1,10 @@ package location import ( + "encoding/json" "testing" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/util" ) @@ -85,3 +87,46 @@ func TestLocationCompare(t *testing.T) { } } } + +func TestLocationMarshal(t *testing.T) { + testCases := map[string]struct { + loc *Location + exp string + }{ + "default json options": { + loc: &Location{ + Text: []byte("text"), + File: "file", + Row: 1, + Col: 1, + }, + exp: `{"file":"file","row":1,"col":1}`, + }, + "including text": { + loc: &Location{ + Text: []byte("text"), + File: "file", + Row: 1, + Col: 1, + JSONOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocationText: true, + }, + }, + }, + exp: `{"file":"file","row":1,"col":1,"text":"dGV4dA=="}`, + }, + } + + for id, tc := range testCases { + t.Run(id, func(t *testing.T) { + bs, err := json.Marshal(tc.loc) + if err != nil { + t.Fatal(err) + } + if string(bs) != tc.exp { + t.Fatalf("Expected %v but got %v", tc.exp, string(bs)) + } + }) + } +} diff --git a/ast/marshal.go b/ast/marshal.go index 891945db8b..53fb112044 100644 --- a/ast/marshal.go +++ b/ast/marshal.go @@ -1,7 +1,11 @@ package ast +import ( + astJSON "github.com/open-policy-agent/opa/ast/json" +) + // customJSON is an interface that can be implemented by AST nodes that // allows the parser to set options for JSON operations on that node. type customJSON interface { - setJSONOptions(JSONOptions) + setJSONOptions(astJSON.Options) } diff --git a/ast/marshal_test.go b/ast/marshal_test.go index 78f404cb0b..d6410102f6 100644 --- a/ast/marshal_test.go +++ b/ast/marshal_test.go @@ -4,9 +4,79 @@ import ( "encoding/json" "testing" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/util" ) +func TestGeneric_MarshalWithLocationJSONOptions(t *testing.T) { + testCases := map[string]struct { + Term *Term + ExpectedJSON string + }{ + "base case, no location options set": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + } + }(), + ExpectedJSON: `{"type":"string","value":"example"}`, + }, + "location included, location text excluded": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + return &Term{ + Value: v, + Location: NewLocation([]byte{}, "example.rego", 1, 2), + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ + Term: true, + }, + IncludeLocationText: false, + }, + }, + } + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"type":"string","value":"example"}`, + }, + "location included, location text also included": { + Term: func() *Term { + v, _ := InterfaceToValue("example") + t := &Term{ + Value: v, + Location: NewLocation([]byte("things"), "example.rego", 1, 2), + } + t.setJSONOptions( + astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ + Term: true, + }, + IncludeLocationText: true, + }, + }, + ) + return t + }(), + ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2,"text":"dGhpbmdz"},"type":"string","value":"example"}`, + }, + } + + for name, data := range testCases { + t.Run(name, func(t *testing.T) { + bs := util.MustMarshalJSON(data.Term) + got := string(bs) + exp := data.ExpectedJSON + + if got != exp { + t.Fatalf("expected:\n%s got\n%s", exp, got) + } + }) + } +} + func TestTerm_MarshalJSON(t *testing.T) { testCases := map[string]struct { Term *Term @@ -28,9 +98,9 @@ func TestTerm_MarshalJSON(t *testing.T) { return &Term{ Value: v, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: false, }, }, @@ -45,9 +115,9 @@ func TestTerm_MarshalJSON(t *testing.T) { return &Term{ Value: v, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: true, }, }, @@ -132,9 +202,9 @@ func TestPackage_MarshalJSON(t *testing.T) { Package: &Package{ Path: EmptyRef(), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Package: false, }, }, @@ -146,9 +216,9 @@ func TestPackage_MarshalJSON(t *testing.T) { Package: &Package{ Path: EmptyRef(), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Package: true, }, }, @@ -188,9 +258,9 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Comment: false, // ignored }, }, @@ -202,9 +272,9 @@ func TestComment_MarshalJSON(t *testing.T) { Comment: &Comment{ Text: []byte("comment"), Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Comment: true, // ignored }, }, @@ -253,9 +323,9 @@ func TestImport_MarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Import: false, }, }, @@ -274,9 +344,9 @@ func TestImport_MarshalJSON(t *testing.T) { return &Import{ Path: &term, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Import: true, }, }, @@ -327,9 +397,9 @@ func TestRule_MarshalJSON(t *testing.T) { "location excluded": { Rule: func() *Rule { r := rule.Copy() - r.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + r.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Rule: false, }, }, @@ -341,9 +411,9 @@ func TestRule_MarshalJSON(t *testing.T) { "location included": { Rule: func() *Rule { r := rule.Copy() - r.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + r.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Rule: true, }, }, @@ -394,9 +464,9 @@ func TestHead_MarshalJSON(t *testing.T) { "location excluded": { Head: func() *Head { h := head.Copy() - h.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + h.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Head: false, }, }, @@ -409,9 +479,9 @@ func TestHead_MarshalJSON(t *testing.T) { "location included": { Head: func() *Head { h := head.Copy() - h.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + h.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Head: true, }, }, @@ -462,9 +532,9 @@ func TestExpr_MarshalJSON(t *testing.T) { "location excluded": { Expr: func() *Expr { e := expr.Copy() - e.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + e.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Expr: false, }, }, @@ -477,9 +547,9 @@ func TestExpr_MarshalJSON(t *testing.T) { "location included": { Expr: func() *Expr { e := expr.Copy() - e.jsonOptions = JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + e.jsonOptions = astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Expr: true, }, }, @@ -581,7 +651,7 @@ func TestSomeDecl_MarshalJSON(t *testing.T) { SomeDecl: &SomeDecl{ Symbols: []*Term{term}, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{SomeDecl: false}}}, + jsonOptions: astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{SomeDecl: false}}}, }, ExpectedJSON: `{"symbols":[{"type":"string","value":"example"}]}`, }, @@ -589,7 +659,7 @@ func TestSomeDecl_MarshalJSON(t *testing.T) { SomeDecl: &SomeDecl{ Symbols: []*Term{term}, Location: NewLocation([]byte{}, "example.rego", 1, 2), - jsonOptions: JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{SomeDecl: true}}}, + jsonOptions: astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{SomeDecl: true}}}, }, ExpectedJSON: `{"location":{"file":"example.rego","row":1,"col":2},"symbols":[{"type":"string","value":"example"}]}`, }, @@ -643,7 +713,7 @@ allow { "location excluded": { Every: func() *Every { e := every.Copy() - e.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{Every: false}}} + e.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{Every: false}}} return e }(), ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"value":{"type":"var","value":"e"}}`, @@ -651,7 +721,7 @@ allow { "location included": { Every: func() *Every { e := every.Copy() - e.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{Every: true}}} + e.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{Every: true}}} return e }(), ExpectedJSON: `{"body":[{"index":0,"terms":[{"type":"ref","value":[{"type":"var","value":"equal"}]},{"type":"var","value":"e"},{"type":"number","value":1}]}],"domain":{"type":"array","value":[{"type":"number","value":1},{"type":"number","value":2},{"type":"number","value":3}]},"key":null,"location":{"file":"example.rego","row":7,"col":2},"value":{"type":"var","value":"e"}}`, @@ -701,7 +771,7 @@ b { "location excluded": { With: func() *With { w := with.Copy() - w.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{With: false}}} + w.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{With: false}}} return w }(), ExpectedJSON: `{"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, @@ -709,7 +779,7 @@ b { "location included": { With: func() *With { w := with.Copy() - w.jsonOptions = JSONOptions{MarshalOptions: JSONMarshalOptions{IncludeLocation: NodeToggle{With: true}}} + w.jsonOptions = astJSON.Options{MarshalOptions: astJSON.MarshalOptions{IncludeLocation: astJSON.NodeToggle{With: true}}} return w }(), ExpectedJSON: `{"location":{"file":"example.rego","row":7,"col":4},"target":{"type":"ref","value":[{"type":"var","value":"input"}]},"value":{"type":"number","value":1}}`, @@ -761,9 +831,9 @@ func TestAnnotations_MarshalJSON(t *testing.T) { }, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{Annotations: false}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{Annotations: false}, }, }, }, @@ -781,9 +851,9 @@ func TestAnnotations_MarshalJSON(t *testing.T) { }, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{Annotations: true}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{Annotations: true}, }, }, }, @@ -824,9 +894,9 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { Path: []*Term{}, Annotations: &Annotations{}, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{AnnotationsRef: false}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{AnnotationsRef: false}, }, }, }, @@ -838,9 +908,9 @@ func TestAnnotationsRef_MarshalJSON(t *testing.T) { Annotations: &Annotations{}, Location: NewLocation([]byte{}, "example.rego", 1, 4), - jsonOptions: JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{AnnotationsRef: true}, + jsonOptions: astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{AnnotationsRef: true}, }, }, }, @@ -916,9 +986,9 @@ package test p = 1`, options: ParserOptions{ ProcessAnnotation: true, - JSONOptions: &JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + JSONOptions: &astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: true, Package: true, Comment: true, diff --git a/ast/parser.go b/ast/parser.go index 5f47636ea3..d32ae4ec5c 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -22,6 +22,7 @@ import ( "github.com/open-policy-agent/opa/ast/internal/scanner" "github.com/open-policy-agent/opa/ast/internal/tokens" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/ast/location" ) @@ -102,40 +103,11 @@ type ParserOptions struct { AllFutureKeywords bool FutureKeywords []string SkipRules bool - JSONOptions *JSONOptions + JSONOptions *astJSON.Options unreleasedKeywords bool // TODO(sr): cleanup generalRuleRefsEnabled bool } -// JSONOptions defines the options for JSON operations, -// currently only marshaling can be configured -type JSONOptions struct { - MarshalOptions JSONMarshalOptions -} - -// JSONMarshalOptions defines the options for JSON marshaling, -// currently only toggling the marshaling of location information is supported -type JSONMarshalOptions struct { - IncludeLocation NodeToggle -} - -// NodeToggle is a generic struct to allow the toggling of -// settings for different ast node types -type NodeToggle struct { - Term bool - Package bool - Comment bool - Import bool - Rule bool - Head bool - Expr bool - SomeDecl bool - Every bool - With bool - Annotations bool - AnnotationsRef bool -} - // NewParser creates and initializes a Parser. func NewParser() *Parser { p := &Parser{ @@ -210,9 +182,9 @@ func (p *Parser) WithSkipRules(skip bool) *Parser { return p } -// WithJSONOptions sets the JSONOptions which will be set on nodes to configure +// WithJSONOptions sets the Options which will be set on nodes to configure // their JSON marshaling behavior. -func (p *Parser) WithJSONOptions(jsonOptions *JSONOptions) *Parser { +func (p *Parser) WithJSONOptions(jsonOptions *astJSON.Options) *Parser { p.po.JSONOptions = jsonOptions return p } diff --git a/ast/parser_ext.go b/ast/parser_ext.go index 9565029ea8..9226a9059a 100644 --- a/ast/parser_ext.go +++ b/ast/parser_ext.go @@ -16,6 +16,8 @@ import ( "fmt" "strings" "unicode" + + astJSON "github.com/open-policy-agent/opa/ast/json" ) // MustParseBody returns a parsed body. @@ -713,7 +715,7 @@ func setRuleModule(rule *Rule, module *Module) { } } -func setJSONOptions(x interface{}, jsonOptions *JSONOptions) { +func setJSONOptions(x interface{}, jsonOptions *astJSON.Options) { vis := NewGenericVisitor(func(x interface{}) bool { if x, ok := x.(customJSON); ok { x.setJSONOptions(*jsonOptions) diff --git a/ast/parser_test.go b/ast/parser_test.go index b4cb9951a9..7a2af2ae05 100644 --- a/ast/parser_test.go +++ b/ast/parser_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/open-policy-agent/opa/ast/internal/tokens" + astJSON "github.com/open-policy-agent/opa/ast/json" ) const ( @@ -3315,9 +3316,9 @@ func TestRuleFromBodyJSONOptions(t *testing.T) { } parserOpts := ParserOptions{ProcessAnnotation: true} - parserOpts.JSONOptions = &JSONOptions{ - MarshalOptions: JSONMarshalOptions{ - IncludeLocation: NodeToggle{ + parserOpts.JSONOptions = &astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Term: true, Package: true, Comment: true, @@ -5136,33 +5137,33 @@ func assertParseModuleJSONOptions(t *testing.T, msg string, input string, opts . rule := m.Rules[0] if rule.Head.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected rule Head JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Head.jsonOptions) + t.Fatalf("Error on test \"%s\": expected rule Head Options\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Head.jsonOptions) } if rule.Body[0].jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected rule Body JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Body[0].jsonOptions) + t.Fatalf("Error on test \"%s\": expected rule Body Options\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.Body[0].jsonOptions) } switch terms := rule.Body[0].Terms.(type) { case []*Term: for _, term := range terms { if term.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, term.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, term.jsonOptions) } } case *SomeDecl: if terms.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) } case *Every: if terms.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) } case *Term: if terms.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected body Term JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) + t.Fatalf("Error on test \"%s\": expected body Term Options\n%v\n, got\n%v", msg, *opt.JSONOptions, terms.jsonOptions) } } if rule.jsonOptions != *opt.JSONOptions { - t.Fatalf("Error on test \"%s\": expected rule JSONOptions\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.jsonOptions) + t.Fatalf("Error on test \"%s\": expected rule Options\n%v\n, got\n%v", msg, *opt.JSONOptions, rule.jsonOptions) } } diff --git a/ast/policy.go b/ast/policy.go index 7579401bcb..fedc738ff2 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -12,6 +12,7 @@ import ( "strings" "time" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/util" ) @@ -153,7 +154,7 @@ type ( Text []byte Location *Location - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Package represents the namespace of the documents produced @@ -162,7 +163,7 @@ type ( Path Ref `json:"path"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Import represents a dependency on a document outside of the policy @@ -172,7 +173,7 @@ type ( Alias Var `json:"alias,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Rule represents a rule as defined in the language. Rules define the @@ -190,7 +191,7 @@ type ( // on the rule (e.g., printing, comparison, visiting, etc.) Module *Module `json:"-"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Head represents the head of a rule. @@ -203,7 +204,7 @@ type ( Assign bool `json:"assign,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // Args represents zero or more arguments to a rule. @@ -222,7 +223,7 @@ type ( Negated bool `json:"negated,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // SomeDecl represents a variable declaration statement. The symbols are variables. @@ -230,7 +231,7 @@ type ( Symbols []*Term `json:"symbols"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } Every struct { @@ -240,7 +241,7 @@ type ( Body Body `json:"body"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } // With represents a modifier on an expression. @@ -249,7 +250,7 @@ type ( Value *Term `json:"value"` Location *Location `json:"location,omitempty"` - jsonOptions JSONOptions + jsonOptions astJSON.Options } ) @@ -428,10 +429,13 @@ func (c *Comment) Equal(other *Comment) bool { return c.Location.Equal(other.Location) && bytes.Equal(c.Text, other.Text) } -func (c *Comment) setJSONOptions(opts JSONOptions) { +func (c *Comment) setJSONOptions(opts astJSON.Options) { // Note: this is not used for location since Comments use default JSON marshaling // behavior with struct field names in JSON. c.jsonOptions = opts + if c.Location != nil { + c.Location.JSONOptions = opts + } } // Compare returns an integer indicating whether pkg is less than, equal to, @@ -478,8 +482,11 @@ func (pkg *Package) String() string { return fmt.Sprintf("package %v", path) } -func (pkg *Package) setJSONOptions(opts JSONOptions) { +func (pkg *Package) setJSONOptions(opts astJSON.Options) { pkg.jsonOptions = opts + if pkg.Location != nil { + pkg.Location.JSONOptions = opts + } } func (pkg *Package) MarshalJSON() ([]byte, error) { @@ -588,8 +595,11 @@ func (imp *Import) String() string { return strings.Join(buf, " ") } -func (imp *Import) setJSONOptions(opts JSONOptions) { +func (imp *Import) setJSONOptions(opts astJSON.Options) { imp.jsonOptions = opts + if imp.Location != nil { + imp.Location.JSONOptions = opts + } } func (imp *Import) MarshalJSON() ([]byte, error) { @@ -699,8 +709,11 @@ func (rule *Rule) String() string { return strings.Join(buf, " ") } -func (rule *Rule) setJSONOptions(opts JSONOptions) { +func (rule *Rule) setJSONOptions(opts astJSON.Options) { rule.jsonOptions = opts + if rule.Location != nil { + rule.Location.JSONOptions = opts + } } func (rule *Rule) MarshalJSON() ([]byte, error) { @@ -769,9 +782,9 @@ func NewHead(name Var, args ...*Term) *Head { return head } -// VarHead creates a head object, initializes its Name, Location, and JSONOptions, +// VarHead creates a head object, initializes its Name, Location, and Options, // and returns the new head. -func VarHead(name Var, location *Location, jsonOpts *JSONOptions) *Head { +func VarHead(name Var, location *Location, jsonOpts *astJSON.Options) *Head { h := NewHead(name) h.Reference[0].Location = location if jsonOpts != nil { @@ -926,8 +939,11 @@ func (head *Head) String() string { return buf.String() } -func (head *Head) setJSONOptions(opts JSONOptions) { +func (head *Head) setJSONOptions(opts astJSON.Options) { head.jsonOptions = opts + if head.Location != nil { + head.Location.JSONOptions = opts + } } func (head *Head) MarshalJSON() ([]byte, error) { @@ -1476,8 +1492,11 @@ func (expr *Expr) String() string { return strings.Join(buf, " ") } -func (expr *Expr) setJSONOptions(opts JSONOptions) { +func (expr *Expr) setJSONOptions(opts astJSON.Options) { expr.jsonOptions = opts + if expr.Location != nil { + expr.Location.JSONOptions = opts + } } func (expr *Expr) MarshalJSON() ([]byte, error) { @@ -1572,8 +1591,11 @@ func (d *SomeDecl) Hash() int { return termSliceHash(d.Symbols) } -func (d *SomeDecl) setJSONOptions(opts JSONOptions) { +func (d *SomeDecl) setJSONOptions(opts astJSON.Options) { d.jsonOptions = opts + if d.Location != nil { + d.Location.JSONOptions = opts + } } func (d *SomeDecl) MarshalJSON() ([]byte, error) { @@ -1646,8 +1668,11 @@ func (q *Every) KeyValueVars() VarSet { return vis.vars } -func (q *Every) setJSONOptions(opts JSONOptions) { +func (q *Every) setJSONOptions(opts astJSON.Options) { q.jsonOptions = opts + if q.Location != nil { + q.Location.JSONOptions = opts + } } func (q *Every) MarshalJSON() ([]byte, error) { @@ -1725,8 +1750,11 @@ func (w *With) SetLoc(loc *Location) { w.Location = loc } -func (w *With) setJSONOptions(opts JSONOptions) { +func (w *With) setJSONOptions(opts astJSON.Options) { w.jsonOptions = opts + if w.Location != nil { + w.Location.JSONOptions = opts + } } func (w *With) MarshalJSON() ([]byte, error) { diff --git a/ast/term.go b/ast/term.go index e89366aa74..7c50b14877 100644 --- a/ast/term.go +++ b/ast/term.go @@ -22,6 +22,7 @@ import ( "github.com/OneOfOne/xxhash" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/ast/location" "github.com/open-policy-agent/opa/util" ) @@ -294,7 +295,7 @@ type Term struct { Value Value `json:"value"` // the value of the Term as represented in Go Location *Location `json:"location,omitempty"` // the location of the Term in the source - jsonOptions JSONOptions + jsonOptions astJSON.Options } // NewTerm returns a new Term object. @@ -419,8 +420,11 @@ func (term *Term) IsGround() bool { return term.Value.IsGround() } -func (term *Term) setJSONOptions(opts JSONOptions) { +func (term *Term) setJSONOptions(opts astJSON.Options) { term.jsonOptions = opts + if term.Location != nil { + term.Location.JSONOptions = opts + } } // MarshalJSON returns the JSON encoding of the term. diff --git a/bundle/bundle.go b/bundle/bundle.go index 5567a2ae59..66c8ce37c6 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/format" "github.com/open-policy-agent/opa/internal/file/archive" "github.com/open-policy-agent/opa/internal/merge" @@ -391,7 +392,7 @@ type Reader struct { verificationConfig *VerificationConfig skipVerify bool processAnnotations bool - jsonOptions *ast.JSONOptions + jsonOptions *astJSON.Options capabilities *ast.Capabilities files map[string]FileInfo // files in the bundle signature payload sizeLimitBytes int64 @@ -463,7 +464,7 @@ func (r *Reader) WithCapabilities(caps *ast.Capabilities) *Reader { } // WithJSONOptions sets the JSONOptions to use when parsing policy files -func (r *Reader) WithJSONOptions(opts *ast.JSONOptions) *Reader { +func (r *Reader) WithJSONOptions(opts *astJSON.Options) *Reader { r.jsonOptions = opts return r } diff --git a/cmd/parse.go b/cmd/parse.go index ef529c8ea7..700794d550 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" pr "github.com/open-policy-agent/opa/internal/presentation" "github.com/open-policy-agent/opa/loader" "github.com/open-policy-agent/opa/util" @@ -72,9 +73,10 @@ func parse(args []string, params *parseParams, stdout io.Writer, stderr io.Write parserOpts := ast.ParserOptions{ProcessAnnotation: true} if exposeLocation { - parserOpts.JSONOptions = &ast.JSONOptions{ - MarshalOptions: ast.JSONMarshalOptions{ - IncludeLocation: ast.NodeToggle{ + parserOpts.JSONOptions = &astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocationText: true, + IncludeLocation: astJSON.NodeToggle{ Term: true, Package: true, Comment: true, diff --git a/cmd/parse_test.go b/cmd/parse_test.go index bfe6b3ff7e..b5371fbce4 100644 --- a/cmd/parse_test.go +++ b/cmd/parse_test.go @@ -129,9 +129,9 @@ func TestParseJSONOutputWithLocations(t *testing.T) { files := map[string]string{ "x.rego": `package x - - p = 1 - `, + +p = 1 +`, } errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), @@ -149,14 +149,16 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 1 + "col": 1, + "text": "cGFja2FnZQ==" }, "path": [ { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "var", "value": "data" @@ -165,7 +167,8 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "string", "value": "x" @@ -180,13 +183,15 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 5, + "text": "MQ==" }, "terms": { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 5, + "text": "MQ==" }, "type": "boolean", "value": true @@ -199,7 +204,8 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 5, + "text": "MQ==" }, "type": "number", "value": 1 @@ -209,7 +215,8 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "cA==" }, "type": "var", "value": "p" @@ -218,21 +225,36 @@ func TestParseJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "cCA9IDE=" } }, "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "cCA9IDE=" } } ] } `, "TEMPDIR", tempDirPath, -1) - if got, want := string(stdout), expectedOutput; got != want { - t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + gotLines := strings.Split(string(stdout), "\n") + wantLines := strings.Split(expectedOutput, "\n") + min := len(gotLines) + if len(wantLines) < min { + min = len(wantLines) + } + + for i := 0; i < min; i++ { + if gotLines[i] != wantLines[i] { + t.Fatalf("Expected line %d to be\n%v\n, got\n%v", i, wantLines[i], gotLines[i]) + } + } + + if len(gotLines) != len(wantLines) { + t.Fatalf("Expected %d lines, got %d", len(wantLines), len(gotLines)) } } @@ -313,9 +335,9 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { files := map[string]string{ "x.rego": `package x - - a.b.c := true - `, + +a.b.c := true +`, } errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), @@ -333,14 +355,16 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 1 + "col": 1, + "text": "cGFja2FnZQ==" }, "path": [ { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "var", "value": "data" @@ -349,7 +373,8 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "string", "value": "x" @@ -364,13 +389,15 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 14 + "col": 10, + "text": "dHJ1ZQ==" }, "terms": { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 14 + "col": 10, + "text": "dHJ1ZQ==" }, "type": "boolean", "value": true @@ -382,7 +409,8 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 14 + "col": 10, + "text": "dHJ1ZQ==" }, "type": "boolean", "value": true @@ -393,7 +421,8 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 5 + "col": 1, + "text": "YQ==" }, "type": "var", "value": "a" @@ -402,7 +431,8 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 7 + "col": 3, + "text": "Yg==" }, "type": "string", "value": "b" @@ -411,7 +441,8 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 9 + "col": 5, + "text": "Yw==" }, "type": "string", "value": "c" @@ -420,21 +451,36 @@ func TestParseRefsJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 5 + "col": 1, + "text": "YS5iLmMgOj0gdHJ1ZQ==" } }, "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 5 + "col": 1, + "text": "YS5iLmMgOj0gdHJ1ZQ==" } } ] } `, "TEMPDIR", tempDirPath, -1) - if got, want := string(stdout), expectedOutput; got != want { - t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + gotLines := strings.Split(string(stdout), "\n") + wantLines := strings.Split(expectedOutput, "\n") + min := len(gotLines) + if len(wantLines) < min { + min = len(wantLines) + } + + for i := 0; i < min; i++ { + if gotLines[i] != wantLines[i] { + t.Fatalf("Expected line %d to be\n%v\n, got\n%v", i, wantLines[i], gotLines[i]) + } + } + + if len(gotLines) != len(wantLines) { + t.Fatalf("Expected %d lines, got %d", len(wantLines), len(gotLines)) } } func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { @@ -442,13 +488,13 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { files := map[string]string{ "x.rego": `package x - default allow = false - allow = true { - input.method == "GET" - input.path = ["getUser", user] - input.user == user - } - `, +default allow = false +allow = true { + input.method == "GET" + input.path = ["getUser", user] + input.user == user +} +`, } errc, stdout, stderr, tempDirPath := testParse(t, files, &parseParams{ format: util.NewEnumFlag(parseFormatJSON, []string{parseFormatPretty, parseFormatJSON}), @@ -466,14 +512,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 1 + "col": 1, + "text": "cGFja2FnZQ==" }, "path": [ { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "var", "value": "data" @@ -482,7 +530,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 1, - "col": 9 + "col": 9, + "text": "eA==" }, "type": "string", "value": "x" @@ -497,13 +546,15 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "ZGVmYXVsdA==" }, "terms": { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "ZGVmYXVsdA==" }, "type": "boolean", "value": true @@ -517,7 +568,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 19 + "col": 17, + "text": "ZmFsc2U=" }, "type": "boolean", "value": false @@ -527,7 +579,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 11 + "col": 9, + "text": "YWxsb3c=" }, "type": "var", "value": "allow" @@ -536,13 +589,15 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 11 + "col": 9, + "text": "YWxsb3cgPSBmYWxzZQ==" } }, "location": { "file": "TEMPDIR/x.rego", "row": 3, - "col": 3 + "col": 1, + "text": "ZGVmYXVsdA==" } }, { @@ -552,14 +607,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 7 + "col": 3, + "text": "aW5wdXQubWV0aG9kID09ICJHRVQi" }, "terms": [ { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 20 + "col": 16, + "text": "PT0=" }, "type": "ref", "value": [ @@ -567,7 +624,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 20 + "col": 16, + "text": "PT0=" }, "type": "var", "value": "equal" @@ -578,7 +636,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 7 + "col": 3, + "text": "aW5wdXQubWV0aG9k" }, "type": "ref", "value": [ @@ -586,7 +645,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 7 + "col": 3, + "text": "aW5wdXQ=" }, "type": "var", "value": "input" @@ -595,7 +655,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 13 + "col": 9, + "text": "bWV0aG9k" }, "type": "string", "value": "method" @@ -606,7 +667,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 5, - "col": 23 + "col": 19, + "text": "IkdFVCI=" }, "type": "string", "value": "GET" @@ -618,14 +680,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 7 + "col": 3, + "text": "aW5wdXQucGF0aCA9IFsiZ2V0VXNlciIsIHVzZXJd" }, "terms": [ { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 18 + "col": 14, + "text": "PQ==" }, "type": "ref", "value": [ @@ -633,7 +697,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 18 + "col": 14, + "text": "PQ==" }, "type": "var", "value": "eq" @@ -644,7 +709,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 7 + "col": 3, + "text": "aW5wdXQucGF0aA==" }, "type": "ref", "value": [ @@ -652,7 +718,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 7 + "col": 3, + "text": "aW5wdXQ=" }, "type": "var", "value": "input" @@ -661,7 +728,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 13 + "col": 9, + "text": "cGF0aA==" }, "type": "string", "value": "path" @@ -672,7 +740,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 20 + "col": 16, + "text": "WyJnZXRVc2VyIiwgdXNlcl0=" }, "type": "array", "value": [ @@ -680,7 +749,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 21 + "col": 17, + "text": "ImdldFVzZXIi" }, "type": "string", "value": "getUser" @@ -689,7 +759,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 6, - "col": 32 + "col": 28, + "text": "dXNlcg==" }, "type": "var", "value": "user" @@ -703,14 +774,16 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 7 + "col": 3, + "text": "aW5wdXQudXNlciA9PSB1c2Vy" }, "terms": [ { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 18 + "col": 14, + "text": "PT0=" }, "type": "ref", "value": [ @@ -718,7 +791,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 18 + "col": 14, + "text": "PT0=" }, "type": "var", "value": "equal" @@ -729,7 +803,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 7 + "col": 3, + "text": "aW5wdXQudXNlcg==" }, "type": "ref", "value": [ @@ -737,7 +812,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 7 + "col": 3, + "text": "aW5wdXQ=" }, "type": "var", "value": "input" @@ -746,7 +822,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 13 + "col": 9, + "text": "dXNlcg==" }, "type": "string", "value": "user" @@ -757,7 +834,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 7, - "col": 21 + "col": 17, + "text": "dXNlcg==" }, "type": "var", "value": "user" @@ -771,7 +849,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 13 + "col": 9, + "text": "dHJ1ZQ==" }, "type": "boolean", "value": true @@ -781,7 +860,8 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 5 + "col": 1, + "text": "YWxsb3c=" }, "type": "var", "value": "allow" @@ -790,21 +870,36 @@ func TestParseRulesBlockJSONOutputWithLocations(t *testing.T) { "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 5 + "col": 1, + "text": "YWxsb3cgPSB0cnVl" } }, "location": { "file": "TEMPDIR/x.rego", "row": 4, - "col": 5 + "col": 1, + "text": "YWxsb3cgPSB0cnVlIHsKICBpbnB1dC5tZXRob2QgPT0gIkdFVCIKICBpbnB1dC5wYXRoID0gWyJnZXRVc2VyIiwgdXNlcl0KICBpbnB1dC51c2VyID09IHVzZXIKfQ==" } } ] } `, "TEMPDIR", tempDirPath, -1) - if got, want := string(stdout), expectedOutput; got != want { - t.Fatalf("Expected output\n%v\n, got\n%v", want, got) + gotLines := strings.Split(string(stdout), "\n") + wantLines := strings.Split(expectedOutput, "\n") + min := len(gotLines) + if len(wantLines) < min { + min = len(wantLines) + } + + for i := 0; i < min; i++ { + if gotLines[i] != wantLines[i] { + t.Fatalf("Expected line %d to be\n%v\n, got\n%v", i, wantLines[i], gotLines[i]) + } + } + + if len(gotLines) != len(wantLines) { + t.Fatalf("Expected %d lines, got %d", len(wantLines), len(gotLines)) } } diff --git a/internal/bundle/inspect/inspect.go b/internal/bundle/inspect/inspect.go index f6d61d54e3..da2dca40d9 100644 --- a/internal/bundle/inspect/inspect.go +++ b/internal/bundle/inspect/inspect.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/bundle" initload "github.com/open-policy-agent/opa/internal/runtime/init" "github.com/open-policy-agent/opa/loader" @@ -32,9 +33,9 @@ func File(path string, includeAnnotations bool) (*Info, error) { b, err := loader.NewFileLoader(). WithSkipBundleVerification(true). WithProcessAnnotation(true). // Always process annotations, for enriching namespace listing - WithJSONOptions(&ast.JSONOptions{ - MarshalOptions: ast.JSONMarshalOptions{ - IncludeLocation: ast.NodeToggle{ + WithJSONOptions(&json.Options{ + MarshalOptions: json.MarshalOptions{ + IncludeLocation: json.NodeToggle{ // Annotation location data is only included if includeAnnotations is set AnnotationsRef: includeAnnotations, }, diff --git a/loader/loader.go b/loader/loader.go index df45259df4..4583f21b44 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -18,6 +18,7 @@ import ( "sigs.k8s.io/yaml" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/bundle" fileurl "github.com/open-policy-agent/opa/internal/file/url" "github.com/open-policy-agent/opa/internal/merge" @@ -100,7 +101,7 @@ type FileLoader interface { WithSkipBundleVerification(bool) FileLoader WithProcessAnnotation(bool) FileLoader WithCapabilities(*ast.Capabilities) FileLoader - WithJSONOptions(*ast.JSONOptions) FileLoader + WithJSONOptions(*astJSON.Options) FileLoader } // NewFileLoader returns a new FileLoader instance. @@ -175,7 +176,7 @@ func (fl *fileLoader) WithCapabilities(caps *ast.Capabilities) FileLoader { } // WithJSONOptions sets the JSONOptions for use when parsing files -func (fl *fileLoader) WithJSONOptions(opts *ast.JSONOptions) FileLoader { +func (fl *fileLoader) WithJSONOptions(opts *astJSON.Options) FileLoader { fl.opts.JSONOptions = opts return fl } diff --git a/loader/loader_test.go b/loader/loader_test.go index 64c7cf9b90..323ba96b30 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/open-policy-agent/opa/ast" + astJSON "github.com/open-policy-agent/opa/ast/json" "github.com/open-policy-agent/opa/bundle" "github.com/open-policy-agent/opa/util" "github.com/open-policy-agent/opa/util/test" @@ -655,9 +656,9 @@ func TestLoadWithJSONOptions(t *testing.T) { } // load the file with JSON options set to include location data - loaded, err := NewFileLoader().WithFS(fsys).WithJSONOptions(&ast.JSONOptions{ - MarshalOptions: ast.JSONMarshalOptions{ - IncludeLocation: ast.NodeToggle{ + loaded, err := NewFileLoader().WithFS(fsys).WithJSONOptions(&astJSON.Options{ + MarshalOptions: astJSON.MarshalOptions{ + IncludeLocation: astJSON.NodeToggle{ Package: true, }, },