Skip to content

Add omni schema plugin's validation code for transform_declarations #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 3, 2020
Merged

Add omni schema plugin's validation code for transform_declarations #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/antchfx/xmlquery v1.3.0
github.com/antchfx/xpath v1.1.10
github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
github.com/google/uuid v1.1.2
github.com/hashicorp/golang-lru v0.5.4
github.com/jf-tech/iohelper v1.0.3
github.com/stretchr/testify v1.6.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/jf-tech/iohelper v1.0.3 h1:304dQL6ZKJEmDkbIivqCGrhmJTi7k7+1e2CC+WiERS4=
Expand Down
7 changes: 2 additions & 5 deletions omniparser/schemaplugin/omni/v2/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,16 @@ package omniv2
import (
"github.com/jf-tech/omniparser/omniparser/errs"
"github.com/jf-tech/omniparser/omniparser/schemaplugin"
"github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2/transform"
)

const (
pluginVersion = "omni.2.0"
fileFormatXML = "xml"
)

// ParseSchema parses, validates and creates an omni-schema based schema plugin.
func ParseSchema(_ *schemaplugin.ParseSchemaCtx) (schemaplugin.Plugin, error) {
return nil, errs.ErrSchemaNotSupported
}

type omniSchema struct {
schemaplugin.Header
Decls map[string]*transform.Decl `json:"transform_declarations"`
}
type schemaPlugin struct{}
235 changes: 235 additions & 0 deletions ...arser/schemaplugin/omni/v2/transform/.snapshots/TestValidateTransformDeclarations-success
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
{
"object": {
"field1": {
"const": "value1",
"fqdn": "FINAL_OUTPUT.field1",
"kind": "const",
"parent": "FINAL_OUTPUT"
},
"field2": {
"xpath_dynamic": {
"custom_func": {
"name": "test_func",
"args": [
{
"xpath": "W/X",
"fqdn": "FINAL_OUTPUT.field2.xpath_dynamic.custom_func(test_func).arg[1]",
"kind": "field",
"parent": "(nil)"
}
],
"fqdn": "FINAL_OUTPUT.field2.xpath_dynamic.custom_func(test_func)"
},
"fqdn": "FINAL_OUTPUT.field2.xpath_dynamic",
"kind": "custom_func",
"children": [
"FINAL_OUTPUT.field2.xpath_dynamic.custom_func(test_func).arg[1]"
],
"parent": "(nil)"
},
"fqdn": "FINAL_OUTPUT.field2",
"kind": "field",
"parent": "FINAL_OUTPUT"
},
"field3": {
"xpath": "E/F/G",
"object": {
"field4": {
"array": [
{
"const": "value4",
"fqdn": "FINAL_OUTPUT.field3.field4.elem[1]",
"kind": "const",
"parent": "FINAL_OUTPUT.field3.field4"
},
{
"xpath": "H/I/J",
"fqdn": "FINAL_OUTPUT.field3.field4.elem[2]",
"kind": "field",
"parent": "FINAL_OUTPUT.field3.field4"
},
{
"xpath": "K/L/M",
"object": {
"field5": {
"xpath": "N/O/P",
"fqdn": "FINAL_OUTPUT.field3.field4.elem[3].field5",
"kind": "field",
"parent": "FINAL_OUTPUT.field3.field4.elem[3]"
}
},
"fqdn": "FINAL_OUTPUT.field3.field4.elem[3]",
"kind": "object",
"children": [
"FINAL_OUTPUT.field3.field4.elem[3].field5"
],
"parent": "FINAL_OUTPUT.field3.field4"
},
{
"xpath": "1/2/3",
"object": {
"field9": {
"xpath": "4/5/6",
"fqdn": "FINAL_OUTPUT.field3.field4.elem[4].field9",
"kind": "field",
"parent": "FINAL_OUTPUT.field3.field4.elem[4]"
}
},
"fqdn": "FINAL_OUTPUT.field3.field4.elem[4]",
"kind": "object",
"children": [
"FINAL_OUTPUT.field3.field4.elem[4].field9"
],
"parent": "FINAL_OUTPUT.field3.field4"
}
],
"fqdn": "FINAL_OUTPUT.field3.field4",
"kind": "array",
"children": [
"FINAL_OUTPUT.field3.field4.elem[1]",
"FINAL_OUTPUT.field3.field4.elem[2]",
"FINAL_OUTPUT.field3.field4.elem[3]",
"FINAL_OUTPUT.field3.field4.elem[4]"
],
"parent": "FINAL_OUTPUT.field3"
}
},
"fqdn": "FINAL_OUTPUT.field3",
"kind": "object",
"children": [
"FINAL_OUTPUT.field3.field4"
],
"parent": "FINAL_OUTPUT"
},
"field6": {
"custom_func": {
"name": "test_func",
"args": [
{
"xpath": "Q/R/S",
"fqdn": "FINAL_OUTPUT.field6.custom_func(test_func).arg[1]",
"kind": "field",
"parent": "FINAL_OUTPUT.field6"
},
{
"custom_func": {
"name": "test_func",
"args": [
{
"xpath": "W/X",
"fqdn": "FINAL_OUTPUT.field6.custom_func(test_func).arg[2].custom_func(test_func).arg[1]",
"kind": "field",
"parent": "FINAL_OUTPUT.field6.custom_func(test_func).arg[2]"
}
],
"fqdn": "FINAL_OUTPUT.field6.custom_func(test_func).arg[2].custom_func(test_func)"
},
"fqdn": "FINAL_OUTPUT.field6.custom_func(test_func).arg[2]",
"kind": "custom_func",
"children": [
"FINAL_OUTPUT.field6.custom_func(test_func).arg[2].custom_func(test_func).arg[1]"
],
"parent": "FINAL_OUTPUT.field6"
}
],
"fqdn": "FINAL_OUTPUT.field6.custom_func(test_func)"
},
"fqdn": "FINAL_OUTPUT.field6",
"kind": "custom_func",
"children": [
"FINAL_OUTPUT.field6.custom_func(test_func).arg[1]",
"FINAL_OUTPUT.field6.custom_func(test_func).arg[2]"
],
"parent": "FINAL_OUTPUT"
},
"field_10": {
"xpath_dynamic": {
"const": "X/Y/Z",
"fqdn": "FINAL_OUTPUT.field_10.xpath_dynamic",
"kind": "const",
"parent": "(nil)"
},
"object": {
"field10": {
"const": "value10",
"fqdn": "FINAL_OUTPUT.field_10.field10",
"kind": "const",
"parent": "FINAL_OUTPUT.field_10"
}
},
"fqdn": "FINAL_OUTPUT.field_10",
"kind": "object",
"children": [
"FINAL_OUTPUT.field_10.field10"
],
"parent": "FINAL_OUTPUT"
},
"field_11": {
"array": [
{
"xpath": "T/U/V",
"fqdn": "FINAL_OUTPUT.field_11.elem[1]",
"kind": "field",
"parent": "FINAL_OUTPUT.field_11"
}
],
"fqdn": "FINAL_OUTPUT.field_11",
"kind": "array",
"children": [
"FINAL_OUTPUT.field_11.elem[1]"
],
"parent": "FINAL_OUTPUT"
},
"field_12": {
"custom_func": {
"name": "test_func",
"args": [
{
"xpath": "W/X",
"fqdn": "FINAL_OUTPUT.field_12.custom_func(test_func).arg[1]",
"kind": "field",
"parent": "FINAL_OUTPUT.field_12"
}
],
"fqdn": "FINAL_OUTPUT.field_12.custom_func(test_func)"
},
"fqdn": "FINAL_OUTPUT.field_12",
"kind": "custom_func",
"children": [
"FINAL_OUTPUT.field_12.custom_func(test_func).arg[1]"
],
"parent": "FINAL_OUTPUT"
},
"field_9": {
"xpath": "1/2/3",
"object": {
"field9": {
"xpath": "4/5/6",
"fqdn": "FINAL_OUTPUT.field_9.field9",
"kind": "field",
"parent": "FINAL_OUTPUT.field_9"
}
},
"fqdn": "FINAL_OUTPUT.field_9",
"kind": "object",
"children": [
"FINAL_OUTPUT.field_9.field9"
],
"parent": "FINAL_OUTPUT"
}
},
"fqdn": "FINAL_OUTPUT",
"kind": "object",
"children": [
"FINAL_OUTPUT.field1",
"FINAL_OUTPUT.field2",
"FINAL_OUTPUT.field3",
"FINAL_OUTPUT.field6",
"FINAL_OUTPUT.field_10",
"FINAL_OUTPUT.field_11",
"FINAL_OUTPUT.field_12",
"FINAL_OUTPUT.field_9"
],
"parent": "(nil)"
}

41 changes: 26 additions & 15 deletions omniparser/schemaplugin/omni/v2/transform/decl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import (
"github.com/jf-tech/omniparser/strs"
)

// Kind specifies the types of omni schema's input elements.
type Kind string

const (
KindUnknown Kind = "unknown"
KindConst Kind = "const"
KindExternal Kind = "external"
KindField Kind = "field"
Expand All @@ -18,6 +20,7 @@ const (
KindTemplate Kind = "template"
)

// ResultType specifies the types of omni schema's output elements.
type ResultType string

const (
Expand All @@ -31,16 +34,20 @@ const (
)

const (
FinalOutput = "FINAL_OUTPUT"
// finalOutput is the special name of a Decl that is designated for the output
// for an omni schema.
finalOutput = "FINAL_OUTPUT"
)

// CustomFuncDecl is the decl for a "custom_func".
type CustomFuncDecl struct {
Name string `json:"name,omitempty"`
Args []*Decl `json:"args,omitempty"`
IgnoreErrorAndReturnEmptyStr bool `json:"ignore_error_and_return_empty_str,omitempty"`
fqdn string // internal; never unmarshaled from a schema.
}

// MarshalJSON is the custom JSON marshaler for CustomFuncDecl.
func (d CustomFuncDecl) MarshalJSON() ([]byte, error) {
type Alias CustomFuncDecl
return json.Marshal(&struct {
Expand All @@ -64,28 +71,30 @@ func (d *CustomFuncDecl) deepCopy() *CustomFuncDecl {
return dest
}

// This is the struct will be unmarshaled from `transform_declarations` section of an omni schema.
// Decl is the type for omni schema's `transform_declarations` declarations.
type Decl struct {
// Applicable for KindConst.
// Const indicates the input element is a cost.
Const *string `json:"const,omitempty"`
// Applicable for KindExternal
// External indicates the input element is from an external property.
External *string `json:"external,omitempty"`
// Applicable for KindField, KindObject, KindTemplate, KindCustomFunc
// XPath specifies an xpath for an input element.
XPath *string `json:"xpath,omitempty"`
// Applicable for KindField, KindObject, KindTemplate, KindCustomFunc
// XPathDynamic specifies a dynamically constructed xpath for an input element.
XPathDynamic *Decl `json:"xpath_dynamic,omitempty"`
// Applicable for KindCustomFunc.
// CustomFunc specifies the input element is a custom function.
CustomFunc *CustomFuncDecl `json:"custom_func,omitempty"`
// Applicable for KindTemplate.
// Template specifies the input element is a template.
Template *string `json:"template,omitempty"`
// Applicable for KindObject.
// Object specifies the input element is an object.
Object map[string]*Decl `json:"object,omitempty"`
// Applicable for KindArray.
// Array specifies the input element is an array.
Array []*Decl `json:"array,omitempty"`
// Applicable for KindConst, KindExternal, KindField or KindCustomFunc.
ResultType *ResultType `json:"result_type,omitempty"`
KeepLeadingTrailingSpace bool `json:"keep_leading_trailing_space,omitempty"`
KeepEmptyOrNull bool `json:"keep_empty_or_null,omitempty"`
// ResultType specifies the desired output type of an element.
ResultType *ResultType `json:"result_type,omitempty"`
// KeepLeadingTrailingSpace specifies space trimming in string value of the output element.
KeepLeadingTrailingSpace bool `json:"keep_leading_trailing_space,omitempty"`
// KeepEmptyOrNull specifies whether or not keep an empty/null output or not.
KeepEmptyOrNull bool `json:"keep_empty_or_null,omitempty"`

// Internal runtime fields that are not unmarshaled from a schema.
fqdn string
Expand All @@ -95,6 +104,7 @@ type Decl struct {
parent *Decl
}

// MarshalJSON is the custom JSON marshaler for Decl.
func (d Decl) MarshalJSON() ([]byte, error) {
emptyToNil := func(s string) string {
return strs.FirstNonBlank(s, "(nil)")
Expand Down Expand Up @@ -161,7 +171,8 @@ func (d *Decl) isXPathSet() bool {
return d.XPath != nil || d.XPathDynamic != nil
}

// Note only deep-copy all the public fields, those internal computed fields are not copied.
// Note only deep-copy all the public fields, those internal computed fields MUST not be copied:
// see explanation in validate.go's computeDeclHash().
func (d *Decl) deepCopy() *Decl {
dest := &Decl{}
dest.Const = strs.CopyStrPtr(d.Const)
Expand Down
Loading