From 9491a05ec75f2b7b75d376d02b0c1be78ef168fd Mon Sep 17 00:00:00 2001 From: "curtis.brown" Date: Mon, 27 Nov 2023 13:21:39 -0500 Subject: [PATCH] Add support for unexpected elements --- .gitignore | 2 + cmd/goxmlstruct/main.go | 4 + element.go | 21 ++++- generator.go | 100 ++++++++++++++++++++ generator_test.go | 204 ++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- value.go | 22 +++-- xmlstruct.go | 4 + 8 files changed, 346 insertions(+), 13 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..65bce91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.idea +/cmd/goxmlstruct/goxmlstruct \ No newline at end of file diff --git a/cmd/goxmlstruct/main.go b/cmd/goxmlstruct/main.go index ae1f3ed..ee686b1 100644 --- a/cmd/goxmlstruct/main.go +++ b/cmd/goxmlstruct/main.go @@ -23,6 +23,8 @@ var ( topLevelAttributes = flag.Bool("top-level-attributes", xmlstruct.DefaultTopLevelAttributes, "include top level attributes") usePointersForOptionalFields = flag.Bool("use-pointers-for-optional-fields", xmlstruct.DefaultUsePointersForOptionalFields, "use pointers for optional fields") useRawToken = flag.Bool("use-raw-token", xmlstruct.DefaultUseRawToken, "use encoding/xml.Decoder.RawToken") + supportUnexpectedElements = flag.Bool("support-unexpected-elements", xmlstruct.DefaultSupportUnexpectedElements, "create field of Node type with xml:any tag on each struct to handle unexpected elements on parse") + unexpectedElementTypeName = flag.String("unexpected-element-type-name", xmlstruct.DefaultUnexpectedElementTypeName, "name for the namedType to contain any unexpected elements encountered on parsing; default: UnexpectedElement") ) func run() error { @@ -46,6 +48,8 @@ func run() error { xmlstruct.WithTopLevelAttributes(*topLevelAttributes), xmlstruct.WithUsePointersForOptionalFields(*usePointersForOptionalFields), xmlstruct.WithUseRawToken(*useRawToken), + xmlstruct.WithSupportUnexpectedElements(*supportUnexpectedElements), + xmlstruct.WithUnexpectedElementTypeName(*unexpectedElementTypeName), ) if flag.NArg() == 0 { diff --git a/element.go b/element.go index e4d2b07..e41050e 100644 --- a/element.go +++ b/element.go @@ -4,10 +4,9 @@ import ( "bytes" "encoding/xml" "fmt" - "io" - "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "io" ) // An element describes an observed XML element, its attributes, chardata, and @@ -214,7 +213,23 @@ func (e *element) writeGoType(w io.Writer, options *generateOptions, indentPrefi return err } } - fmt.Fprintf(w, " `xml:\"%s\"`\n", childElement.name.Local) + + if e.name.Local == options.unexpectedElementTypeName { + switch childElement.name.Local { + case "Attrs": + fmt.Fprint(w, " `xml:\",any,attr\"`\n") + case "Content": + fmt.Fprintf(w, " `xml:\",innerxml\"`\n") + case "Nodes": + fmt.Fprintf(w, " `xml:\",any\"`\n") + } + } else { + if childElement.name.Local == fmt.Sprintf("%ss", options.unexpectedElementTypeName) { + fmt.Fprintf(w, " `xml:\",any\"`\n") + } else { + fmt.Fprintf(w, " `xml:\"%s\"`\n", childElement.name.Local) + } + } } fmt.Fprintf(w, "%s}", indentPrefix) diff --git a/generator.go b/generator.go index 6841375..2fe8b8f 100644 --- a/generator.go +++ b/generator.go @@ -20,6 +20,8 @@ var ( SkipDir = fs.SkipDir //lint:ignore ST1012 SkipFile is not an error SkipFile = errors.New("skip file") //nolint:errname + + unexpectedElement, unexpectedElements *element ) // A Generator observes XML documents and generates Go structs into which the @@ -41,6 +43,8 @@ type Generator struct { typeOrder map[xml.Name]int usePointersForOptionalFields bool useRawToken bool + supportUnexpectedElements bool + unexpectedElementTypeName string typeElements map[xml.Name]*element } @@ -148,6 +152,23 @@ func WithUseRawToken(useRawToken bool) GeneratorOption { } } +// WithSupportUnexpectedElements sets whether to support unexpected elements +// on structs through use of a field of the catch-all type Node and xml:any +// struct tag on each generated struct +func WithSupportUnexpectedElements(supportUnexpectedElements bool) GeneratorOption { + return func(g *Generator) { + g.supportUnexpectedElements = supportUnexpectedElements + } +} + +// WithUnexpectedElementTypeName specifies the name of the named type to contain +// any unexpected elements encountered during parsing +func WithUnexpectedElementTypeName(unexpectedElementTypeName string) GeneratorOption { + return func(g *Generator) { + g.unexpectedElementTypeName = unexpectedElementTypeName + } +} + // NewGenerator returns a new Generator with the given options. func NewGenerator(options ...GeneratorOption) *Generator { g := &Generator{ @@ -164,6 +185,8 @@ func NewGenerator(options ...GeneratorOption) *Generator { typeOrder: make(map[xml.Name]int), usePointersForOptionalFields: DefaultUsePointersForOptionalFields, useRawToken: DefaultUseRawToken, + supportUnexpectedElements: DefaultSupportUnexpectedElements, + unexpectedElementTypeName: DefaultUnexpectedElementTypeName, typeElements: make(map[xml.Name]*element), } g.exportNameFunc = func(name xml.Name) string { @@ -189,9 +212,21 @@ func (g *Generator) Generate() ([]byte, error) { intType: g.intType, preserveOrder: g.preserveOrder, usePointersForOptionalFields: g.usePointersForOptionalFields, + supportUnexpectedElements: g.supportUnexpectedElements, + unexpectedElementTypeName: g.unexpectedElementTypeName, } var typeElements []*element + + if g.supportUnexpectedElements { + initializeUnexpectedElements(g.unexpectedElementTypeName) + options.importPackageNames["encoding/xml"] = struct{}{} + options.namedTypes = make(map[xml.Name]*element) + options.namedTypes[xml.Name{Local: g.unexpectedElementTypeName}] = unexpectedElement + g.typeElements[xml.Name{Local: g.unexpectedElementTypeName}] = unexpectedElement + appendUnexpectedElements(&g.typeElements, g.namedTypes, g.unexpectedElementTypeName) + } + if g.namedTypes { options.namedTypes = maps.Clone(g.typeElements) options.simpleTypes = make(map[xml.Name]struct{}) @@ -216,6 +251,13 @@ func (g *Generator) Generate() ([]byte, error) { aExportedName := options.exportNameFunc(a.name) bExportedName := options.exportNameFunc(b.name) switch { + // Force unexpected element struct to the top of the source + case aExportedName == g.unexpectedElementTypeName && bExportedName != g.unexpectedElementTypeName: + return -1 + case aExportedName == g.unexpectedElementTypeName && bExportedName == g.unexpectedElementTypeName: + return 0 + case aExportedName != g.unexpectedElementTypeName && bExportedName == g.unexpectedElementTypeName: + return 1 case aExportedName < bExportedName: return -1 case aExportedName == bExportedName: @@ -356,3 +398,61 @@ FOR: } } } + +func appendUnexpectedElements(elementsMap *map[xml.Name]*element, createNamedTypes bool, unexpectedElementTypeName string) { + unexpectedElementsTypeName := fmt.Sprintf("%ss", unexpectedElementTypeName) + for _, element := range *elementsMap { + if element.name.Local == unexpectedElementTypeName { + continue + } + if createNamedTypes == false { + appendUnexpectedElements(&element.childElements, createNamedTypes, unexpectedElementTypeName) + } + element.childElements[xml.Name{Local: unexpectedElementsTypeName}] = unexpectedElements + element.repeatedChildren[xml.Name{Local: unexpectedElementsTypeName}] = struct{}{} + } +} + +func initializeUnexpectedElements(UnexpectedElementTypeName string) { + unexpectedElementTypeNameWithPointerSymbol := fmt.Sprintf("*%s", UnexpectedElementTypeName) + UnexpectedElementsTypeName := fmt.Sprintf("%ss", UnexpectedElementTypeName) + unexpectedElement = &element{ + name: xml.Name{Local: UnexpectedElementTypeName}, + childElements: map[xml.Name]*element{ + xml.Name{Local: "XMLName"}: &element{ + name: xml.Name{Local: "XMLName"}, + charDataValue: value{ + unexpectedElementTypeName: "xml.Name", + }, + }, + xml.Name{Local: "Attrs"}: &element{ + name: xml.Name{Local: "Attrs"}, + charDataValue: value{ + unexpectedElementTypeName: "xml.Attr", + }, + }, + xml.Name{Local: "Content"}: &element{ + name: xml.Name{Local: "Content"}, + charDataValue: value{ + unexpectedElementTypeName: "[]byte", + }, + }, + xml.Name{Local: "Nodes"}: &element{ + name: xml.Name{Local: "Nodes"}, + charDataValue: value{ + unexpectedElementTypeName: unexpectedElementTypeNameWithPointerSymbol, + }, + }, + }, + repeatedChildren: map[xml.Name]struct{}{ + xml.Name{Local: "Attrs"}: struct{}{}, + xml.Name{Local: "Nodes"}: struct{}{}, + }, + } + unexpectedElements = &element{ + name: xml.Name{Local: UnexpectedElementsTypeName}, + charDataValue: value{ + unexpectedElementTypeName: unexpectedElementTypeNameWithPointerSymbol, + }, + } +} diff --git a/generator_test.go b/generator_test.go index 41e2a80..7f5922b 100644 --- a/generator_test.go +++ b/generator_test.go @@ -232,6 +232,210 @@ func TestGenerator(t *testing.T) { `}`, ), }, + { + name: "unexpected_elements", + options: []xmlstruct.GeneratorOption{ + xmlstruct.WithSupportUnexpectedElements(true), + }, + xmlStr: joinLines( + ``, + ` `, + ` `, + ` `, + ` `, + ` `, + ` `, + ``, + ), + expectedStr: joinLines( + xmlstruct.DefaultHeader, + ``, + `package main`, + ``, + "import \"encoding/xml\"", + ``, + `type UnexpectedElement struct {`, + "\tAttrs []xml.Attr `xml:\",any,attr\"`", + "\tContent []byte `xml:\",innerxml\"`", + "\tNodes []*UnexpectedElement `xml:\",any\"`", + "\tXMLName xml.Name", + `}`, + ``, + `type A struct {`, + "\tB struct {", + "\t\tC struct {", + "\t\t\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + "\t\t} `xml:\"c\"`", + "\t\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + "\t} `xml:\"b\"`", + "\tD struct {", + "\t\tE struct {", + "\t\t\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + "\t\t} `xml:\"e\"`", + "\t\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + "\t} `xml:\"d\"`", + "\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + `}`, + ), + }, + { + name: "unexpected_elements_custom_name", + options: []xmlstruct.GeneratorOption{ + xmlstruct.WithSupportUnexpectedElements(true), + xmlstruct.WithUnexpectedElementTypeName("MyType"), + }, + xmlStr: joinLines( + ``, + ` `, + ` `, + ` `, + ` `, + ` `, + ` `, + ``, + ), + expectedStr: joinLines( + xmlstruct.DefaultHeader, + ``, + `package main`, + ``, + "import \"encoding/xml\"", + ``, + `type MyType struct {`, + "\tAttrs []xml.Attr `xml:\",any,attr\"`", + "\tContent []byte `xml:\",innerxml\"`", + "\tNodes []*MyType `xml:\",any\"`", + "\tXMLName xml.Name", + `}`, + ``, + `type A struct {`, + "\tB struct {", + "\t\tC struct {", + "\t\t\tMyTypes []*MyType `xml:\",any\"`", + "\t\t} `xml:\"c\"`", + "\t\tMyTypes []*MyType `xml:\",any\"`", + "\t} `xml:\"b\"`", + "\tD struct {", + "\t\tE struct {", + "\t\t\tMyTypes []*MyType `xml:\",any\"`", + "\t\t} `xml:\"e\"`", + "\t\tMyTypes []*MyType `xml:\",any\"`", + "\t} `xml:\"d\"`", + "\tMyTypes []*MyType `xml:\",any\"`", + `}`, + ), + }, + { + name: "unexpected_elements_named_types", + options: []xmlstruct.GeneratorOption{ + xmlstruct.WithNamedTypes(true), + xmlstruct.WithSupportUnexpectedElements(true), + }, + xmlStr: joinLines( + ``, + ` `, + ` `, + ` `, + ` `, + ` `, + ` `, + ``, + ), + expectedStr: joinLines( + xmlstruct.DefaultHeader, + ``, + `package main`, + ``, + "import \"encoding/xml\"", + ``, + `type UnexpectedElement struct {`, + "\tAttrs []xml.Attr `xml:\",any,attr\"`", + "\tContent []byte `xml:\",innerxml\"`", + "\tNodes []*UnexpectedElement `xml:\",any\"`", + "\tXMLName xml.Name", + `}`, + ``, + `type A struct {`, + "\tB B `xml:\"b\"`", //nolint:dupword + "\tD D `xml:\"d\"`", //nolint:dupword + "\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + `}`, + ``, + `type B struct {`, + "\tC C `xml:\"c\"`", + "\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + `}`, + ``, + `type C struct {`, + "\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + `}`, + ``, + `type D struct {`, + "\tE E `xml:\"e\"`", + "\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + `}`, + ``, + `type E struct {`, + "\tUnexpectedElements []*UnexpectedElement `xml:\",any\"`", + `}`, + ), + }, + { + name: "unexpected_elements_named_types_custom_name", + options: []xmlstruct.GeneratorOption{ + xmlstruct.WithNamedTypes(true), + xmlstruct.WithSupportUnexpectedElements(true), + xmlstruct.WithUnexpectedElementTypeName("MyType"), + }, + xmlStr: joinLines( + ``, + ` `, + ` `, + ` `, + ` `, + ` `, + ` `, + ``, + ), + expectedStr: joinLines( + xmlstruct.DefaultHeader, + ``, + `package main`, + ``, + "import \"encoding/xml\"", + ``, + `type MyType struct {`, + "\tAttrs []xml.Attr `xml:\",any,attr\"`", + "\tContent []byte `xml:\",innerxml\"`", + "\tNodes []*MyType `xml:\",any\"`", + "\tXMLName xml.Name", + `}`, + ``, + `type A struct {`, + "\tB B `xml:\"b\"`", //nolint:dupword + "\tD D `xml:\"d\"`", //nolint:dupword + "\tMyTypes []*MyType `xml:\",any\"`", + `}`, + ``, + `type B struct {`, + "\tC C `xml:\"c\"`", + "\tMyTypes []*MyType `xml:\",any\"`", + `}`, + ``, + `type C struct {`, + "\tMyTypes []*MyType `xml:\",any\"`", + `}`, + ``, + `type D struct {`, + "\tE E `xml:\"e\"`", + "\tMyTypes []*MyType `xml:\",any\"`", + `}`, + ``, + `type E struct {`, + "\tMyTypes []*MyType `xml:\",any\"`", + `}`, + ), + }, // FIXME make the following test pass /* { diff --git a/go.mod b/go.mod index 32aa03d..00946f6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/twpayne/go-xmlstruct -go 1.19 +go 1.21 require ( github.com/alecthomas/assert/v2 v2.3.0 diff --git a/value.go b/value.go index 297c697..56c8bd8 100644 --- a/value.go +++ b/value.go @@ -9,15 +9,16 @@ import ( // A value describes an observed simple value, either an attribute value or // chardata. type value struct { - boolCount int - float64Count int - intCount int - name xml.Name - observations int - optional bool - repeated bool - stringCount int - timeCount int + boolCount int + float64Count int + intCount int + name xml.Name + observations int + optional bool + repeated bool + stringCount int + timeCount int + unexpectedElementTypeName string } // goType returns the most specific Go type that can represent all of the values @@ -46,6 +47,9 @@ func (v *value) goType(options *generateOptions) string { if options.usePointersForOptionalFields && v.optional { prefix += "*" } + if v.unexpectedElementTypeName != "" { + return v.unexpectedElementTypeName + } switch { case distinctTypes == 0: return "struct{}" diff --git a/xmlstruct.go b/xmlstruct.go index 1505a41..9cf75e5 100644 --- a/xmlstruct.go +++ b/xmlstruct.go @@ -24,6 +24,8 @@ const ( DefaultTimeLayout = "2006-01-02T15:04:05Z" DefaultUsePointersForOptionalFields = true DefaultUseRawToken = false + DefaultSupportUnexpectedElements = false + DefaultUnexpectedElementTypeName = "UnexpectedElement" ) var ( @@ -100,6 +102,8 @@ type generateOptions struct { preserveOrder bool simpleTypes map[xml.Name]struct{} usePointersForOptionalFields bool + supportUnexpectedElements bool + unexpectedElementTypeName string } // sortedKeys returns the keys of m in order.