Add PathStruct definition and child ctor generating code for ypathgen (…

…#315)

* Add PathStruct definition and child constructor generating code for ypathgen.

This is the first step to a generating library (ypathgen) alongside ygen
which generates an API for creating YANG paths given a set of YANG
schema files. While ygen creates a set of structs for storing the data
of a YANG tree, ypathgen creates a set of structs for generating paths
denoting nodes in a YANG tree. This avoids any manual construction of
paths.

Envisioned API example:
oc.Interface("eth0").Counters().InOctets()

Each selector constructs and returns a child struct that keeps track of
the absolute path it represents.

This change includes the generation of these child constructors.

List of primary changes:
- ygot/path_types.go defines the base struct and interface to be
implemented by every path struct node.
- KeyValueAsString() is enhanced to allow ygen.Binary to be converted to
a string for use by gNMI as a list key value.
- A GetDirectories() call is added to ygen for retrieving a parsed map of
Directory elements for use by the ypathgen generation library. The
output of this function is very similar to part of what ygen uses for
its generation.
- The configuration struct for ygen code generation is reorganized.
- pathgen.go contains the child path struct constructor generation code.

* gofmt gogen_test.go

demo/getting_started/interfaces_test.go

@@ -26,9 +26,13 @@ func TestGenerateCode(t *testing.T) {
 	}{{
 		name: "openconfig interfaces",
 		inConfig: &ygen.GeneratorConfig{
-			CompressOCPaths:    true,
-			ExcludeModules:     []string{"ietf-interfaces"},
-			GenerateFakeRoot:   true,
+			ParseOptions: ygen.ParseOpts{
+				ExcludeModules: []string{"ietf-interfaces"},
+			},
+			TransformationOptions: ygen.TransformationOpts{
+				CompressOCPaths:  true,
+				GenerateFakeRoot: true,
+			},
 			GenerateJSONSchema: true,
 		},
 		inFiles: []string{
@@ -39,9 +43,13 @@ func TestGenerateCode(t *testing.T) {
 	}, {
 		name: "openconfig interfaces with no compression",
 		inConfig: &ygen.GeneratorConfig{
-			CompressOCPaths:    false,
-			ExcludeModules:     []string{"ietf-interfaces"},
-			GenerateFakeRoot:   true,
+			ParseOptions: ygen.ParseOpts{
+				ExcludeModules: []string{"ietf-interfaces"},
+			},
+			TransformationOptions: ygen.TransformationOpts{
+				CompressOCPaths:  false,
+				GenerateFakeRoot: true,
+			},
 			GenerateJSONSchema: true,
 		},
 		inFiles: []string{

generator/generator.go

@@ -232,15 +232,20 @@ func main() {
 
 	// Perform the code generation.
 	cg := ygen.NewYANGCodeGenerator(&ygen.GeneratorConfig{
-		CompressOCPaths:    *compressPaths,
-		ExcludeModules:     modsExcluded,
+		ParseOptions: ygen.ParseOpts{
+			ExcludeModules: modsExcluded,
+			YANGParseOptions: yang.Options{
+				IgnoreSubmoduleCircularDependencies: *ignoreCircDeps,
+			},
+			ExcludeState: *excludeState,
+		},
+		TransformationOptions: ygen.TransformationOpts{
+			CompressOCPaths:  *compressPaths,
+			GenerateFakeRoot: *generateFakeRoot,
+			FakeRootName:     *fakeRootName,
+		},
 		PackageName:        *packageName,
-		GenerateFakeRoot:   *generateFakeRoot,
-		FakeRootName:       *fakeRootName,
 		GenerateJSONSchema: *generateSchema,
-		YANGParseOptions: yang.Options{
-			IgnoreSubmoduleCircularDependencies: *ignoreCircDeps,
-		},
 		GoOptions: ygen.GoOpts{
 			YgotImportPath:       *ygotImportPath,
 			YtypesImportPath:     *ytypesImportPath,
@@ -254,7 +259,6 @@ func main() {
 			GenerateLeafGetters:  *generateLeafGetters,
 			IncludeModelData:     *includeModelData,
 		},
-		ExcludeState: *excludeState,
 	})
 
 	generatedGoCode, err := cg.GenerateGoCode(generateModules, includePaths)

proto_generator/protogenerator.go

@@ -92,15 +92,20 @@ func main() {
 
 	// Perform the code generation.
 	cg := ygen.NewYANGCodeGenerator(&ygen.GeneratorConfig{
-		CompressOCPaths:  *compressPaths,
-		ExcludeModules:   modsExcluded,
-		PackageName:      *packageName,
-		GenerateFakeRoot: *generateFakeRoot,
-		FakeRootName:     *fakeRootName,
-		Caller:           *callerName,
-		YANGParseOptions: yang.Options{
-			IgnoreSubmoduleCircularDependencies: *ignoreCircDeps,
+		ParseOptions: ygen.ParseOpts{
+			ExcludeModules: modsExcluded,
+			YANGParseOptions: yang.Options{
+				IgnoreSubmoduleCircularDependencies: *ignoreCircDeps,
+			},
+			ExcludeState: *excludeState,
 		},
+		TransformationOptions: ygen.TransformationOpts{
+			CompressOCPaths:  *compressPaths,
+			GenerateFakeRoot: *generateFakeRoot,
+			FakeRootName:     *fakeRootName,
+		},
+		PackageName: *packageName,
+		Caller:      *callerName,
 		ProtoOptions: ygen.ProtoOpts{
 			BaseImportPath:      *baseImportPath,
 			YwrapperPath:        *ywrapperPath,
@@ -110,7 +115,6 @@ func main() {
 			NestedMessages:      !*packageHierarchy,
 			EnumPackageName:     *enumPackageName,
 		},
-		ExcludeState: *excludeState,
 	})
 
 	generatedProtoCode, err := cg.GenerateProto3(generateModules, includePaths)

ygen/codegen.go

@@ -45,31 +45,12 @@ type YANGCodeGenerator struct {
 
 // GeneratorConfig stores the configuration options used for code generation.
 type GeneratorConfig struct {
-	// CompressOCPaths indicates whether paths should be compressed in the output
-	// of an OpenConfig schema.
-	CompressOCPaths bool
-	// ExcludeModules specifies any modules that are included within the set of
-	// modules that should have code generated for them that should be ignored during
-	// code generation. This is due to the fact that some schemas (e.g., OpenConfig
-	// interfaces) currently result in overlapping entities (e.g., /interfaces).
-	ExcludeModules []string
 	// PackageName is the name that should be used for the generating package.
 	PackageName string
 	// Caller is the name of the binary calling the generator library, it is
 	// included in the header of output files for debugging purposes. If a
 	// string is not specified, the location of the library is utilised.
 	Caller string
-	// YANGParseOptions provides the options that should be handed to the
-	// github.com/openconfig/goyang/pkg/yang library. These specify how the
-	// input YANG files should be parsed.
-	YANGParseOptions yang.Options
-	// GenerateFakeRoot specifies whether an entity that represents the
-	// root of the YANG schema tree should be generated in the generated
-	// code.
-	GenerateFakeRoot bool
-	// FakeRootName specifies the name of the struct that should be generated
-	// representing the root.
-	FakeRootName string
 	// GenerateJSONSchema stores a boolean which defines whether to generate
 	// the JSON corresponding to the YANG schema parsed to generate the
 	// output code.
@@ -78,18 +59,65 @@ type GeneratorConfig struct {
 	// generation function, such that it can be handled by an external
 	// library.
 	StoreRawSchema bool
+	// ParseOptions contains parsing options for a given set of schema files.
+	ParseOptions ParseOpts
+	// TransformationOptions contains options for how the generated code
+	// may be transformed from a simple 1:1 mapping with respect to the
+	// given YANG schema.
+	TransformationOptions TransformationOpts
 	// GoOptions stores a struct which stores Go code generation specific
 	// options for the code generaton.
 	GoOptions GoOpts
 	// ProtoOptions stores a struct which contains Protobuf specific options.
 	ProtoOptions ProtoOpts
+}
+
+// DirectoryGenConfig contains the configuration necessary to generate a set of
+// Directory objects for a given schema. The set of Directory objects is the
+// intermediate representation generated by ygen, which can be useful for
+// external code generation libraries which can make use of it.
+type DirectoryGenConfig struct {
+	// ParseOptions contains parsing options for a given set of schema files.
+	ParseOptions ParseOpts
+	// TransformationOptions contains options for how the generated code
+	// may be transformed from a simple 1:1 mapping with respect to the
+	// given YANG schema.
+	TransformationOptions TransformationOpts
+}
+
+// ParseOpts contains parsing configuration for a given schema.
+type ParseOpts struct {
+	// ExcludeModules specifies any modules that are included within the set of
+	// modules that should have code generated for them that should be ignored during
+	// code generation. This is due to the fact that some schemas (e.g., OpenConfig
+	// interfaces) currently result in overlapping entities (e.g., /interfaces).
+	ExcludeModules []string
+	// YANGParseOptions provides the options that should be handed to the
+	// github.com/openconfig/goyang/pkg/yang library. These specify how the
+	// input YANG files should be parsed.
+	YANGParseOptions yang.Options
 	// ExcludeState specifies whether config false values should be
 	// included in the generated code output. When set, all values that are
 	// not writeable (i.e., config false) within the YANG schema and their
 	// children are excluded from the generated code.
 	ExcludeState bool
 }
 
+// TransformationOpts specifies transformations to the generated code with
+// respect to the input schema.
+type TransformationOpts struct {
+	// CompressOCPaths indicates whether paths should be compressed in the output
+	// of an OpenConfig schema.
+	CompressOCPaths bool
+	// GenerateFakeRoot specifies whether an entity that represents the
+	// root of the YANG schema tree should be generated in the generated
+	// code.
+	GenerateFakeRoot bool
+	// FakeRootName specifies the name of the struct that should be generated
+	// representing the root.
+	FakeRootName string
+}
+
 // GoOpts stores Go specific options for the code generation library.
 type GoOpts struct {
 	// SchemaVarName is the name for the variable which stores the compressed
@@ -297,13 +325,13 @@ func (cg *YANGCodeGenerator) GenerateGoCode(yangFiles, includePaths []string) (*
 	// Store the returned schematree within the state for this code generation.
 	cg.state.schematree = mdef.schemaTree
 
-	directory, errs := cg.state.buildDirectoryDefinitions(mdef.directoryEntries, cg.Config.CompressOCPaths, cg.Config.GenerateFakeRoot, golang, cg.Config.ExcludeState)
+	directory, errs := cg.state.buildDirectoryDefinitions(mdef.directoryEntries, cg.Config.TransformationOptions.CompressOCPaths, cg.Config.TransformationOptions.GenerateFakeRoot, golang, cg.Config.ParseOptions.ExcludeState)
 	if errs != nil {
 		return nil, errs
 	}
 
 	var rootName string
-	if rootName = resolveRootName(cg.Config.FakeRootName, defaultRootName, cg.Config.GenerateFakeRoot); rootName != "" {
+	if rootName = resolveRootName(cg.Config.TransformationOptions.FakeRootName, defaultRootName, cg.Config.TransformationOptions.GenerateFakeRoot); rootName != "" {
 		if r, ok := directory[fmt.Sprintf("/%s", rootName)]; ok {
 			rootName = r.Name
 		}
@@ -329,7 +357,7 @@ func (cg *YANGCodeGenerator) GenerateGoCode(yangFiles, includePaths []string) (*
 	var structSnippets []GoStructCodeSnippet
 	for _, directoryName := range orderedDirNames {
 		structOut, errs := writeGoStruct(dirNameMap[directoryName], directory, cg.state,
-			cg.Config.CompressOCPaths, cg.Config.GenerateJSONSchema, cg.Config.GoOptions)
+			cg.Config.TransformationOptions.CompressOCPaths, cg.Config.GenerateJSONSchema, cg.Config.GoOptions)
 		if errs != nil {
 			codegenErr = util.AppendErrs(codegenErr, errs)
 			continue
@@ -343,7 +371,7 @@ func (cg *YANGCodeGenerator) GenerateGoCode(yangFiles, includePaths []string) (*
 		}
 	}
 
-	goEnums, errs := cg.state.findEnumSet(mdef.enumEntries, cg.Config.CompressOCPaths, false)
+	goEnums, errs := cg.state.findEnumSet(mdef.enumEntries, cg.Config.TransformationOptions.CompressOCPaths, false)
 	if errs != nil {
 		codegenErr = util.AppendErrs(codegenErr, errs)
 		return nil, codegenErr
@@ -359,7 +387,7 @@ func (cg *YANGCodeGenerator) GenerateGoCode(yangFiles, includePaths []string) (*
 	var enumTypeMapCode string
 	if cg.Config.GenerateJSONSchema {
 		var err error
-		rawSchema, err = buildJSONTree(mdef.modules, cg.state.uniqueDirectoryNames, mdef.directoryEntries["/"], cg.Config.CompressOCPaths)
+		rawSchema, err = buildJSONTree(mdef.modules, cg.state.uniqueDirectoryNames, mdef.directoryEntries["/"], cg.Config.TransformationOptions.CompressOCPaths)
 		if err != nil {
 			util.AppendErr(codegenErr, fmt.Errorf("error marshalling JSON schema: %v", err))
 		}
@@ -392,11 +420,45 @@ func (cg *YANGCodeGenerator) GenerateGoCode(yangFiles, includePaths []string) (*
 	}, nil
 }
 
-// generateEnumCode takes a map of enumerated-type entries keyed by their
-// names, and returns a slice of their definition snippets and an enum
-// string-value look-up snippet. The look-up snippet defines a map keyed by the
-// enums' type names, whose value is another map which can be used to look up a
-// given enum value's string representation given its integer representation.
+// GetDirectories parses YANG files and returns a path-keyed map of Directory
+// entries that is the intermediate representation used by ygen for subsequent
+// code generation. This representation may be useful to external code
+// generation libraries that make use of such information, e.g. if their output
+// code depends on a ygen-generated type. yangFiles is a slice of strings
+// containing the path to a set of YANG files which contain YANG modules.
+// includePaths is slice of strings which specifies the set of paths that are
+// to be searched for associated models (e.g., modules that are included by the
+// specified set of modules, or submodules of those modules).
+// Any errors encountered during code generation are returned.
+func (dcg *DirectoryGenConfig) GetDirectories(yangFiles, includePaths []string) (map[string]*Directory, util.Errors) {
+	if !dcg.TransformationOptions.CompressOCPaths {
+		return nil, util.Errors{fmt.Errorf("GetDirectories currently does not have unit tests for CompressOCPaths=false; if support needed, add unit tests and remove this error")}
+	}
+
+	cg := &GeneratorConfig{ParseOptions: dcg.ParseOptions, TransformationOptions: dcg.TransformationOptions}
+	// Extract the entities to be mapped into structs and enumerations in the output
+	// Go code. Extract the schematree from the modules provided such that it can be
+	// used to reference entities within the tree.
+	mdef, errs := mappedDefinitions(yangFiles, includePaths, cg)
+	if errs != nil {
+		return nil, errs
+	}
+
+	dirsToProcess := map[string]*yang.Entry(mdef.directoryEntries)
+
+	// TODO(wenbli): need to use state to return data about the returned directories (e.g. names & type names used in the generated Go code).
+	state := newGenState()
+
+	// Store the returned schematree within the state for this code generation.
+	state.schematree = mdef.schemaTree
+
+	directory, errs := state.buildDirectoryDefinitions(dirsToProcess, cg.TransformationOptions.CompressOCPaths, cg.TransformationOptions.GenerateFakeRoot, golang, cg.ParseOptions.ExcludeState)
+	if errs != nil {
+		return nil, errs
+	}
+	return directory, nil
+}
+
 func generateEnumCode(goEnums map[string]*yangEnum) ([]string, string, util.Errors) {
 	// orderedEnumNames is used to get the enumerated types that have been
 	// identified in alphabetical order, such that they are returned in a
@@ -456,7 +518,7 @@ func (cg *YANGCodeGenerator) GenerateProto3(yangFiles, includePaths []string) (*
 
 	cg.state.schematree = mdef.schemaTree
 
-	penums, errs := cg.state.findEnumSet(mdef.enumEntries, cg.Config.CompressOCPaths, true)
+	penums, errs := cg.state.findEnumSet(mdef.enumEntries, cg.Config.TransformationOptions.CompressOCPaths, true)
 	if errs != nil {
 		return nil, errs
 	}
@@ -465,7 +527,7 @@ func (cg *YANGCodeGenerator) GenerateProto3(yangFiles, includePaths []string) (*
 		return nil, errs
 	}
 
-	protoMsgs, errs := cg.state.buildDirectoryDefinitions(mdef.directoryEntries, cg.Config.CompressOCPaths, cg.Config.GenerateFakeRoot, protobuf, cg.Config.ExcludeState)
+	protoMsgs, errs := cg.state.buildDirectoryDefinitions(mdef.directoryEntries, cg.Config.TransformationOptions.CompressOCPaths, cg.Config.TransformationOptions.GenerateFakeRoot, protobuf, cg.Config.ParseOptions.ExcludeState)
 	if errs != nil {
 		return nil, errs
 	}
@@ -525,7 +587,7 @@ func (cg *YANGCodeGenerator) GenerateProto3(yangFiles, includePaths []string) (*
 		m := msgMap[n]
 
 		genMsg, errs := writeProto3Msg(m, protoMsgs, cg.state, &protoMsgConfig{
-			compressPaths:       cg.Config.CompressOCPaths,
+			compressPaths:       cg.Config.TransformationOptions.CompressOCPaths,
 			basePackageName:     basePackageName,
 			enumPackageName:     enumPackageName,
 			baseImportPath:      cg.Config.ProtoOptions.BaseImportPath,
@@ -578,7 +640,7 @@ func (cg *YANGCodeGenerator) GenerateProto3(yangFiles, includePaths []string) (*
 			Imports:                stringKeys(pkgImports[n]),
 			SourceYANGFiles:        yangFiles,
 			SourceYANGIncludePaths: includePaths,
-			CompressPaths:          cg.Config.CompressOCPaths,
+			CompressPaths:          cg.Config.TransformationOptions.CompressOCPaths,
 			CallerName:             cg.Config.Caller,
 			YwrapperPath:           ywrapperPath,
 			YextPath:               yextPath,
@@ -684,14 +746,14 @@ type mappedYANGDefinitions struct {
 // It returns a mappedYANGDefinitions struct populated with the directory, enum
 // entries in the input schemas as well as the calculated schema tree.
 func mappedDefinitions(yangFiles, includePaths []string, cfg *GeneratorConfig) (*mappedYANGDefinitions, util.Errors) {
-	modules, errs := processModules(yangFiles, includePaths, cfg.YANGParseOptions)
+	modules, errs := processModules(yangFiles, includePaths, cfg.ParseOptions.YANGParseOptions)
 	if errs != nil {
 		return nil, errs
 	}
 
 	// Build a map of excluded modules to simplify lookup.
 	excluded := map[string]bool{}
-	for _, e := range cfg.ExcludeModules {
+	for _, e := range cfg.ParseOptions.ExcludeModules {
 		excluded[e] = true
 	}
 
@@ -701,7 +763,7 @@ func mappedDefinitions(yangFiles, includePaths []string, cfg *GeneratorConfig) (
 	enums := map[string]*yang.Entry{}
 	var rootElems, treeElems []*yang.Entry
 	for _, module := range modules {
-		errs = append(errs, findMappableEntities(module, dirs, enums, cfg.ExcludeModules, cfg.CompressOCPaths, modules)...)
+		errs = append(errs, findMappableEntities(module, dirs, enums, cfg.ParseOptions.ExcludeModules, cfg.TransformationOptions.CompressOCPaths, modules)...)
 		if module == nil {
 			errs = append(errs, errors.New("found a nil module in the returned module set"))
 			continue
@@ -728,8 +790,8 @@ func mappedDefinitions(yangFiles, includePaths []string, cfg *GeneratorConfig) (
 
 	// If we were asked to generate a fake root entity, then go and find the top-level entities that
 	// we were asked for.
-	if cfg.GenerateFakeRoot {
-		if err := createFakeRoot(dirs, rootElems, cfg.FakeRootName, cfg.CompressOCPaths); err != nil {
+	if cfg.TransformationOptions.GenerateFakeRoot {
+		if err := createFakeRoot(dirs, rootElems, cfg.TransformationOptions.FakeRootName, cfg.TransformationOptions.CompressOCPaths); err != nil {
 			return nil, []error{err}
 		}
 	}