Split Go generation into gogen package.

.github/workflows/go.yml

@@ -87,7 +87,7 @@ jobs:
             # fake ygot and ytype paths specified in generation options.
             "openconfig-options-explicit.formatted-txt.go"
           )
-          for f in ygen/testdata/schema/*.formatted-txt; do
+          for f in gogen/testdata/schema/*.formatted-txt; do
             if [[ ${skipped[@]} =~ $(basename $f) ]]; then
               continue
             fi
@@ -96,7 +96,7 @@ jobs:
 
           skipped=(
           )
-          for f in ygen/testdata/structs/*.formatted-txt; do
+          for f in gogen/testdata/structs/*.formatted-txt; do
             if [[ ${skipped[@]} =~ $(basename $f) ]]; then
               continue
             fi
@@ -114,8 +114,8 @@ jobs:
             fi
             filename=$(basename $f)
             f_prefix="${filename%%.*}"
-            ygen_file="ygen/testdata/structs/${f_prefix}.formatted-txt"
-            test-go-build $f $ygen_file
+            go_file="gogen/testdata/structs/${f_prefix}.formatted-txt"
+            test-go-build $f $go_file
           done
 
   static_analysis:
@@ -131,7 +131,6 @@ jobs:
 
       - name: Install required static analysis tools
         run: |
-          go install github.com/go-playground/overalls@latest
           go install github.com/mattn/goveralls@latest
           go install honnef.co/go/tools/cmd/staticcheck@latest
 
@@ -144,12 +143,27 @@ jobs:
 
       - name: Run coverage
         run: |
-          overalls -project ${GITHUB_WORKSPACE} -covermode=count -ignore=".git,vendor,integration_tests,ygot/schema_tests,ygen/schema_tests,ypathgen/path_tests,demo,experimental/ygotutils,generator,ytypes/schema_tests,ypathgen/generator"
+          pkgs="$(go list ./... | grep -v 'github.com/openconfig/ygot/exampleoc' | grep -v 'github.com/openconfig/ygot/uexampleoc' | grep -v 'github.com/openconfig/ygot/proto/' | grep -v 'github.com/openconfig/ygot/demo/' | grep -v 'github.com/openconfig/ygot/integration_tests' | grep -v 'github.com/openconfig/ygot/generator' | tr '\n' ' ')"
+          for p in $pkgs; do
+            dir=$(echo $p | sed -e 's/^github\.com\/openconfig\/ygot\///')
+            go test -covermode count -coverprofile profile.coverprofile -outputdir $dir $p
+          done
+
+          # Overwrite results for gogen/protogen since they also cover ygen's
+          # code.
+          go test -covermode count -coverprofile profile.coverprofile -outputdir gogen -coverpkg github.com/openconfig/ygot/gogen,github.com/openconfig/ygot/ygen github.com/openconfig/ygot/gogen
+          #go test -covermode count -coverprofile profile.coverprofile -outputdir protogen -coverpkg github.com/openconfig/ygot/protogen,github.com/openconfig/ygot/ygen github.com/openconfig/ygot/protogen
+
+          echo 'mode: count' > concatenated.coverprofile
+          for p in $pkgs; do
+            dir=$(echo $p | sed -e 's/^github\.com\/openconfig\/ygot\///')
+            tail -n +2 $dir/profile.coverprofile >> concatenated.coverprofile
+          done
 
       - name: Submit coverage
         uses: shogo82148/actions-goveralls@v1
         with:
-          path-to-profile: overalls.coverprofile
+          path-to-profile: concatenated.coverprofile
 
       - name: Go vet
         run: |

demo/getting_started/interfaces_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/openconfig/ygot/genutil"
+	"github.com/openconfig/ygot/gogen"
 	"github.com/openconfig/ygot/ygen"
 )
 
@@ -22,14 +23,12 @@ func TestGenerateCode(t *testing.T) {
 	tests := []struct {
 		name     string
 		inConfig *ygen.GeneratorConfig
+		inGoOpts *gogen.GoOpts
 		inFiles  []string
 		inPaths  []string
 	}{{
 		name: "openconfig interfaces",
 		inConfig: &ygen.GeneratorConfig{
-			GoOptions: ygen.GoOpts{
-				GenerateSimpleUnions: true,
-			},
 			ParseOptions: ygen.ParseOpts{
 				ExcludeModules: []string{"ietf-interfaces"},
 			},
@@ -39,6 +38,9 @@ func TestGenerateCode(t *testing.T) {
 			},
 			GenerateJSONSchema: true,
 		},
+		inGoOpts: &gogen.GoOpts{
+			GenerateSimpleUnions: true,
+		},
 		inFiles: []string{
 			filepath.Join(TestRoot, "yang", "openconfig-interfaces.yang"),
 			filepath.Join(TestRoot, "yang", "openconfig-if-ip.yang"),
@@ -47,9 +49,6 @@ func TestGenerateCode(t *testing.T) {
 	}, {
 		name: "openconfig interfaces with no compression",
 		inConfig: &ygen.GeneratorConfig{
-			GoOptions: ygen.GoOpts{
-				GenerateSimpleUnions: true,
-			},
 			ParseOptions: ygen.ParseOpts{
 				ExcludeModules: []string{"ietf-interfaces"},
 			},
@@ -58,6 +57,9 @@ func TestGenerateCode(t *testing.T) {
 			},
 			GenerateJSONSchema: true,
 		},
+		inGoOpts: &gogen.GoOpts{
+			GenerateSimpleUnions: true,
+		},
 		inFiles: []string{
 			filepath.Join(TestRoot, "yang", "openconfig-interfaces.yang"),
 			filepath.Join(TestRoot, "yang", "openconfig-if-ip.yang"),
@@ -66,10 +68,10 @@ func TestGenerateCode(t *testing.T) {
 	}}
 
 	for _, tt := range tests {
-		cg := ygen.NewYANGCodeGenerator(tt.inConfig)
-		got, err := cg.GenerateGoCode(tt.inFiles, tt.inPaths)
+		cg := gogen.NewCodeGenerator(tt.inConfig, tt.inGoOpts)
+		got, err := cg.Generate(tt.inFiles, tt.inPaths)
 		if err != nil {
-			t.Errorf("%s: GenerateGoCode(%v, %v): Config: %v, got unexpected error: %v", tt.name, tt.inFiles, tt.inPaths, tt.inConfig, err)
+			t.Errorf("%s: Generate(%v, %v): Config: %v, got unexpected error: %v", tt.name, tt.inFiles, tt.inPaths, tt.inConfig, err)
 			continue
 		}
 

generator/generator.go

@@ -29,6 +29,7 @@ import (
 	log "github.com/golang/glog"
 	"github.com/openconfig/goyang/pkg/yang"
 	"github.com/openconfig/ygot/genutil"
+	"github.com/openconfig/ygot/gogen"
 	"github.com/openconfig/ygot/ygen"
 	"github.com/openconfig/ygot/ypathgen"
 )
@@ -85,7 +86,7 @@ var (
 	goyangImportPath        = flag.String("goyang_path", genutil.GoDefaultGoyangImportPath, "The import path to use for goyang's yang package.")
 	generateRename          = flag.Bool("generate_rename", false, "If set to true, rename methods are generated for lists within the Go code.")
 	addAnnotations          = flag.Bool("annotations", false, "If set to true, metadata annotations are added within the generated structs.")
-	annotationPrefix        = flag.String("annotation_prefix", ygen.DefaultAnnotationPrefix, "String to be appended to each metadata field within the generated structs if annoations is set to true.")
+	annotationPrefix        = flag.String("annotation_prefix", gogen.DefaultAnnotationPrefix, "String to be appended to each metadata field within the generated structs if annoations is set to true.")
 	addYangPresence         = flag.Bool("yangpresence", false, "If set to true, a tag will be added to the field of a generated Go struct to indicate when a YANG presence container is being used.")
 	generateAppend          = flag.Bool("generate_append", false, "If set to true, append methods are generated for YANG lists (Go maps) within the Go code.")
 	generateGetters         = flag.Bool("generate_getters", false, "If set to true, getter methdos that retrieve or create an element are generated for YANG container (Go struct pointer) or list (Go map) fields within the generated code.")
@@ -108,10 +109,10 @@ var (
 	packageSuffix           = flag.String("path_struct_package_suffix", "path", "Suffix to append to generated Go package names, when split_pathstructs_by_module=true.")
 )
 
-// writeGoCodeSingleFile takes a ygen.GeneratedGoCode struct and writes the Go code
+// writeGoCodeSingleFile takes a gogen.GeneratedCode struct and writes the Go code
 // snippets contained within it to the io.Writer, w, provided as an argument.
 // The output includes a package header which is generated.
-func writeGoCodeSingleFile(w io.Writer, goCode *ygen.GeneratedGoCode) error {
+func writeGoCodeSingleFile(w io.Writer, goCode *gogen.GeneratedCode) error {
 	// Write the package header to the supplier writer.
 	fmt.Fprint(w, goCode.CommonHeader)
 	fmt.Fprint(w, goCode.OneOffHeader)
@@ -151,11 +152,11 @@ func writeGoPathCodeSingleFile(w io.Writer, pathCode *ypathgen.GeneratedPathCode
 
 // splitCodeByFileN generates a map, keyed by filename, to a string containing
 // the code to be output to that filename. It allows division of a
-// ygen.GeneratedGoCode struct into a set of source files. It divides the
+// gogen.GeneratedCode struct into a set of source files. It divides the
 // methods, interfaces, and enumeration code snippets into their own files.
 // Structs are output into files by splitting them evenly among the input split
 // number.
-func splitCodeByFileN(goCode *ygen.GeneratedGoCode, fileN int) (map[string]string, error) {
+func splitCodeByFileN(goCode *gogen.GeneratedCode, fileN int) (map[string]string, error) {
 	structN := len(goCode.Structs)
 	if fileN < 1 || fileN > structN {
 		return nil, fmt.Errorf("requested %d files, but must be between 1 and %d (number of schema structs)", fileN, structN)
@@ -324,7 +325,7 @@ func main() {
 		}
 
 		// Perform the code generation.
-		cg := ygen.NewYANGCodeGenerator(&ygen.GeneratorConfig{
+		cg := gogen.NewCodeGenerator(&ygen.GeneratorConfig{
 			ParseOptions: ygen.ParseOpts{
 				ExcludeModules:        modsExcluded,
 				SkipEnumDeduplication: *skipEnumDedup,
@@ -345,7 +346,8 @@ func main() {
 			PackageName:         *packageName,
 			GenerateJSONSchema:  *generateSchema,
 			IncludeDescriptions: *includeDescriptions,
-			GoOptions: ygen.GoOpts{
+		},
+			&gogen.GoOpts{
 				YgotImportPath:                      *ygotImportPath,
 				YtypesImportPath:                    *ytypesImportPath,
 				GoyangImportPath:                    *goyangImportPath,
@@ -363,9 +365,9 @@ func main() {
 				IncludeModelData:                    *includeModelData,
 				AppendEnumSuffixForSimpleUnionEnums: *appendEnumSuffixForSimpleUnionEnums,
 			},
-		})
+		)
 
-		generatedGoCode, errs := cg.GenerateGoCode(generateModules, includePaths)
+		generatedGoCode, errs := cg.Generate(generateModules, includePaths)
 		if errs != nil {
 			log.Exitf("ERROR Generating GoStruct Code: %v\n", errs)
 		}

generator/generator_test.go

@@ -22,19 +22,19 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/kylelemons/godebug/pretty"
 	"github.com/openconfig/gnmi/errdiff"
-	"github.com/openconfig/ygot/ygen"
+	"github.com/openconfig/ygot/gogen"
 	"github.com/openconfig/ygot/ypathgen"
 )
 
 func TestWriteGoCode(t *testing.T) {
 	tests := []struct {
 		name     string
-		inGoCode *ygen.GeneratedGoCode
+		inGoCode *gogen.GeneratedCode
 		wantCode string
 	}{{
 		name: "single element structs and enums",
-		inGoCode: &ygen.GeneratedGoCode{
-			Structs: []ygen.GoStructCodeSnippet{{
+		inGoCode: &gogen.GeneratedCode{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructDef: `structOne`,
 			}},
 			Enums:   []string{`enumOne`},
@@ -46,8 +46,8 @@ enumOne
 `,
 	}, {
 		name: "multi-element structs and enums",
-		inGoCode: &ygen.GeneratedGoCode{
-			Structs: []ygen.GoStructCodeSnippet{{
+		inGoCode: &gogen.GeneratedCode{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructDef: "structOne",
 			}, {
 				StructDef: "structTwo",
@@ -63,15 +63,15 @@ enumTwo
 `,
 	}, {
 		name: "json string code",
-		inGoCode: &ygen.GeneratedGoCode{
+		inGoCode: &gogen.GeneratedCode{
 			JSONSchemaCode: "foo",
 		},
 		wantCode: `
 foo
 `,
 	}, {
 		name: "enum type map",
-		inGoCode: &ygen.GeneratedGoCode{
+		inGoCode: &gogen.GeneratedCode{
 			EnumTypeMap: "map",
 		},
 		wantCode: `
@@ -95,16 +95,16 @@ map
 func TestSplitCodeByFileN(t *testing.T) {
 	tests := []struct {
 		name             string
-		in               *ygen.GeneratedGoCode
+		in               *gogen.GeneratedCode
 		inFileN          int
 		want             map[string]string
 		wantErrSubstring string
 	}{{
 		name: "simple struct with all only structs populated",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "name",
 				StructDef:  "def\n",
 				ListKeys:   "name_key",
@@ -122,10 +122,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		},
 	}, {
 		name: "less than 1 file requested for splitting",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "name",
 				StructDef:  "def\n",
 				ListKeys:   "name_key",
@@ -137,10 +137,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		wantErrSubstring: "requested 0 files",
 	}, {
 		name: "more than # of structs files requested for splitting",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "name",
 				StructDef:  "def\n",
 				ListKeys:   "name_key",
@@ -152,10 +152,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		wantErrSubstring: "requested 2 files",
 	}, {
 		name: "two structs with enums populated",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "s1",
 				StructDef:  "s1def\n",
 				ListKeys:   "s1key",
@@ -181,10 +181,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		},
 	}, {
 		name: "two structs, separated into two files",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "s1",
 				StructDef:  "s1def\n",
 				ListKeys:   "s1key",
@@ -210,10 +210,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		},
 	}, {
 		name: "five structs, separated into four files",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "s1",
 				StructDef:  "s1def\n",
 				ListKeys:   "s1key",
@@ -253,10 +253,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		},
 	}, {
 		name: "five structs, separated into three files",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "s1",
 				StructDef:  "s1def\n",
 				ListKeys:   "s1key",
@@ -295,10 +295,10 @@ func TestSplitCodeByFileN(t *testing.T) {
 		},
 	}, {
 		name: "five structs, separated into two files",
-		in: &ygen.GeneratedGoCode{
+		in: &gogen.GeneratedCode{
 			CommonHeader: "common_header\n",
 			OneOffHeader: "oneoff_header\n",
-			Structs: []ygen.GoStructCodeSnippet{{
+			Structs: []gogen.GoStructCodeSnippet{{
 				StructName: "s1",
 				StructDef:  "s1def\n",
 				ListKeys:   "s1key",