Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

Commit ad925dd

Browse files
lavalamplavalamp
committed
Add verification to code gen
1 parent d524bd8 commit ad925dd

File tree

16 files changed

+193
-22
lines changed

16 files changed

+193
-22
lines changed

‎cmd/libs/go2idl/args/args.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,17 @@ type GeneratorArgs struct {
5656

5757
// Where to get copyright header text.
5858
GoHeaderFilePath string
59+
60+
// If true, only verify, don't write anything.
61+
VerifyOnly bool
5962
}
6063

6164
func (g *GeneratorArgs) AddFlags(fs *pflag.FlagSet) {
6265
fs.StringSliceVarP(&g.InputDirs, "input-dirs", "i", g.InputDirs, "Comma-separated list of import paths to get input types from.")
6366
fs.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/ or ./ if $GOPATH is not set.")
6467
fs.StringVarP(&g.OutputPackagePath, "output-package", "p", g.OutputPackagePath, "Base package path.")
6568
fs.StringVarP(&g.GoHeaderFilePath, "go-header-file", "h", g.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
69+
fs.BoolVar(&g.VerifyOnly, "verify-only", g.VerifyOnly, "If true, only verify existing output, do not write anything.")
6670
}
6771

6872
// LoadGoBoilerplate loads the boilerplate file passed to --go-header-file.
@@ -75,6 +79,8 @@ func (g *GeneratorArgs) LoadGoBoilerplate() ([]byte, error) {
7579
return b, nil
7680
}
7781

82+
// NewBuilder makes a new parser.Builder and populates it with the input
83+
// directories.
7884
func (g *GeneratorArgs) NewBuilder() (*parser.Builder, error) {
7985
b := parser.New()
8086
for _, d := range g.InputDirs {
@@ -112,6 +118,7 @@ func (g *GeneratorArgs) Execute(nameSystems namer.NameSystems, defaultSystem str
112118
return fmt.Errorf("Failed making a context: %v", err)
113119
}
114120

121+
c.Verify = g.VerifyOnly
115122
packages := pkgs(c, g)
116123
if err := c.ExecutePackages(g.OutputBase, packages); err != nil {
117124
return fmt.Errorf("Failed executing generator: %v", err)

‎cmd/libs/go2idl/generator/execute.go

Lines changed: 68 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"fmt"
2222
"go/format"
2323
"io"
24+
"io/ioutil"
2425
"log"
2526
"os"
2627
"path/filepath"
@@ -30,17 +31,29 @@ import (
3031
"k8s.io/kubernetes/cmd/libs/go2idl/types"
3132
)
3233

34+
func errs2strings(errors []error) []string {
35+
strs := make([]string, len(errors))
36+
for i := range errors {
37+
strs[i] = errors[i].Error()
38+
}
39+
return strs
40+
}
41+
3342
// ExecutePackages runs the generators for every package in 'packages'. 'outDir'
3443
// is the base directory in which to place all the generated packages; it
3544
// should be a physical path on disk, not an import path. e.g.:
3645
// /path/to/home/path/to/gopath/src/
3746
// Each package has its import path already, this will be appended to 'outDir'.
3847
func (c *Context) ExecutePackages(outDir string, packages Packages) error {
48+
var errors []error
3949
for _, p := range packages {
4050
if err := c.ExecutePackage(outDir, p); err != nil {
41-
return err
51+
errors = append(errors, err)
4252
}
4353
}
54+
if len(errors) > 0 {
55+
return fmt.Errorf("some packages had errors:\n%v\n", strings.Join(errs2strings(errors), "\n"))
56+
}
4457
return nil
4558
}
4659

@@ -61,15 +74,53 @@ func (ft golangFileType) AssembleFile(f *File, pathname string) error {
6174
return et.Error()
6275
}
6376
if formatted, err := format.Source(b.Bytes()); err != nil {
64-
log.Printf("Warning: unable to run gofmt on %q (%v).", pathname, err)
65-
_, err = destFile.Write(b.Bytes())
77+
err = fmt.Errorf("unable to run gofmt on %q (%v).", pathname, err)
78+
// Write the file anyway, so they can see what's going wrong and fix the generator.
79+
if _, err2 := destFile.Write(b.Bytes()); err2 != nil {
80+
return err2
81+
}
6682
return err
6783
} else {
6884
_, err = destFile.Write(formatted)
6985
return err
7086
}
7187
}
7288

89+
func (ft golangFileType) VerifyFile(f *File, pathname string) error {
90+
log.Printf("Verifying file %q", pathname)
91+
friendlyName := filepath.Join(f.PackageName, f.Name)
92+
b := &bytes.Buffer{}
93+
et := NewErrorTracker(b)
94+
ft.assemble(et, f)
95+
if et.Error() != nil {
96+
return et.Error()
97+
}
98+
formatted, err := format.Source(b.Bytes())
99+
if err != nil {
100+
return fmt.Errorf("unable to gofmt the output for %q: %v", friendlyName, err)
101+
}
102+
existing, err := ioutil.ReadFile(pathname)
103+
if err != nil {
104+
return fmt.Errorf("unable to read file %q for comparison: %v", friendlyName, err)
105+
}
106+
if bytes.Compare(formatted, existing) == 0 {
107+
return nil
108+
}
109+
// Be nice and find the first place where they differ
110+
i := 0
111+
for i < len(formatted) && i < len(existing) && formatted[i] == existing[i] {
112+
i++
113+
}
114+
eDiff, fDiff := existing[i:], formatted[i:]
115+
if len(eDiff) > 100 {
116+
eDiff = eDiff[:100]
117+
}
118+
if len(fDiff) > 100 {
119+
fDiff = fDiff[:100]
120+
}
121+
return fmt.Errorf("output for %q differs; first existing/expected diff: \n %q\n %q", friendlyName, string(eDiff), string(fDiff))
122+
}
123+
73124
func (ft golangFileType) assemble(w io.Writer, f *File) {
74125
w.Write(f.Header)
75126
fmt.Fprintf(w, "package %v\n\n", f.PackageName)
@@ -149,7 +200,7 @@ func (c *Context) addNameSystems(namers namer.NameSystems) *Context {
149200
// import path already, this will be appended to 'outDir'.
150201
func (c *Context) ExecutePackage(outDir string, p Package) error {
151202
path := filepath.Join(outDir, p.Path())
152-
log.Printf("Executing package %v into %v", p.Name(), path)
203+
log.Printf("Processing package %q, disk location %q", p.Name(), path)
153204
// Filter out any types the *package* doesn't care about.
154205
packageContext := c.filteredBy(p.Filter)
155206
os.MkdirAll(path, 0755)
@@ -207,15 +258,26 @@ func (c *Context) ExecutePackage(outDir string, p Package) error {
207258
}
208259
}
209260

261+
var errors []error
210262
for _, f := range files {
263+
finalPath := filepath.Join(path, f.Name)
211264
assembler, ok := c.FileTypes[f.FileType]
212265
if !ok {
213266
return fmt.Errorf("the file type %q registered for file %q does not exist in the context", f.FileType, f.Name)
214267
}
215-
if err := assembler.AssembleFile(f, filepath.Join(path, f.Name)); err != nil {
216-
return err
268+
var err error
269+
if c.Verify {
270+
err = assembler.VerifyFile(f, finalPath)
271+
} else {
272+
err = assembler.AssembleFile(f, finalPath)
273+
}
274+
if err != nil {
275+
errors = append(errors, err)
217276
}
218277
}
278+
if len(errors) > 0 {
279+
return fmt.Errorf("errors in package %q:\n%v\n", p.Name(), strings.Join(errs2strings(errors), "\n"))
280+
}
219281
return nil
220282
}
221283

‎cmd/libs/go2idl/generator/generator.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ type File struct {
6262

6363
type FileType interface {
6464
AssembleFile(f *File, path string) error
65+
VerifyFile(f *File, path string) error
6566
}
6667

6768
// Packages is a list of packages to generate.
@@ -158,6 +159,10 @@ type Context struct {
158159
// A set of types this context can process. If this is empty or nil,
159160
// the default "golang" filetype will be provided.
160161
FileTypes map[string]FileType
162+
163+
// If true, Execute* calls will just verify that the existing output is
164+
// correct. (You may set this after calling NewContext.)
165+
Verify bool
161166
}
162167

163168
// NewContext generates a context from the given builder, naming systems, and

‎cmd/libs/go2idl/set-gen/generators/sets.go

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ package generators
1919

2020
import (
2121
"io"
22-
"os"
23-
"strings"
2422

2523
"k8s.io/kubernetes/cmd/libs/go2idl/args"
2624
"k8s.io/kubernetes/cmd/libs/go2idl/generator"
@@ -57,9 +55,7 @@ func Packages(_ *generator.Context, arguments *args.GeneratorArgs) generator.Pac
5755
PackagePath: arguments.OutputPackagePath,
5856
HeaderText: append(boilerplate, []byte(
5957
`
60-
// This file was autogenerated by the command:
61-
// $ `+strings.Join(os.Args, " ")+`
62-
// Do not edit it manually!
58+
// This file was autogenerated by set-gen. Do not edit it manually!
6359
6460
`)...),
6561
PackageDocumentation: []byte(

‎cmd/libs/go2idl/set-gen/main.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ limitations under the License.
2525
package main
2626

2727
import (
28+
"os"
29+
2830
"k8s.io/kubernetes/cmd/libs/go2idl/args"
2931
"k8s.io/kubernetes/cmd/libs/go2idl/set-gen/generators"
3032

@@ -44,6 +46,8 @@ func main() {
4446
generators.DefaultNameSystem(),
4547
generators.Packages,
4648
); err != nil {
47-
glog.Fatalf("Error: %v", err)
49+
glog.Errorf("Error: %v", err)
50+
os.Exit(1)
4851
}
52+
glog.Info("Completed successfully.")
4953
}

‎hack/after-build/run-codegen.sh

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
3+
# Copyright 2015 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
22+
source "${KUBE_ROOT}/hack/lib/init.sh"
23+
24+
kube::golang::setup_env
25+
26+
setgen=$(kube::util::find-binary "set-gen")
27+
28+
# Please do not add any logic to this shell script. Add logic to the go code
29+
# that generates the set-gen program.
30+
#
31+
# This can be called with one flag, --verify-only, so it works for both the
32+
# update- and verify- scripts.
33+
${setgen} "$@"
34+
35+
# You may add additional calls of code generators like set-gen below.

‎hack/update-all.sh

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,12 @@ fi
4242

4343
BASH_TARGETS="codecgen
4444
generated-conversions
45-
generated-deep-copies
46-
generated-docs
47-
generated-swagger-docs
45+
generated-deep-copies
46+
generated-docs
47+
generated-swagger-docs
4848
swagger-spec
49-
api-reference-docs"
49+
api-reference-docs
50+
codegen"
5051

5152

5253
for t in $BASH_TARGETS

‎hack/update-codegen.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
# Copyright 2014 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
22+
23+
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
24+
source "${KUBE_ROOT}/hack/lib/init.sh"
25+
26+
kube::golang::setup_env
27+
28+
"${KUBE_ROOT}/hack/build-go.sh" cmd/libs/go2idl/set-gen
29+
30+
"${KUBE_ROOT}/hack/after-build/run-codegen.sh" "$@"

‎hack/verify-codegen.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
# Copyright 2014 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
22+
23+
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
24+
source "${KUBE_ROOT}/hack/lib/init.sh"
25+
26+
kube::golang::setup_env
27+
28+
"${KUBE_ROOT}/hack/build-go.sh" cmd/libs/go2idl/set-gen
29+
30+
"${KUBE_ROOT}/hack/after-build/run-codegen.sh" --verify-only

‎hack/verify-flags/known-flags.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ update-period
323323
upgrade-target
324324
use-kubernetes-cluster-service
325325
user-whitelist
326+
verify-only
326327
watch-cache
327328
watch-only
328329
whitelist-override-label

‎pkg/util/sets/byte.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ limitations under the License.
1515
*/
1616

1717
// This file was autogenerated by the command:
18-
// $ cmd/libs/go2idl/set-gen/set-gen
18+
// $ ./set-gen
1919
// Do not edit it manually!
2020

2121
package sets

‎pkg/util/sets/doc.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ limitations under the License.
1515
*/
1616

1717
// This file was autogenerated by the command:
18-
// $ cmd/libs/go2idl/set-gen/set-gen
18+
// $ ./set-gen
1919
// Do not edit it manually!
2020

2121
// Package sets has auto-generated set types.

‎pkg/util/sets/empty.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ limitations under the License.
1515
*/
1616

1717
// This file was autogenerated by the command:
18-
// $ cmd/libs/go2idl/set-gen/set-gen
18+
// $ ./set-gen
1919
// Do not edit it manually!
2020

2121
package sets

‎pkg/util/sets/int.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ limitations under the License.
1515
*/
1616

1717
// This file was autogenerated by the command:
18-
// $ cmd/libs/go2idl/set-gen/set-gen
18+
// $ ./set-gen
1919
// Do not edit it manually!
2020

2121
package sets

‎pkg/util/sets/int64.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ limitations under the License.
1515
*/
1616

1717
// This file was autogenerated by the command:
18-
// $ cmd/libs/go2idl/set-gen/set-gen
18+
// $ ./set-gen
1919
// Do not edit it manually!
2020

2121
package sets

0 commit comments

Comments
 (0)