Tool for generating k8s types wrapping Istio types

Gopkg.toml

@@ -37,6 +37,10 @@
   name = "github.com/google/uuid"
   version = "1.1.0"
 
+[[override]]
+  name = "k8s.io/gengo"
+  revision = "f8a0810f38afb8478882b3835a615aebfda39afa"
+
 [prune]
   go-tests = true
   unused-packages = true

cmd/kubetype-gen/Makefile

@@ -0,0 +1,59 @@
+
+define kubetype-gen =
+$(shell go run ./main.go -i $$(go list ${1}) -p "$$(go list .)/testdata/test_output/${2}" 2>&1)
+endef
+
+test: run-negative-tests run-positive-tests
+
+check-env:
+	@if ! git diff --quiet HEAD -- tests; then \
+	    echo "FAIL: tests contain uncommitted changes. Please commit changes in ./testdata."; \
+	    false; \
+	fi
+
+run-positive-tests: run-positive-defaults-test run-positive-types-test
+
+run-positive-defaults-test: check-env
+	@echo "Running $@"
+	$(eval $@_output = $(call kubetype-gen,./testdata/test_input/positive/defaults,defaults))
+	@echo "${$@_output}" | grep -q "Errors occurred while scanning input.  See previous output for details."; test $$? || (echo "kubetype-gen command failed" && echo "${$@_output}" && false)
+	@if ! git diff --quiet HEAD -- tests || [ -n "$$(git ls-files -o -- tests)" ]; then \
+	    echo "FAIL: output files have changed"; \
+	    false; \
+	fi
+
+run-positive-types-test: check-env
+	@echo "Running $@"
+	$(eval $@_output = $(call kubetype-gen,./testdata/test_input/positive/types,types))
+	@echo "${$@_output}" | grep -q "Errors occurred while scanning input.  See previous output for details."; test $$? || (echo "kubetype-gen command failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Invalid value specified for +kubetype-gen:kubeType in .*/test_input/positive/types.EmptyKubeType.  Using default name EmptyKubeType." || (echo "Missing warning for types.EmptyKubeType" && echo "${$@_output}" && false)
+	@if ! git diff --quiet HEAD -- tests || [ -n "$$(git ls-files -o -- tests)" ]; then \
+	    echo "FAIL: output files have changed"; \
+	    false; \
+	fi
+
+
+run-negative-tests: run-negative-defaults-empty-group-test run-negative-defaults-invalid-group-version-test run-negative-types-test
+
+run-negative-defaults-empty-group-test: check-env
+	@echo "Running $@"
+	$(eval $@_output = $(call kubetype-gen,./testdata/test_input/negative/defaults/emptygroup,fail))
+	@echo "${$@_output}" | grep -q "Errors occurred while scanning input.  See previous output for details." || (echo "kubetype-gen command should have failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Invalid Group/Version for package .*/test_input/negative/defaults/emptygroup, Group not specified for Group/Version: groupversion" || (echo "negative/defaults/emptygroup test failed" && echo "${$@_output}" && false)
+
+run-negative-defaults-invalid-group-version-test: check-env
+	@echo "Running $@"
+	$(eval $@_output = $(call kubetype-gen,./testdata/test_input/negative/defaults/invalidgroupversion,fail))
+	@echo "${$@_output}" | grep -q "Errors occurred while scanning input.  See previous output for details." || (echo "kubetype-gen command should have failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Could not calculate Group/Version for package .*/test_input/negative/defaults/invalidgroupversion: invalid group version 'group/version/version' specified: unexpected GroupVersion string: group/version/version" || (echo "negative/defaults/invalidgroupversion test failed" && echo "${$@_output}" && false)
+
+run-negative-types-test: check-env
+	@echo "Running $@"
+	$(eval $@_output = $(call kubetype-gen,./testdata/test_input/negative/types,fail))
+	@echo "${$@_output}" | grep -q "Errors occurred while scanning input.  See previous output for details." || (echo "kubetype-gen command should have failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -qE "duplicate kube types specified for istio.io/tools/cmd/kubetype-gen/testdata/test_output/fail/group/version.AGoodType.  duplicated by: \[istio.io/tools/cmd/kubetype-gen/testdata/test_input/negative/types.(AGoodType|DuplicateKubeType) istio.io/tools/cmd/kubetype-gen/testdata/test_input/negative/types.(AGoodType|DuplicateKubeType)\]" || (echo "types.DuplicateKubeType test failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Invalid Group/Version for type .*/test_input/negative/types.EmptyGroup: groupversion" || (echo "types.EmptyGroup test failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Could not calculate Group/Version for type .*/test_input/negative/types.InvalidGroupVersion: invalid group version 'group/version/version' specified" || (echo "types.InvalidGroupVersion test failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Invalid Group/Version for type .*/test_input/negative/types.NoGroupVersion: <nil>" || (echo "types.NoGroupVersion test failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -qE "Overlapping packages for Group/Versions group2(.name.io)?/version and group2(.name.io)?/version" || (echo "types.OverlappingPackageGroupType test failed" && echo "${$@_output}" && false)
+	@echo "${$@_output}" | grep -q "Could not create metadata for type: .*/test_input/negative/types.OverlappingPackageGroupType." || (echo "types.OverlappingPackageGroupType test failed" && echo "${$@_output}" && false)

cmd/kubetype-gen/README.md

@@ -0,0 +1,139 @@
+# kubetype-gen
+
+`kubetype-gen` is a utility for generating Kubernetes type wrappers (`types.go`)
+and registration code (`register.go`) for existing types that need to be used
+within Kubernetes (e.g. a Custom Resource Definition based on an existing type).
+
+The type definitions take the form:
+
+```go
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type SomeType struct {
+	v1.TypeMeta `json:",inline"`
+	// +optional
+	v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Spec defines the implementation of this definition.
+	// +optional
+	Spec pkg.SomeType `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
+}
+```
+
+The `register.go` file takes the standard form, including all the generated
+types, as well as their corresponding `List` types.
+
+## Usage
+
+The generator is enabled by applying comment tags (`+tagName`) to types and
+packages.
+
+The general command line usage:
+
+`${GOPATH}/bin/kubetype-gen -i <input-packages> -p <output-package>`
+
+For example:
+
+`${GOPATH}/bin/kubetype-gen -i istio.io/api/authentication/... -p istio.io/api/kube/apis`
+
+The tool can also be used with `go run`, e.g. `go run vendor/istio.io/tools/cmd/kubetype-gen/main.go ...`
+
+### Package Tags
+
+The generator is enabled for a source package by adding tags to the package
+comments in its `doc.go` file.  The following tags are required:
+
+**+kubetype-gen:groupVersion=group/version**
+> Tells the code generator that the types should be registered with the
+> *group/version*.  The generated types will be emitted into the package,
+> *.../group/version*.
+
+Note, the same *groupVersion* values may be used in multiple packages, resulting
+in the types from all those packages being generated into the same output
+package and registered with the same *group/version*.
+
+Example `doc.go`:
+```go
+// Package v1alpha1 tags supporting code generate
+//+kubetype-gen:groupVersion=authentication.istio.io/v1alpha1
+package v1alpha1
+
+```
+
+### Type Tags
+
+The generator is enabled for a specific type by adding tags to the type's
+comments.  The following tags are available:
+
+**+kubetype-gen**
+> Generate a Kubernetes type for this type.
+
+**+kubetype-gen:groupVersion=\<group/version>**
+> Tells the code generator that the type should be registered with the
+> *group/version*.  The generated type will be emitted into the package,
+> *.../group/version*.
+
+**+kubetype-gen:kubeType=\<name>**
+> The generated Kubenetes type should use the specified *name*.  If this tag is
+> not specified, the generated type will use the same name as the type.  This
+> tag may be specified multiple times, which will result in a generated type for
+> each specified *name*.
+
+**+kubetype-gen:\<name>:tag=\<tag>**
+> The generated Kubenetes type named *name* should have the specified *tag*
+> added to its generated comments.  The tag should not include the leading `+`
+> as this will be added by the generator.  This may be specified multiple times,
+> once for each *tag* that should be added to the generated type.
+
+Example type comments:
+```go
+// +kubetype-gen
+// +kubetype-gen:groupVersion=authentication.istio.io/v1alpha1
+// +kubetype-gen:kubeType=Policy
+// +kubetype-gen:kubeType=MeshPolicy
+// +kubetype-gen:MeshPolicy:tag=genclient:nonNamespaced
+type Policy struct {
+```
+
+### Other Considerations
+
+#### Tags For Other Kubernetes Code Generators
+
+In addition to adding `kubetype-gen` tags to types, you should consider adding
+tags used by other Kubernetes code generators.  The comments from the source
+types will be copied over directly to the generated Kubernetes types, allowing
+you to run other code generators on the generated types (e.g. to generate
+clientsets, informers, etc.).  This allows you to control what gets generated
+from the other generators.  Note, the generator adds the
+`+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object` tag to the
+generated types.
+
+#### Generating `DeepCopy()` Functions
+
+If you are using your types in Kubernetes, you will most likely need to be able
+to copy them.  This will require you to use `deepcopy-gen` on your source types.
+This can be problematic depending on how your types are structured, meaning you
+may have to manually implement functions for some types.  You will also want to
+run the generator manually so you have direct control over the input packages,
+e.g.
+
+```
+${GOPATH}/bin/deepcopy-gen -i istio.io/api/authentication/... -O zz_generated.deepcopy -h vendor/istio.io/tools/cmd/kubetype-gen/boilerplate.go.txt
+```
+
+#### Protobuf Serializers ####
+
+If using protobuf for the source types, you will need to ensure that the
+packages for the source types include a `generated.proto` file.  The Kubernetes
+`go-to-protobuf` tool looks for these files in the source packages.  If you are
+not using this mechanism, a simple work around is to use
+`import public "package/to/some_existing.proto";` statements to include the
+upstream `.proto` files for the generated types.  Also be aware that the
+ `go-to-protobuf` generator assumes imports use full import path, e.g.
+ `istio.io/api/authentication/v1alpha1/policy.proto` vs. something like
+ `authentication/v1alpha1/policy.proto`.
+
+When running `go-to-protobuf`, add the packages for the existing types to the
+`--apimachinery-packages` using the `-` prefix.  This tells the code generator
+to look for `generated.proto` files in those packages, but do not generate
+code for the types within them.  This will limit `go-to-protobuf` to generating
+code only for the types generated by `kubetype-gen`.

cmd/kubetype-gen/boilerplate.go.txt

@@ -0,0 +1,13 @@
+// Copyright 2019 Istio Authors
+//
+//   Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//   Unless required by applicable law or agreed to in writing, software
+//   distributed under the License is distributed on an "AS IS" BASIS,
+//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//   See the License for the specific language governing permissions and
+//   limitations under the License.

cmd/kubetype-gen/generators/naming.go

@@ -0,0 +1,19 @@
+package generators
+
+import (
+	"k8s.io/gengo/namer"
+)
+
+// NameSystems used by the kubetype generator
+func NameSystems(generatedPackage string, tracker namer.ImportTracker) namer.NameSystems {
+	return namer.NameSystems{
+		"public":       namer.NewPublicNamer(0),
+		"raw":          namer.NewRawNamer(generatedPackage, tracker),
+		"publicPlural": namer.NewPublicPluralNamer(map[string]string{}),
+	}
+}
+
+// DefaultNameSystem to use if none is specified
+func DefaultNameSystem() string {
+	return "public"
+}

cmd/kubetype-gen/generators/package.go

@@ -0,0 +1,38 @@
+package generators
+
+import (
+	"istio.io/tools/cmd/kubetype-gen/metadata"
+	"k8s.io/gengo/generator"
+	"k8s.io/gengo/types"
+)
+
+// NewPackageGenerator generates source for a scanned package, specifically k8s styled doc.go, types.go and register.go files
+func NewPackageGenerator(source metadata.PackageMetadata, boilerplate []byte) generator.Package {
+	return &generator.DefaultPackage{
+		PackageName: source.TargetPackage().Name,
+		PackagePath: source.TargetPackage().Path,
+		HeaderText:  boilerplate,
+		PackageDocumentation: []byte(`
+// Package has auto-generated kube type wrappers for raw types.
+// +k8s:openapi-gen=true
+// +k8s:deepcopy-gen=package
+`),
+		FilterFunc: func(c *generator.Context, t *types.Type) bool {
+			for _, it := range source.RawTypes() {
+				if t == it {
+					return true
+				}
+			}
+			return false
+		},
+		GeneratorList: []generator.Generator{
+			// generate types.go
+			NewTypesGenerator(source),
+			// generate register.go
+			NewRegisterGenerator(source),
+			generator.DefaultGen{
+				OptionalName: "doc",
+			},
+		},
+	}
+}