WIP: Apply for client-go's typed client

api/api-rules/aggregator_violation_exceptions.list

@@ -6,6 +6,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIRe
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,ServerAddressByClientCIDRs
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,Versions
+API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,ApplyOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,CreateOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,DeleteOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,FieldsV1,Raw

api/api-rules/apiextensions_violation_exceptions.list

@@ -6,6 +6,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIRe
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,ServerAddressByClientCIDRs
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,Versions
+API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,ApplyOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,CreateOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,DeleteOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,FieldsV1,Raw

api/api-rules/codegen_violation_exceptions.list

@@ -6,6 +6,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIRe
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,ServerAddressByClientCIDRs
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,Versions
+API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,ApplyOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,CreateOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,DeleteOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,FieldsV1,Raw

api/api-rules/sample_apiserver_violation_exceptions.list

@@ -6,6 +6,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIRe
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,ServerAddressByClientCIDRs
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,Versions
+API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,ApplyOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,CreateOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,DeleteOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,FieldsV1,Raw

api/api-rules/violation_exceptions.list

@@ -314,6 +314,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIRe
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,ServerAddressByClientCIDRs
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,APIVersions,Versions
+API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,ApplyOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,CreateOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,DeleteOptions,DryRun
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/apis/meta/v1,FieldsV1,Raw

hack/update-codegen.sh

@@ -29,10 +29,12 @@ kube::golang::setup_env
 go install k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/client-gen
 go install k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/lister-gen
 go install k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/informer-gen
+go install k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/applyconfiguration-gen
 
 clientgen=$(kube::util::find-binary "client-gen")
 listergen=$(kube::util::find-binary "lister-gen")
 informergen=$(kube::util::find-binary "informer-gen")
+applyconfigurationgen=$(kube::util::find-binary "applyconfiguration-gen")
 
 IFS=" " read -r -a GROUP_VERSIONS <<< "${KUBE_AVAILABLE_GROUP_VERSIONS}"
 GV_DIRS=()
@@ -54,9 +56,25 @@ done
 # delimit by commas for the command
 GV_DIRS_CSV=$(IFS=',';echo "${GV_DIRS[*]// /,}";IFS=$)
 
+applyconfigurationgen_external_apis=()
+# because client-gen doesn't do policy/v1alpha1, we have to skip it too
+kube::util::read-array applyconfigurationgen_external_apis < <(
+  cd "${KUBE_ROOT}/staging/src"
+  find k8s.io/api -name types.go -print0 | xargs -0 -n1 dirname | sort | grep -v pkg.apis.policy.v1alpha1
+)
+applyconfigurationgen_external_apis+=("k8s.io/apimachinery/pkg/apis/meta/v1")
+applyconfigurationgen_external_apis_csv=$(IFS=,; echo "${applyconfigurationgen_external_apis[*]}")
+applyconfigurations_package="k8s.io/client-go/applyconfigurations"
+${applyconfigurationgen} \
+  --output-base "${KUBE_ROOT}/vendor" \
+  --output-package "${applyconfigurations_package}" \
+  --input-dirs "${applyconfigurationgen_external_apis_csv}" \
+  --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \
+  "$@"
+
 # This can be called with one flag, --verify-only, so it works for both the
 # update- and verify- scripts.
-${clientgen} --output-base "${KUBE_ROOT}/vendor" --output-package="k8s.io/client-go" --clientset-name="kubernetes" --input-base="k8s.io/api" --input="${GV_DIRS_CSV}" --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" "$@"
+${clientgen} --output-base "${KUBE_ROOT}/vendor" --output-package="k8s.io/client-go" --clientset-name="kubernetes" --input-base="k8s.io/api" --input="${GV_DIRS_CSV}" --apply-configuration-package "${applyconfigurations_package}" --go-header-file "${KUBE_ROOT}/hack/boilerplate/boilerplate.generatego.txt" "$@"
 
 listergen_external_apis=()
 kube::util::read-array listergen_external_apis < <(

pkg/api/testing/BUILD

@@ -85,6 +85,7 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = [
+        "applyconfiguration_test.go",
         "backward_compatibility_test.go",
         "conversion_test.go",
         "copy_test.go",
@@ -129,7 +130,10 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//staging/src/k8s.io/client-go/applyconfigurations:go_default_library",
+        "//staging/src/k8s.io/client-go/applyconfigurations/core/v1:go_default_library",
         "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/github.com/google/go-cmp/cmp:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",
         "//vendor/github.com/json-iterator/go:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",

pkg/api/testing/applyconfiguration_test.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"math/rand"
+	"reflect"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	fuzz "github.com/google/gofuzz"
+	"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/applyconfigurations"
+	v1mf "k8s.io/client-go/applyconfigurations/core/v1"
+
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	api "k8s.io/kubernetes/pkg/apis/core"
+)
+
+func doRoundTripApplyConfigurations(t *testing.T, internalVersion schema.GroupVersion, externalVersion schema.GroupVersion, kind string, applyConfig interface{}) {
+	// We do fuzzing on the internal version of the object, and only then
+	// convert to the external version. This is because custom fuzzing
+	// function are only supported for internal objects.
+	internalObj, err := legacyscheme.Scheme.New(internalVersion.WithKind(kind))
+	if err != nil {
+		t.Fatalf("Couldn't create internal object %v: %v", kind, err)
+	}
+	seed := rand.Int63()
+	fuzzer.FuzzerFor(FuzzerFuncs, rand.NewSource(seed), legacyscheme.Codecs).
+		// We are explicitly overwriting custom fuzzing functions, to ensure
+		// that InitContainers and their statuses are not generated. This is
+		// because in this test we are simply doing json operations, in which
+		// those disappear.
+		Funcs(
+			func(s *api.PodSpec, c fuzz.Continue) {
+				c.FuzzNoCustom(s)
+				s.InitContainers = nil
+			},
+			func(s *api.PodStatus, c fuzz.Continue) {
+				c.FuzzNoCustom(s)
+				s.InitContainerStatuses = nil
+			},
+		).Fuzz(internalObj)
+
+	item, err := legacyscheme.Scheme.New(externalVersion.WithKind(kind))
+	if err != nil {
+		t.Fatalf("Couldn't create external object %v: %v", kind, err)
+	}
+	if err := legacyscheme.Scheme.Convert(internalObj, item, nil); err != nil {
+		t.Fatalf("Conversion for %v failed: %v", kind, err)
+	}
+	u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(item)
+	if err != nil {
+		t.Errorf("ToUnstructured failed: %v", err)
+		return
+	}
+
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(u, applyConfig)
+	if err != nil {
+		t.Errorf("FromUnstructured failed: %v", err)
+		return
+	}
+	rtObj := reflect.New(reflect.TypeOf(item).Elem()).Interface().(runtime.Object)
+	rtUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(applyConfig)
+	if err != nil {
+		t.Errorf("DefaultUnstructuredConverter.ToUnstructured failed: %v", err)
+		return
+	}
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(rtUnstructured, rtObj)
+	if err != nil {
+		t.Errorf("FromUnstructured failed: %v", err)
+		return
+	}
+	if !apiequality.Semantic.DeepEqual(item, rtObj) {
+		t.Errorf("Object changed, diff: %v", cmp.Diff(item, rtObj))
+	}
+}
+
+// TestRoundTripApplyConfigurations converts a each known object type to to the apply builder for that object
+// type, then converts it back to the object type and verifies it is unchanged.
+func TestRoundTripApplyConfigurations(t *testing.T) {
+	for gvk := range legacyscheme.Scheme.AllKnownTypes() {
+		if nonRoundTrippableTypes.Has(gvk.Kind) {
+			continue
+		}
+		if gvk.Version == runtime.APIVersionInternal {
+			continue
+		}
+		builder := applyconfigurations.ForKind(gvk)
+		if builder == nil {
+			t.Logf("Skipping: %s", gvk)
+			continue // TODO: how do we know the right ones were skipped?
+		}
+
+		t.Run(gvk.String(), func(t *testing.T) {
+			for i := 0; i < 50; i++ {
+				doRoundTripApplyConfigurations(t, schema.GroupVersion{Group: gvk.Group, Version: runtime.APIVersionInternal}, gvk.GroupVersion(), gvk.Kind, builder)
+				if t.Failed() {
+					break
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkApplyConfigurationsFromUnstructured(b *testing.B) {
+	items := benchmarkItems(b)
+	convertor := runtime.DefaultUnstructuredConverter
+	unstr := make([]map[string]interface{}, len(items))
+	for i := range items {
+		item, err := convertor.ToUnstructured(&items[i])
+		if err != nil || item == nil {
+			b.Fatalf("unexpected error: %v", err)
+		}
+		unstr = append(unstr, item)
+	}
+	size := len(items)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		builder := &v1mf.PodApplyConfiguration{}
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr[i%size], builder); err != nil {
+			b.Fatalf("unexpected error: %v", err)
+		}
+	}
+	b.StopTimer()
+}
+
+func BenchmarkApplyConfigurationsToUnstructured(b *testing.B) {
+	items := benchmarkItems(b)
+	convertor := runtime.DefaultUnstructuredConverter
+	builders := make([]*v1mf.PodApplyConfiguration, len(items))
+	for i := range items {
+		item, err := convertor.ToUnstructured(&items[i])
+		if err != nil || item == nil {
+			b.Fatalf("unexpected error: %v", err)
+		}
+		builder := &v1mf.PodApplyConfiguration{}
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(item, builder); err != nil {
+			b.Fatalf("unexpected error: %v", err)
+		}
+		builders[i] = builder
+	}
+	b.ResetTimer()
+	size := len(items)
+	for i := 0; i < b.N; i++ {
+		builder := builders[i%size]
+		_, err := runtime.DefaultUnstructuredConverter.ToUnstructured(builder)
+		if err != nil {
+			b.Fatalf("unexpected error: %v", err)
+		}
+	}
+	b.StopTimer()
+}

pkg/controller/replication/BUILD

@@ -30,6 +30,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//staging/src/k8s.io/client-go/applyconfigurations/apps/v1:go_default_library",
         "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",

pkg/controller/replication/conversion.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/watch"
+	appsv1apply "k8s.io/client-go/applyconfigurations/apps/v1"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1"
@@ -247,6 +248,11 @@ func (c conversionClient) Patch(ctx context.Context, name string, pt types.Patch
 	return nil, errors.New("Patch() is not implemented for conversionClient")
 }
 
+func (c conversionClient) Apply(context.Context, *appsv1apply.ReplicaSetApplyConfiguration, string, metav1.ApplyOptions, ...string) (result *apps.ReplicaSet, err error) {
+	// This is not used by RSC.
+	return nil, errors.New("Apply() is not implemented for conversionClient")
+}
+
 func (c conversionClient) GetScale(ctx context.Context, name string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) {
 	// This is not used by RSC.
 	return nil, errors.New("GetScale() is not implemented for conversionClient")

pkg/controller/testutil/BUILD

@@ -23,6 +23,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//staging/src/k8s.io/client-go/applyconfigurations/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",

pkg/controller/testutil/test_utils.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/watch"
+	v1apply "k8s.io/client-go/applyconfigurations/core/v1"
 	"k8s.io/client-go/kubernetes/fake"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -349,6 +350,24 @@ func (m *FakeNodeHandler) Patch(_ context.Context, name string, pt types.PatchTy
 	return &updatedNode, nil
 }
 
+// Apply applies a Node in the fake store.
+func (m *FakeNodeHandler) Apply(ctx context.Context, node *v1apply.NodeApplyConfiguration, fieldManager string, opts metav1.ApplyOptions, subresources ...string) (result *v1.Node, err error) {
+	patchOpts := opts.ToPatchOptions(fieldManager)
+	data, err := json.Marshal(node)
+	if err != nil {
+		return nil, err
+	}
+	meta := node.ObjectMeta
+	if meta == nil {
+		return nil, fmt.Errorf("node.ObjectMeta must be provided to Apply")
+	}
+	name := meta.Name
+	if name == nil {
+		return nil, fmt.Errorf("node.ObjectMeta.Name must be provided to Apply")
+	}
+	return m.Patch(ctx, *name, types.ApplyPatchType, data, patchOpts, subresources...)
+}
+
 // FakeRecorder is used as a fake during testing.
 type FakeRecorder struct {
 	sync.Mutex