kubectl: Allow parallelizing diffing

staging/src/k8s.io/cli-runtime/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.2
+	golang.org/x/sync v0.2.0
 	golang.org/x/text v0.9.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.0.0

staging/src/k8s.io/cli-runtime/pkg/resource/builder.go

@@ -79,6 +79,8 @@ type Builder struct {
 	stdinInUse bool
 	dir        bool
 
+	visitorConcurrency int
+
 	labelSelector     *string
 	fieldSelector     *string
 	selectAll         bool
@@ -233,6 +235,13 @@ func (b *Builder) AddError(err error) *Builder {
 	return b
 }
 
+// VisitorConcurrency sets the number of concurrent visitors to use when
+// visiting lists.
+func (b *Builder) VisitorConcurrency(concurrency int) *Builder {
+	b.visitorConcurrency = concurrency
+	return b
+}
+
 // FilenameParam groups input in two categories: URLs and files (files, directories, STDIN)
 // If enforceNamespace is false, namespaces in the specs will be allowed to
 // override the default namespace. If it is true, namespaces that don't match
@@ -1124,7 +1133,10 @@ func (b *Builder) visitByPaths() *Result {
 	if b.continueOnError {
 		visitors = EagerVisitorList(b.paths)
 	} else {
-		visitors = VisitorList(b.paths)
+		visitors = ConcurrentVisitorList{
+			visitors:    b.paths,
+			concurrency: b.visitorConcurrency,
+		}
 	}
 
 	if b.flatten {

staging/src/k8s.io/cli-runtime/pkg/resource/visitor.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/sync/errgroup"
 	"golang.org/x/text/encoding/unicode"
 	"golang.org/x/text/transform"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -201,6 +202,33 @@ func (l VisitorList) Visit(fn VisitorFunc) error {
 	return nil
 }
 
+type ConcurrentVisitorList struct {
+	visitors    []Visitor
+	concurrency int
+}
+
+func (l ConcurrentVisitorList) Visit(fn VisitorFunc) error {
+	g := errgroup.Group{}
+
+	// Concurrency 1 just runs the visitors sequentially, this is the default
+	// as it preserves the previous behavior, but allows components to opt into
+	// concurrency.
+	concurrency := 1
+	if l.concurrency > concurrency {
+		concurrency = l.concurrency
+	}
+	g.SetLimit(concurrency)
+
+	for i := range l.visitors {
+		i := i
+		g.Go(func() error {
+			return l.visitors[i].Visit(fn)
+		})
+	}
+
+	return g.Wait()
+}
+
 // EagerVisitorList implements Visit for the sub visitors it contains. All errors
 // will be captured and returned at the end of iteration.
 type EagerVisitorList []Visitor

staging/src/k8s.io/kubectl/go.mod

@@ -82,6 +82,7 @@ require (
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/term v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

staging/src/k8s.io/kubectl/pkg/cmd/diff/diff.go

@@ -107,6 +107,7 @@ type DiffOptions struct {
 	ForceConflicts    bool
 	ShowManagedFields bool
 
+	Concurrency      int
 	Selector         string
 	OpenAPISchema    openapi.Resources
 	DynamicClient    dynamic.Interface
@@ -166,6 +167,7 @@ func NewCmdDiff(f cmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Co
 	cmd.Flags().StringArray("prune-allowlist", []string{}, "Overwrite the default whitelist with <group/version/kind> for --prune")
 	cmd.Flags().Bool("prune", false, "Include resources that would be deleted by pruning. Can be used with -l and default shows all resources would be pruned")
 	cmd.Flags().BoolVar(&options.ShowManagedFields, "show-managed-fields", options.ShowManagedFields, "If true, include managed fields in the diff.")
+	cmd.Flags().IntVar(&options.Concurrency, "concurrency", 1, "Number of objects to process in parallel when diffing against the live version. Larger number = faster, but more memory, I/O and CPU over that shorter period of time.")
 	cmdutil.AddFilenameOptionFlags(cmd, &options.FilenameOptions, usage)
 	cmdutil.AddServerSideApplyFlags(cmd)
 	cmdutil.AddFieldManagerFlagVar(cmd, &options.FieldManager, apply.FieldManagerClientSideApply)
@@ -683,6 +685,7 @@ func (o *DiffOptions) Run() error {
 
 	r := o.Builder.
 		Unstructured().
+		VisitorConcurrency(o.Concurrency).
 		NamespaceParam(o.CmdNamespace).DefaultNamespace().
 		FilenameParam(o.EnforceNamespace, &o.FilenameOptions).
 		LabelSelectorParam(o.Selector).

staging/src/k8s.io/sample-cli-plugin/go.mod

@@ -45,6 +45,7 @@ require (
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect

vendor/modules.txt

@@ -994,6 +994,7 @@ golang.org/x/oauth2/jws
 golang.org/x/oauth2/jwt
 # golang.org/x/sync v0.2.0
 ## explicit
+golang.org/x/sync/errgroup
 golang.org/x/sync/singleflight
 # golang.org/x/sys v0.8.0
 ## explicit; go 1.17