diff --git a/prow/config/inrepoconfig.go b/prow/config/inrepoconfig.go index f978e311e59cb..c6443ead6963c 100644 --- a/prow/config/inrepoconfig.go +++ b/prow/config/inrepoconfig.go @@ -379,7 +379,13 @@ func (c *InRepoConfigGitCache) ClientFor(org, repo string) (git.RepoClient, erro if cached != nil || err != nil { return cached, err } - coreClient, err := c.ClientFactory.ClientFor(org, repo) + repoOpts := c.ClientFactory.RepoOpts{ + // We only need ".prow" because the default "cone mode" of sparse + // checkouts already include files at the toplevel. + sparseCheckoutDirs: []string{".prow"}, + shareObjectsWithSourceRepo: true, + } + coreClient, err := c.ClientFactory.ClientForWithRepoOpts(org, repo, repoOpts) if err != nil { return nil, err } diff --git a/prow/git/v2/adapter.go b/prow/git/v2/adapter.go index e2715db701574..540b1afcc3c3a 100644 --- a/prow/git/v2/adapter.go +++ b/prow/git/v2/adapter.go @@ -59,6 +59,12 @@ func (a *clientFactoryAdapter) ClientFor(org, repo string) (RepoClient, error) { return &repoClientAdapter{Repo: r}, err } +// v1 clients do not support customizing pre-repo options. +func (a *clientFactoryAdapter) ClientForWithRepoOpts(org, repo string, repoOpts RepoOpts) (RepoClient, error) { + r, err := a.Client.Clone(org, repo) + return &repoClientAdapter{Repo: r}, err +} + type repoClientAdapter struct { *git.Repo } diff --git a/prow/git/v2/client_factory.go b/prow/git/v2/client_factory.go index d44198ed27bbe..8b1f36b3a4195 100644 --- a/prow/git/v2/client_factory.go +++ b/prow/git/v2/client_factory.go @@ -35,6 +35,10 @@ type ClientFactory interface { ClientFromDir(org, repo, dir string) (RepoClient, error) // ClientFor creates a client that operates on a new clone of the repo. ClientFor(org, repo string) (RepoClient, error) + // ClientForWithRepoOpts is like ClientFor, but allows you to customize the + // setup of the cloned repo (such as sparse checkouts instead of using the + // default full clone). + ClientForWithRepoOpts(org, repo string, repoOpts RepoOpts) (RepoClient, error) // Clean removes the caches used to generate clients Clean() error @@ -75,6 +79,21 @@ type ClientFactoryOpts struct { Persist *bool } +// These options are scoped to the repo, not the ClientFactory level. The reason +// for the separation is to allow a single process to have for example repos +// that are both sparsely checked out and non-sparsely checked out. +type RepoOpts struct { + // sparseCheckoutDirs is the list of directories that the working tree + // should have. If non-nil and empty, then the working tree only has files + // reachable from the root. If non-nil and non-empty, then those additional + // directories from the root are also checked out (populated) in the working + // tree, recursively. + sparseCheckoutDirs *[]string + // This is the `--share` flag to `git clone`. For cloning from a local + // source, it allows bypassing the copying of all objects. + shareObjectsWithSourceRepo bool +} + // Apply allows to use a ClientFactoryOpts as Opt func (cfo *ClientFactoryOpts) Apply(target *ClientFactoryOpts) { if cfo.Host != "" { @@ -287,6 +306,17 @@ func (c *clientFactory) ClientFromDir(org, repo, dir string) (RepoClient, error) // org and repo are used for determining where the repo is cloned, cloneURI // overrides org/repo for cloning. func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) { + // These RepoOpts are the default, and is here to preserve behavior of + // existing callsites that did not previously know about + // ClientForWithRepoOpts. + repoOpts := RepoOpts{ + sparseCheckoutDirs: nil, + shareObjectsWithSourceRepo: false, + } + return c.ClientForWithRepoOpts(org, repo, repoOpts) +} + +func (c *clientFactory) ClientForWithRepoOpts(org, repo string, repoOpts RepoOpts) (RepoClient, error) { cacheDir := path.Join(c.cacheDir, org, repo) c.logger.WithFields(logrus.Fields{"org": org, "repo": repo, "dir": cacheDir}).Debug("Creating a client from the cache.") cacheClientCacher, _, _, err := c.bootstrapClients(org, repo, cacheDir) @@ -324,7 +354,7 @@ func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) { } // initialize the new derivative repo from the cache - if err := repoClientCloner.Clone(cacheDir); err != nil { + if err := repoClientCloner.CloneWithRepoOpts(cacheDir, repoOpts); err != nil { return nil, err } diff --git a/prow/git/v2/interactor.go b/prow/git/v2/interactor.go index c82af6585d08c..a3535b78f6ba1 100644 --- a/prow/git/v2/interactor.go +++ b/prow/git/v2/interactor.go @@ -88,6 +88,7 @@ type cacher interface { type cloner interface { // Clone clones the repository from a local path. Clone(from string) error + CloneWithRepoOpts(from string, repoOpts RepoOpts) error } // MergeOpt holds options for git merge operations. @@ -139,10 +140,47 @@ func (i *interactor) IsDirty() (bool, error) { // Clone clones the repository from a local path. func (i *interactor) Clone(from string) error { + repoOpts := RepoOpts{ + sparseCheckoutDirs: nil, + shareObjectsWithSourceRepo: false, + } + return i.CloneWithRepoOpts(from, repoOpts) +} + +// Clone clones the repository from a local path, but additionally use any +// repository options to customize the clone behavior. +func (i *interactor) CloneWithRepoOpts(from string, repoOpts RepoOpts) error { i.logger.Infof("Creating a clone of the repo at %s from %s", i.dir, from) - if out, err := i.executor.Run("clone", from, i.dir); err != nil { + cloneArgs := []string{"clone"} + + if repoOpts.shareObjectsWithSourceRepo { + cloneArgs = append(cloneArgs, "--shared") + } + + // Handle sparse checkouts. + if repoOpts.sparseCheckoutDirs != nil { + cloneArgs = append(cloneArgs, "--sparse") + } + cloneArgs = append(cloneArgs, from) + cloneArgs = append(cloneArgs, i.dir) + + if out, err := i.executor.Run(cloneArgs...); err != nil { return fmt.Errorf("error creating a clone: %w %v", err, string(out)) } + + // For sparse checkouts, we have to do some additional housekeeping after + // the clone is completed. We use Git's global "-C " flag to + // switch to that directory before running the "sparse-checkout" command, + // because otherwise the command will fail (because it will try to run the + // command in the $PWD, which is not the same as the just-created clone + // directory (i.dir)). + if repoOpts.sparseCheckoutDirs != nil { + sparseCheckoutArgs := []string{"-C", i.dir, "sparse-checkout", "set"} + sparseCheckoutArgs = append(sparseCheckoutArgs, (*repoOpts.sparseCheckoutDirs)...) + if out, err := i.executor.Run(sparseCheckoutArgs...); err != nil { + return fmt.Errorf("error setting it to a sparse checkout: %w %v", err, string(out)) + } + } return nil } diff --git a/prow/git/v2/interactor_test.go b/prow/git/v2/interactor_test.go index 687453c494d03..a3f9e51280235 100644 --- a/prow/git/v2/interactor_test.go +++ b/prow/git/v2/interactor_test.go @@ -93,6 +93,100 @@ func TestInteractor_Clone(t *testing.T) { } } +func TestInteractor_CloneWithRepoOpts(t *testing.T) { + var testCases = []struct { + name string + dir string + from string + remote RemoteResolver + repoOpts RepoOpts + responses map[string]execResponse + expectedCalls [][]string + expectedErr bool + }{ + { + name: "blank RepoOpts", + dir: "/else", + from: "/somewhere", + repoOpts: RepoOpts{}, + responses: map[string]execResponse{ + "clone /somewhere /else": { + out: []byte(`ok`), + }, + }, + expectedCalls: [][]string{ + {"clone", "/somewhere", "/else"}, + }, + expectedErr: false, + }, + { + name: "shared git objects", + dir: "/else", + from: "/somewhere", + repoOpts: RepoOpts{ + sparseCheckoutDirs: nil, + shareObjectsWithSourceRepo: true, + }, + responses: map[string]execResponse{ + "clone --shared /somewhere /else": { + out: []byte(`ok`), + }, + }, + expectedCalls: [][]string{ + {"clone", "--shared", "/somewhere", "/else"}, + }, + expectedErr: false, + }, + { + name: "shared git objects and sparse checkout", + dir: "/else", + from: "/somewhere", + repoOpts: RepoOpts{ + sparseCheckoutDirs: &[]string{"a", "b"}, + shareObjectsWithSourceRepo: true, + }, + responses: map[string]execResponse{ + "clone --shared --sparse /somewhere /else": { + out: []byte(`ok`), + }, + "-C /else sparse-checkout set a b": { + out: []byte(`ok`), + }, + }, + expectedCalls: [][]string{ + {"clone", "--shared", "--sparse", "/somewhere", "/else"}, + {"-C", "/else", "sparse-checkout", "set", "a", "b"}, + }, + expectedErr: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + e := fakeExecutor{ + records: [][]string{}, + responses: testCase.responses, + } + i := interactor{ + executor: &e, + remote: testCase.remote, + dir: testCase.dir, + logger: logrus.WithField("test", testCase.name), + } + actualErr := i.CloneWithRepoOpts(testCase.from, testCase.repoOpts) + if testCase.expectedErr && actualErr == nil { + t.Errorf("%s: expected an error but got none", testCase.name) + } + if !testCase.expectedErr && actualErr != nil { + t.Errorf("%s: expected no error but got one: %v", testCase.name, actualErr) + } + if actual, expected := e.records, testCase.expectedCalls; !reflect.DeepEqual(actual, expected) { + t.Errorf("%s: got incorrect git calls: %v", testCase.name, diff.ObjectReflectDiff(actual, expected)) + } + }) + } +} + func TestInteractor_MirrorClone(t *testing.T) { var testCases = []struct { name string