Shared repositories (git mirrors) between checkouts

.buildkite/steps/tests.sh

@@ -4,4 +4,7 @@ set -euo pipefail
 GO111MODULE=off go get gotest.tools/gotestsum
 
 echo '+++ Running tests'
-gotestsum --junitfile "junit-${OSTYPE}.xml" ./...
+gotestsum --junitfile "junit-${OSTYPE}.xml" -- -count=1 -failfast ./...
+
+echo '+++ Running integration tests for git-mirrors experiment'
+TEST_EXPERIMENT=git-mirrors gotestsum --junitfile "junit-${OSTYPE}-git-mirrors.xml" -- -count=1 -failfast ./bootstrap/integration

EXPERIMENTS.md

@@ -18,6 +18,14 @@ If an experiment doesn't exist, no error will be raised.
 
 ## Available Experiments
 
+### `git-mirrors`
+
+Maintain a single bare git mirror with for each repository on a host that is shared amongst multiple agents and pipelines. Checkouts reference the git mirror using `git clone --reference`, as do submodules.
+
+You must set a `git-mirrors-path` in your config for this to work.
+
+**Status**: broadly useful, we'd like this to be the standard behaviour in 4.0. 👍👍
+
 ### `agent-socket`
 
 The agent currently exposes a per-session token to jobs called `BUILDKITE_AGENT_ACCESS_TOKEN`. This token can be used for pipeline uploads, meta-data get/set and artifact access within the job. Leaking it in logging can be dangerous, as anyone with that token can access whatever your agent could.

agent/agent_configuration.go

@@ -5,8 +5,11 @@ type AgentConfiguration struct {
 	BootstrapScript            string
 	BuildPath                  string
 	HooksPath                  string
+	GitMirrorsPath             string
+	GitMirrorsLockTimeout      int
 	PluginsPath                string
 	GitCloneFlags              string
+	GitCloneMirrorFlags        string
 	GitCleanFlags              string
 	GitSubmodules              bool
 	SSHKeyscan                 bool

agent/job_runner.go

@@ -394,6 +394,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 		`BUILDKITE_BIN_PATH`,
 		`BUILDKITE_CONFIG_PATH`,
 		`BUILDKITE_BUILD_PATH`,
+		`BUILDKITE_GIT_MIRRORS_PATH`,
 		`BUILDKITE_HOOKS_PATH`,
 		`BUILDKITE_PLUGINS_PATH`,
 		`BUILDKITE_SSH_KEYSCAN`,
@@ -402,6 +403,8 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 		`BUILDKITE_PLUGINS_ENABLED`,
 		`BUILDKITE_LOCAL_HOOKS_ENABLED`,
 		`BUILDKITE_GIT_CLONE_FLAGS`,
+		`BUILDKITE_GIT_CLONE_MIRROR_FLAGS`,
+		`BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT`,
 		`BUILDKITE_GIT_CLEAN_FLAGS`,
 		`BUILDKITE_SHELL`,
 	}
@@ -440,6 +443,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 	// Add options from the agent configuration
 	env["BUILDKITE_CONFIG_PATH"] = r.conf.AgentConfiguration.ConfigPath
 	env["BUILDKITE_BUILD_PATH"] = r.conf.AgentConfiguration.BuildPath
+	env["BUILDKITE_GIT_MIRRORS_PATH"] = r.conf.AgentConfiguration.GitMirrorsPath
 	env["BUILDKITE_HOOKS_PATH"] = r.conf.AgentConfiguration.HooksPath
 	env["BUILDKITE_PLUGINS_PATH"] = r.conf.AgentConfiguration.PluginsPath
 	env["BUILDKITE_SSH_KEYSCAN"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.SSHKeyscan)
@@ -448,7 +452,9 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 	env["BUILDKITE_PLUGINS_ENABLED"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.PluginsEnabled)
 	env["BUILDKITE_LOCAL_HOOKS_ENABLED"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.LocalHooksEnabled)
 	env["BUILDKITE_GIT_CLONE_FLAGS"] = r.conf.AgentConfiguration.GitCloneFlags
+	env["BUILDKITE_GIT_CLONE_MIRROR_FLAGS"] = r.conf.AgentConfiguration.GitCloneMirrorFlags
 	env["BUILDKITE_GIT_CLEAN_FLAGS"] = r.conf.AgentConfiguration.GitCleanFlags
+	env["BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT"] = fmt.Sprintf("%d", r.conf.AgentConfiguration.GitMirrorsLockTimeout)
 	env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell
 	env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(), ",")
 

bootstrap/bootstrap.go

@@ -17,6 +17,7 @@ import (
 	"github.com/buildkite/agent/agent/plugin"
 	"github.com/buildkite/agent/bootstrap/shell"
 	"github.com/buildkite/agent/env"
+	"github.com/buildkite/agent/experiments"
 	"github.com/buildkite/agent/process"
 	"github.com/buildkite/agent/retry"
 	"github.com/buildkite/shellwords"
@@ -352,6 +353,11 @@ func dirForAgentName(agentName string) string {
 	return badCharsPattern.ReplaceAllString(agentName, "-")
 }
 
+func dirForRepository(repository string) string {
+	badCharsPattern := regexp.MustCompile("[[:^alnum:]]")
+	return badCharsPattern.ReplaceAllString(repository, "-")
+}
+
 // Given a repository, it will add the host to the set of SSH known_hosts on the machine
 func addRepositoryHostToSSHKnownHosts(sh *shell.Shell, repository string) {
 	if fileExists(repository) {
@@ -946,16 +952,128 @@ func hasGitSubmodules(sh *shell.Shell) bool {
 	return fileExists(filepath.Join(sh.Getwd(), ".gitmodules"))
 }
 
+func hasGitCommit(sh *shell.Shell, gitDir string, commit string) bool {
+	// Resolve commit to an actual commit object
+	output, err := sh.RunAndCapture("git", "--git-dir", gitDir, "rev-parse", commit+"^{commit}")
+	if err != nil {
+		return false
+	}
+
+	// Filter out commitish things like HEAD et al
+	if strings.TrimSpace(output) != commit {
+		return false
+	}
+
+	// Otherwise it's a commit in the repo
+	return true
+}
+
+func (b *Bootstrap) updateGitMirror() (string, error) {
+	// Create a unique directory for the repository mirror
+	mirrorDir := filepath.Join(b.Config.GitMirrorsPath, dirForRepository(b.Repository))
+
+	// Create the mirrors path if it doesn't exist
+	if baseDir := filepath.Dir(mirrorDir); !fileExists(baseDir) {
+		b.shell.Commentf("Creating \"%s\"", baseDir)
+		if err := os.MkdirAll(baseDir, 0777); err != nil {
+			return "", err
+		}
+	}
+
+	b.shell.Chdir(b.Config.GitMirrorsPath)
+
+	lockTimeout := time.Second * time.Duration(b.GitMirrorsLockTimeout)
+
+	if b.Debug {
+		b.shell.Commentf("Acquiring mirror repository clone lock")
+	}
+
+	// Lock the mirror dir to prevent concurrent clones
+	mirrorCloneLock, err := b.shell.LockFile(mirrorDir+".clonelock", lockTimeout)
+	if err != nil {
+		return "", err
+	}
+	defer mirrorCloneLock.Unlock()
+
+	// If we don't have a mirror, we need to clone it
+	if !fileExists(mirrorDir) {
+		b.shell.Commentf("Cloning a mirror of the repository to %q", mirrorDir)
+		if err := gitClone(b.shell, b.GitCloneMirrorFlags, b.Repository, mirrorDir); err != nil {
+			return "", err
+		}
+
+		return mirrorDir, nil
+	}
+
+	// If it exists, immediately release the clone lock
+	mirrorCloneLock.Unlock()
+
+	// Check if the mirror has a commit, this is atomic so should be safe to do
+	if hasGitCommit(b.shell, mirrorDir, b.Commit) {
+		b.shell.Commentf("Commit %q exists in mirror", b.Commit)
+		return mirrorDir, nil
+	}
+
+	if b.Debug {
+		b.shell.Commentf("Acquiring mirror repository update lock")
+	}
+
+	// Lock the mirror dir to prevent concurrent updates
+	mirrorUpdateLock, err := b.shell.LockFile(mirrorDir+".updatelock", lockTimeout)
+	if err != nil {
+		return "", err
+	}
+	defer mirrorUpdateLock.Unlock()
+
+	// Check again after we get a lock, in case the other process has already updated
+	if hasGitCommit(b.shell, mirrorDir, b.Commit) {
+		b.shell.Commentf("Commit %q exists in mirror", b.Commit)
+		return mirrorDir, nil
+	}
+
+	b.shell.Commentf("Updating existing repository mirror to find commit %s", b.Commit)
+
+	// Update the the origin of the repository so we can gracefully handle repository renames
+	if err := b.shell.Run("git", "--git-dir", mirrorDir, "remote", "set-url", "origin", b.Repository); err != nil {
+		return "", err
+	}
+
+	// Update our mirror
+	if err := b.shell.Run("git", "--git-dir", mirrorDir, "remote", "update", "--prune"); err != nil {
+		return "", err
+	}
+
+	return mirrorDir, nil
+}
+
 // defaultCheckoutPhase is called by the CheckoutPhase if no global or plugin checkout
 // hook exists. It performs the default checkout on the Repository provided in the config
 func (b *Bootstrap) defaultCheckoutPhase() error {
-	// Make sure the build directory exists
+	if b.SSHKeyscan {
+		addRepositoryHostToSSHKnownHosts(b.shell, b.Repository)
+	}
+
+	var mirrorDir string
+
+	// If we can, get a mirror of the git repository to use for reference later
+	if experiments.IsEnabled(`git-mirrors`) && b.Config.GitMirrorsPath != "" && b.Config.Repository != "" {
+		b.shell.Commentf("Using git-mirrors experiment 🧪")
+
+		var err error
+		mirrorDir, err = b.updateGitMirror()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Make sure the build directory exists and that we change directory into it
 	if err := b.createCheckoutDir(); err != nil {
 		return err
 	}
 
-	if b.SSHKeyscan {
-		addRepositoryHostToSSHKnownHosts(b.shell, b.Repository)
+	gitCloneFlags := b.GitCloneFlags
+	if mirrorDir != "" {
+		gitCloneFlags += fmt.Sprintf(" --reference %q", mirrorDir)
 	}
 
 	// Does the git directory exist?
@@ -966,7 +1084,7 @@ func (b *Bootstrap) defaultCheckoutPhase() error {
 			return err
 		}
 	} else {
-		if err := gitClone(b.shell, b.GitCloneFlags, b.Repository, "."); err != nil {
+		if err := gitClone(b.shell, gitCloneFlags, b.Repository, "."); err != nil {
 			return err
 		}
 	}
@@ -1053,19 +1171,6 @@ func (b *Bootstrap) defaultCheckoutPhase() error {
 	}
 
 	if gitSubmodules {
-		// submodules might need their fingerprints verified too
-		if b.SSHKeyscan {
-			b.shell.Commentf("Checking to see if submodule urls need to be added to known_hosts")
-			submoduleRepos, err := gitEnumerateSubmoduleURLs(b.shell)
-			if err != nil {
-				b.shell.Warningf("Failed to enumerate git submodules: %v", err)
-			} else {
-				for _, repository := range submoduleRepos {
-					addRepositoryHostToSSHKnownHosts(b.shell, repository)
-				}
-			}
-		}
-
 		// `submodule sync` will ensure the .git/config
 		// matches the .gitmodules file.  The command
 		// is only available in git version 1.8.1, so
@@ -1076,9 +1181,23 @@ func (b *Bootstrap) defaultCheckoutPhase() error {
 			b.shell.Warningf("Failed to recursively sync git submodules. This is most likely because you have an older version of git installed (" + gitVersionOutput + ") and you need version 1.8.1 and above. If you're using submodules, it's highly recommended you upgrade if you can.")
 		}
 
+		// Checking for submodule repositories
+		submoduleRepos, err := gitEnumerateSubmoduleURLs(b.shell)
+		if err != nil {
+			b.shell.Warningf("Failed to enumerate git submodules: %v", err)
+		} else {
+			for _, repository := range submoduleRepos {
+				// submodules might need their fingerprints verified too
+				if b.SSHKeyscan {
+					addRepositoryHostToSSHKnownHosts(b.shell, repository)
+				}
+			}
+		}
+
 		if err := b.shell.Run("git", "submodule", "update", "--init", "--recursive", "--force"); err != nil {
 			return err
 		}
+
 		if err := b.shell.Run("git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"); err != nil {
 			return err
 		}

bootstrap/config.go

@@ -64,6 +64,9 @@ type Config struct {
 	// Flags to pass to "git clone" command
 	GitCloneFlags string `env:"BUILDKITE_GIT_CLONE_FLAGS"`
 
+	// Flags to pass to "git clone" command for mirroring
+	GitCloneMirrorFlags string
+
 	// Flags to pass to "git clean" command
 	GitCleanFlags string `env:"BUILDKITE_GIT_CLEAN_FLAGS"`
 
@@ -85,6 +88,12 @@ type Config struct {
 	// Path where the builds will be run
 	BuildPath string
 
+	// Path where the repository mirrors are stored
+	GitMirrorsPath string
+
+	// Seconds to wait before allowing git mirror clone lock to be acquired
+	GitMirrorsLockTimeout int
+
 	// Path to the buildkite-agent binary
 	BinPath string
 

bootstrap/git.go

@@ -21,7 +21,7 @@ func gitClone(sh *shell.Shell, gitCloneFlags, repository, dir string) error {
 
 	commandArgs := []string{"clone"}
 	commandArgs = append(commandArgs, individualCloneFlags...)
-	commandArgs = append(commandArgs, "--", repository, ".")
+	commandArgs = append(commandArgs, "--", repository, dir)
 
 	if err = sh.Run("git", commandArgs...); err != nil {
 		return err

bootstrap/integration/bootstrap_tester.go

@@ -17,21 +17,24 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/buildkite/agent/experiments"
+
 	"github.com/buildkite/bintest"
 )
 
 // BootstrapTester invokes a buildkite-agent bootstrap script with a temporary environment
 type BootstrapTester struct {
-	Name       string
-	Args       []string
-	Env        []string
-	HomeDir    string
-	PathDir    string
-	BuildDir   string
-	HooksDir   string
-	PluginsDir string
-	Repo       *gitRepository
-	Output     string
+	Name          string
+	Args          []string
+	Env           []string
+	HomeDir       string
+	PathDir       string
+	BuildDir      string
+	GitMirrorsDir string
+	HooksDir      string
+	PluginsDir    string
+	Repo          *gitRepository
+	Output        string
 
 	cmd      *exec.Cmd
 	cmdLock  sync.Mutex
@@ -101,6 +104,21 @@ func NewBootstrapTester() (*BootstrapTester, error) {
 		PluginsDir: pluginsDir,
 	}
 
+	// Support testing experiments
+	if exp := experiments.Enabled(); len(exp) > 0 {
+		bt.Env = append(bt.Env, `BUILDKITE_AGENT_EXPERIMENT=`+strings.Join(exp, ","))
+
+		if experiments.IsEnabled(`git-mirrors`) {
+			gitMirrorsDir, err := ioutil.TempDir("", "bootstrap-git-mirrors")
+			if err != nil {
+				return nil, err
+			}
+
+			bt.GitMirrorsDir = gitMirrorsDir
+			bt.Env = append(bt.Env, "BUILDKITE_GIT_MIRRORS_PATH="+gitMirrorsDir)
+		}
+	}
+
 	// Windows requires certain env variables to be present
 	if runtime.GOOS == "windows" {
 		bt.Env = append(bt.Env,
@@ -321,6 +339,11 @@ func (b *BootstrapTester) Close() error {
 	if err := os.RemoveAll(b.PluginsDir); err != nil {
 		return err
 	}
+	if b.GitMirrorsDir != "" {
+		if err := os.RemoveAll(b.GitMirrorsDir); err != nil {
+			return err
+		}
+	}
 	return nil
 }