feat(internal/librarian): only generate libraries with changed APIs (#2618)

The new -generate-unchanged flag can be used to generate all (unblocked)
libraries. Libraries which are blocked by configuration are still not
generated when -generate-unchanged is specified.

Specifying -library ignores this check (and blocking) and will always
generate the specified library.

Fixes #818

Author: jskeet

cmd/librarian/doc.go

@@ -104,6 +104,9 @@ Flags:
 	-build
 	  	If true, Librarian will build each generated library by invoking the
 	  	language-specific container.
+	-generate-unchanged
+	  	If true, librarian generates libraries even if none of their associated APIs
+	  	have changed. This does not override generation being blocked by configuration.
 	-host-mount string
 	  	For use when librarian is running in a container. A mapping of a
 	  	directory from the host to the container, in the format

internal/config/config.go

@@ -123,12 +123,18 @@ type Config struct {
 	// expected.
 	CommandName string
 
-	// Commit determines whether to creat a commit for the release but not create
+	// Commit determines whether to create a commit for the release but not create
 	// a pull request.
 	//
 	// This flag is ignored if Push is set to true.
 	Commit bool
 
+	// GenerateUnchanged determines whether to generate libraries where none of
+	// the associated APIs have changed since the commit at which they were last
+	// generated. Note that this does not override any configuration indicating
+	// that the library should not be automatically generated.
+	GenerateUnchanged bool
+
 	// GitHubAPIEndpoint is the GitHub API endpoint to use for all GitHub API
 	// operations.
 	//

internal/gitrepo/gitrepo.go

@@ -55,6 +55,7 @@ type Repository interface {
 	CleanUntracked(paths []string) error
 	pushRefSpec(refSpec string) error
 	Checkout(commitHash string) error
+	GetHashForPath(commitHash, path string) (string, error)
 }
 
 const RootPath = "."
@@ -419,20 +420,20 @@ func commitMatchesPath(path string, commit *object.Commit, parentCommit *object.
 	if path == RootPath {
 		return true, nil
 	}
-	currentPathHash, err := getHashForPathOrEmpty(commit, path)
+	currentPathHash, err := getHashForPath(commit, path)
 	if err != nil {
 		return false, err
 	}
-	parentPathHash, err := getHashForPathOrEmpty(parentCommit, path)
+	parentPathHash, err := getHashForPath(parentCommit, path)
 	if err != nil {
 		return false, err
 	}
 	return currentPathHash != parentPathHash, nil
 }
 
-// getHashForPathOrEmpty returns the hash for a path at a given commit, or an
+// getHashForPath returns the hash for a path at a given commit, or an
 // empty string if the path (file or directory) did not exist.
-func getHashForPathOrEmpty(commit *object.Commit, path string) (string, error) {
+func getHashForPath(commit *object.Commit, path string) (string, error) {
 	tree, err := commit.Tree()
 	if err != nil {
 		return "", err
@@ -663,3 +664,19 @@ func (r *LocalRepository) Checkout(commitSha string) error {
 		Hash: plumbing.NewHash(commitSha),
 	})
 }
+
+// GetHashForPath returns a tree hash for the specified path,
+// at the given commit in this repository. If the path does not exist
+// at the commit, an empty string is returned rather than an error,
+// as the purpose of this function is to allow callers to determine changes
+// in the tree. (A path going from missing to anything else, or vice versa,
+// indicates a change. A path being missing at two different commits is not a change.)
+func (r *LocalRepository) GetHashForPath(commitHash, path string) (string, error) {
+	// This public function just delegates to the internal function that uses a Commit
+	// object instead of the hash.
+	commit, err := r.repo.CommitObject(plumbing.NewHash(commitHash))
+	if err != nil {
+		return "", err
+	}
+	return getHashForPath(commit, path)
+}

internal/gitrepo/gitrepo_test.go

@@ -904,27 +904,34 @@ func TestGetDir(t *testing.T) {
 	}
 }
 
-func TestGetHashForPathOrEmpty(t *testing.T) {
+// TestGetHashForPath tests the internal getHashForPath, but
+// via the public GetHashForPath function which accepts a commit hash
+// instead of a Commit object, to avoid duplicate testing.
+func TestGetHashForPath(t *testing.T) {
 	t.Parallel()
 
-	setupInitialRepo := func(t *testing.T) (*git.Repository, *object.Commit) {
+	setupInitialRepo := func(t *testing.T) (LocalRepository, *object.Commit) {
 		t.Helper()
-		repo, _ := initTestRepo(t)
+		repo, dir := initTestRepo(t)
 		commit := createAndCommit(t, repo, "initial.txt", []byte("initial content"), "initial commit")
-		return repo, commit
+		localRepository := LocalRepository{
+			Dir:  dir,
+			repo: repo,
+		}
+		return localRepository, commit
 	}
 
 	for _, test := range []struct {
 		name     string
-		setup    func(t *testing.T) (commit *object.Commit, path string)
+		setup    func(t *testing.T) (repo LocalRepository, commit *object.Commit, path string)
 		wantHash func(commit *object.Commit, path string) string
 		wantErr  bool
 	}{
 		{
 			name: "existing file",
-			setup: func(t *testing.T) (*object.Commit, string) {
-				_, commit := setupInitialRepo(t)
-				return commit, "initial.txt"
+			setup: func(t *testing.T) (LocalRepository, *object.Commit, string) {
+				localRepository, commit := setupInitialRepo(t)
+				return localRepository, commit, "initial.txt"
 			},
 			wantHash: func(commit *object.Commit, path string) string {
 				tree, err := commit.Tree()
@@ -940,8 +947,9 @@ func TestGetHashForPathOrEmpty(t *testing.T) {
 		},
 		{
 			name: "existing directory",
-			setup: func(t *testing.T) (*object.Commit, string) {
-				repo, _ := setupInitialRepo(t)
+			setup: func(t *testing.T) (LocalRepository, *object.Commit, string) {
+				localRepository, _ := setupInitialRepo(t)
+				repo := localRepository.repo
 				// Create a directory and a file inside it to ensure the directory gets a hash
 				_ = createAndCommit(t, repo, "my_dir/file_in_dir.txt", []byte("content of file in dir"), "add dir and file")
 				head, err := repo.Head()
@@ -952,7 +960,7 @@ func TestGetHashForPathOrEmpty(t *testing.T) {
 				if err != nil {
 					t.Fatalf("repo.CommitObject failed: %v", err)
 				}
-				return commit, "my_dir"
+				return localRepository, commit, "my_dir"
 			},
 			wantHash: func(commit *object.Commit, path string) string {
 				tree, err := commit.Tree()
@@ -968,28 +976,29 @@ func TestGetHashForPathOrEmpty(t *testing.T) {
 		},
 		{
 			name: "non-existent file",
-			setup: func(t *testing.T) (*object.Commit, string) {
-				_, commit := setupInitialRepo(t)
-				return commit, "non_existent_file.txt"
+			setup: func(t *testing.T) (LocalRepository, *object.Commit, string) {
+				localRepository, commit := setupInitialRepo(t)
+				return localRepository, commit, "non_existent_file.txt"
 			},
 			wantHash: func(commit *object.Commit, path string) string {
 				return ""
 			},
 		},
 		{
 			name: "non-existent directory",
-			setup: func(t *testing.T) (*object.Commit, string) {
-				_, commit := setupInitialRepo(t)
-				return commit, "non_existent_dir"
+			setup: func(t *testing.T) (LocalRepository, *object.Commit, string) {
+				localRepository, commit := setupInitialRepo(t)
+				return localRepository, commit, "non_existent_dir"
 			},
 			wantHash: func(commit *object.Commit, path string) string {
 				return ""
 			},
 		},
 		{
 			name: "file in subdirectory",
-			setup: func(t *testing.T) (*object.Commit, string) {
-				repo, _ := setupInitialRepo(t)
+			setup: func(t *testing.T) (LocalRepository, *object.Commit, string) {
+				localRepository, _ := setupInitialRepo(t)
+				repo := localRepository.repo
 				_ = createAndCommit(t, repo, "another_dir/sub_dir/nested_file.txt", []byte("nested content"), "add nested file")
 				head, err := repo.Head()
 				if err != nil {
@@ -999,7 +1008,7 @@ func TestGetHashForPathOrEmpty(t *testing.T) {
 				if err != nil {
 					t.Fatalf("repo.CommitObject failed: %v", err)
 				}
-				return commit, "another_dir/sub_dir/nested_file.txt"
+				return localRepository, commit, "another_dir/sub_dir/nested_file.txt"
 			},
 			wantHash: func(commit *object.Commit, path string) string {
 				tree, err := commit.Tree()
@@ -1017,17 +1026,52 @@ func TestGetHashForPathOrEmpty(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 
-			commit, path := test.setup(t)
+			localRepository, commit, path := test.setup(t)
 
-			got, err := getHashForPathOrEmpty(commit, path)
+			got, err := localRepository.GetHashForPath(commit.Hash.String(), path)
 			if (err != nil) != test.wantErr {
-				t.Errorf("getHashForPathOrEmpty() error = %v, wantErr %v", err, test.wantErr)
+				t.Errorf("getHashForPath() error = %v, wantErr %v", err, test.wantErr)
 				return
 			}
 
 			wantHash := test.wantHash(commit, path)
 			if diff := cmp.Diff(wantHash, got); diff != "" {
-				t.Errorf("getHashForPathOrEmpty() mismatch (-want +got):\n%s", diff)
+				t.Errorf("getHashForPath() mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+// TestGetHashForPathBadCommitHash tests the one path not
+// otherwise tested in TestGetHashForPath, where we can't
+// get the commit for the hash.
+func TestGetHashForPathBadCommitHash(t *testing.T) {
+	repo, dir := initTestRepo(t)
+	localRepository := LocalRepository{
+		Dir:  dir,
+		repo: repo,
+	}
+	for _, test := range []struct {
+		name       string
+		commitHash string
+	}{
+		{
+			name:       "empty hash",
+			commitHash: "",
+		},
+		{
+			name:       "invalid hash",
+			commitHash: "bad-hash",
+		},
+		{
+			name:       "hash not in repo",
+			commitHash: "d93e160f57f0a6eccd6e230dd40f465988bede63",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			_, err := localRepository.GetHashForPath(test.commitHash, "path/to/file")
+			if err == nil {
+				t.Error("GetHashForPath() err = nil, should fail when an invalid or absent hash is provided")
 			}
 		})
 	}

internal/librarian/flags.go

@@ -45,6 +45,12 @@ func addFlagCommit(fs *flag.FlagSet, cfg *config.Config) {
 a pull request. This flag is ignored if push is set to true.`)
 }
 
+func addFlagGenerateUnchanged(fs *flag.FlagSet, cfg *config.Config) {
+	fs.BoolVar(&cfg.GenerateUnchanged, "generate-unchanged", false,
+		`If true, librarian generates libraries even if none of their associated APIs
+have changed. This does not override generation being blocked by configuration.`)
+}
+
 func addFlagGitHubAPIEndpoint(fs *flag.FlagSet, cfg *config.Config) {
 	fs.StringVar(&cfg.GitHubAPIEndpoint, "github-api-endpoint", "",
 		`The GitHub API endpoint to use for all GitHub API operations.