feat(internal): add force option for regen (#4310)

The forceAll option indicates that all proto files should be
regenerated.

Author: codyoss

internal/gapicgen/cmd/genbot/bot.go

@@ -26,12 +26,20 @@ import (
 	"cloud.google.com/go/internal/gapicgen/git"
 )
 
-func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error {
+type botConfig struct {
+	githubAccessToken string
+	githubUsername    string
+	githubName        string
+	githubEmail       string
+	forceAll          bool
+}
+
+func genBot(ctx context.Context, c botConfig) error {
 	for k, v := range map[string]string{
-		"githubAccessToken": githubAccessToken,
-		"githubUsername":    githubUsername,
-		"githubName":        githubName,
-		"githubEmail":       githubEmail,
+		"githubAccessToken": c.githubAccessToken,
+		"githubUsername":    c.githubUsername,
+		"githubName":        c.githubName,
+		"githubEmail":       c.githubEmail,
 	} {
 		if v == "" {
 			log.Printf("missing or empty value for required flag --%s\n", k)
@@ -40,7 +48,7 @@ func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName,
 	}
 
 	// Setup the client and git environment.
-	githubClient, err := git.NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken)
+	githubClient, err := git.NewGithubClient(ctx, c.githubUsername, c.githubName, c.githubEmail, c.githubAccessToken)
 	if err != nil {
 		return err
 	}
@@ -73,7 +81,7 @@ func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName,
 		return nil
 	}
 
-	return generate(ctx, githubClient)
+	return generate(ctx, githubClient, c.forceAll)
 }
 
 // hasCreatedPRToday checks if the created time of a PR is from today.

internal/gapicgen/cmd/genbot/generate.go

@@ -31,7 +31,7 @@ import (
 
 // generate downloads sources and generates pull requests for go-genproto and
 // google-cloud-go if needed.
-func generate(ctx context.Context, githubClient *git.GithubClient) error {
+func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool) error {
 	log.Println("creating temp dir")
 	tmpDir, err := ioutil.TempDir("", "update-genproto")
 	if err != nil {
@@ -71,6 +71,7 @@ func generate(ctx context.Context, githubClient *git.GithubClient) error {
 		GenprotoDir:   genprotoDir,
 		GapicDir:      gocloudDir,
 		ProtoDir:      protoDir,
+		ForceAll:      forceAll,
 	}
 	changes, err := generator.Generate(ctx, conf)
 	if err != nil {

internal/gapicgen/cmd/genbot/main.go

@@ -41,6 +41,7 @@ func main() {
 	githubName := flag.String("githubName", os.Getenv("GITHUB_NAME"), "The name of the author for git commits.")
 	githubEmail := flag.String("githubEmail", os.Getenv("GITHUB_EMAIL"), "The email address of the author.")
 	localMode := flag.Bool("local", strToBool(os.Getenv("GENBOT_LOCAL_MODE")), "Enables generating sources locally. This mode will not open any pull requests.")
+	forceAll := flag.Bool("forceAll", strToBool(os.Getenv("GENBOT_FORCE_ALL")), "Enables regenerating everything regardless of changes in googleapis.")
 
 	// flags for local mode
 	googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
@@ -67,7 +68,13 @@ func main() {
 		}
 		return
 	}
-	if err := genBot(ctx, *githubAccessToken, *githubUsername, *githubName, *githubEmail); err != nil {
+	if err := genBot(ctx, botConfig{
+		githubAccessToken: *githubAccessToken,
+		githubUsername:    *githubUsername,
+		githubName:        *githubName,
+		githubEmail:       *githubEmail,
+		forceAll:          *forceAll,
+	}); err != nil {
 		log.Fatal(err)
 	}
 }

internal/gapicgen/generator/config.go

@@ -1021,6 +1021,7 @@ var microgenGapicConfigs = []*microgenConfig{
 		gRPCServiceConfigPath: "google/cloud/recommendationengine/v1beta1/recommendationengine_grpc_service_config.json",
 		apiServiceConfigPath:  "google/cloud/recommendationengine/v1beta1/recommendationengine_v1beta1.yaml",
 		releaseLevel:          "beta",
+		stopGeneration:        true,
 	},
 	{
 		inputDirectoryPath:    "google/cloud/gkehub/v1beta1",

internal/gapicgen/generator/generator.go

@@ -37,6 +37,7 @@ type Config struct {
 	OnlyGenerateGapic bool
 	LocalMode         bool
 	RegenOnly         bool
+	ForceAll          bool
 }
 
 // Generate generates genproto and gapics.

internal/gapicgen/generator/genproto.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -57,6 +58,7 @@ type GenprotoGenerator struct {
 	genprotoDir   string
 	googleapisDir string
 	protoSrcDir   string
+	forceAll      bool
 }
 
 // NewGenprotoGenerator creates a new GenprotoGenerator.
@@ -65,6 +67,7 @@ func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
 		genprotoDir:   c.GenprotoDir,
 		googleapisDir: c.GoogleapisDir,
 		protoSrcDir:   filepath.Join(c.ProtoDir, "/src"),
+		forceAll:      c.ForceAll,
 	}
 }
 
@@ -186,6 +189,9 @@ func (g *GenprotoGenerator) protoc(fileNames []string) error {
 // getUpdatedPackages parses all of the new commits to find what packages need
 // to be regenerated.
 func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) {
+	if g.forceAll {
+		return g.getAllPackages()
+	}
 	files, err := git.UpdateFilesSinceHash(g.googleapisDir, googleapisHash)
 	if err != nil {
 		return nil, err
@@ -205,6 +211,41 @@ func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[strin
 	return pkgFiles, nil
 }
 
+func (g *GenprotoGenerator) getAllPackages() (map[string][]string, error) {
+	seenFiles := make(map[string]bool)
+	pkgFiles := make(map[string][]string)
+	for _, root := range []string{g.googleapisDir} {
+		walkFn := func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") {
+				return nil
+			}
+
+			switch rel, err := filepath.Rel(root, path); {
+			case err != nil:
+				return err
+			case seenFiles[rel]:
+				return nil
+			default:
+				seenFiles[rel] = true
+			}
+
+			pkg, err := goPkg(path)
+			if err != nil {
+				return err
+			}
+			pkgFiles[pkg] = append(pkgFiles[pkg], path)
+			return nil
+		}
+		if err := filepath.Walk(root, walkFn); err != nil {
+			return nil, err
+		}
+	}
+	return pkgFiles, nil
+}
+
 // moveAndCleanupGeneratedSrc moves all generated src to their correct locations
 // in the repository, because protoc puts it in a folder called `generated/``.
 func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {