Skip to content

Commit

Permalink
feat(internal): add force option for regen (#4310)
Browse files Browse the repository at this point in the history
The forceAll option indicates that all proto files should be
regenerated.
  • Loading branch information
codyoss authored Jun 24, 2021
1 parent 8805c8a commit de654eb
Show file tree
Hide file tree
Showing 6 changed files with 68 additions and 9 deletions.
22 changes: 15 additions & 7 deletions internal/gapicgen/cmd/genbot/bot.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,20 @@ import (
"cloud.google.com/go/internal/gapicgen/git"
)

func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error {
type botConfig struct {
githubAccessToken string
githubUsername string
githubName string
githubEmail string
forceAll bool
}

func genBot(ctx context.Context, c botConfig) error {
for k, v := range map[string]string{
"githubAccessToken": githubAccessToken,
"githubUsername": githubUsername,
"githubName": githubName,
"githubEmail": githubEmail,
"githubAccessToken": c.githubAccessToken,
"githubUsername": c.githubUsername,
"githubName": c.githubName,
"githubEmail": c.githubEmail,
} {
if v == "" {
log.Printf("missing or empty value for required flag --%s\n", k)
Expand All @@ -40,7 +48,7 @@ func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName,
}

// Setup the client and git environment.
githubClient, err := git.NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken)
githubClient, err := git.NewGithubClient(ctx, c.githubUsername, c.githubName, c.githubEmail, c.githubAccessToken)
if err != nil {
return err
}
Expand Down Expand Up @@ -73,7 +81,7 @@ func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName,
return nil
}

return generate(ctx, githubClient)
return generate(ctx, githubClient, c.forceAll)
}

// hasCreatedPRToday checks if the created time of a PR is from today.
Expand Down
3 changes: 2 additions & 1 deletion internal/gapicgen/cmd/genbot/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (

// generate downloads sources and generates pull requests for go-genproto and
// google-cloud-go if needed.
func generate(ctx context.Context, githubClient *git.GithubClient) error {
func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool) error {
log.Println("creating temp dir")
tmpDir, err := ioutil.TempDir("", "update-genproto")
if err != nil {
Expand Down Expand Up @@ -71,6 +71,7 @@ func generate(ctx context.Context, githubClient *git.GithubClient) error {
GenprotoDir: genprotoDir,
GapicDir: gocloudDir,
ProtoDir: protoDir,
ForceAll: forceAll,
}
changes, err := generator.Generate(ctx, conf)
if err != nil {
Expand Down
9 changes: 8 additions & 1 deletion internal/gapicgen/cmd/genbot/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func main() {
githubName := flag.String("githubName", os.Getenv("GITHUB_NAME"), "The name of the author for git commits.")
githubEmail := flag.String("githubEmail", os.Getenv("GITHUB_EMAIL"), "The email address of the author.")
localMode := flag.Bool("local", strToBool(os.Getenv("GENBOT_LOCAL_MODE")), "Enables generating sources locally. This mode will not open any pull requests.")
forceAll := flag.Bool("forceAll", strToBool(os.Getenv("GENBOT_FORCE_ALL")), "Enables regenerating everything regardless of changes in googleapis.")

// flags for local mode
googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
Expand All @@ -67,7 +68,13 @@ func main() {
}
return
}
if err := genBot(ctx, *githubAccessToken, *githubUsername, *githubName, *githubEmail); err != nil {
if err := genBot(ctx, botConfig{
githubAccessToken: *githubAccessToken,
githubUsername: *githubUsername,
githubName: *githubName,
githubEmail: *githubEmail,
forceAll: *forceAll,
}); err != nil {
log.Fatal(err)
}
}
Expand Down
1 change: 1 addition & 0 deletions internal/gapicgen/generator/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1021,6 +1021,7 @@ var microgenGapicConfigs = []*microgenConfig{
gRPCServiceConfigPath: "google/cloud/recommendationengine/v1beta1/recommendationengine_grpc_service_config.json",
apiServiceConfigPath: "google/cloud/recommendationengine/v1beta1/recommendationengine_v1beta1.yaml",
releaseLevel: "beta",
stopGeneration: true,
},
{
inputDirectoryPath: "google/cloud/gkehub/v1beta1",
Expand Down
1 change: 1 addition & 0 deletions internal/gapicgen/generator/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type Config struct {
OnlyGenerateGapic bool
LocalMode bool
RegenOnly bool
ForceAll bool
}

// Generate generates genproto and gapics.
Expand Down
41 changes: 41 additions & 0 deletions internal/gapicgen/generator/genproto.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
Expand Down Expand Up @@ -57,6 +58,7 @@ type GenprotoGenerator struct {
genprotoDir string
googleapisDir string
protoSrcDir string
forceAll bool
}

// NewGenprotoGenerator creates a new GenprotoGenerator.
Expand All @@ -65,6 +67,7 @@ func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
genprotoDir: c.GenprotoDir,
googleapisDir: c.GoogleapisDir,
protoSrcDir: filepath.Join(c.ProtoDir, "/src"),
forceAll: c.ForceAll,
}
}

Expand Down Expand Up @@ -186,6 +189,9 @@ func (g *GenprotoGenerator) protoc(fileNames []string) error {
// getUpdatedPackages parses all of the new commits to find what packages need
// to be regenerated.
func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) {
if g.forceAll {
return g.getAllPackages()
}
files, err := git.UpdateFilesSinceHash(g.googleapisDir, googleapisHash)
if err != nil {
return nil, err
Expand All @@ -205,6 +211,41 @@ func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[strin
return pkgFiles, nil
}

func (g *GenprotoGenerator) getAllPackages() (map[string][]string, error) {
seenFiles := make(map[string]bool)
pkgFiles := make(map[string][]string)
for _, root := range []string{g.googleapisDir} {
walkFn := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") {
return nil
}

switch rel, err := filepath.Rel(root, path); {
case err != nil:
return err
case seenFiles[rel]:
return nil
default:
seenFiles[rel] = true
}

pkg, err := goPkg(path)
if err != nil {
return err
}
pkgFiles[pkg] = append(pkgFiles[pkg], path)
return nil
}
if err := filepath.Walk(root, walkFn); err != nil {
return nil, err
}
}
return pkgFiles, nil
}

// moveAndCleanupGeneratedSrc moves all generated src to their correct locations
// in the repository, because protoc puts it in a folder called `generated/``.
func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {
Expand Down

0 comments on commit de654eb

Please sign in to comment.