From d45e49d36275daa28aa645af661f14bd4645ad8c Mon Sep 17 00:00:00 2001 From: Cody Oss <6331106+codyoss@users.noreply.github.com> Date: Fri, 5 Feb 2021 09:02:03 -0700 Subject: [PATCH] chore: refactor genlocal into genbot (#3674) Previously if you wanted to run genlocal you needed to have a bunch of dependencies installed. Some of which can be a little tricky to install correctly. These dependencies happen to be the same as genbot. For this reason this commit refactors genlocal into genbot so that the genbot in local mode can be run from a docker container that has the exact same deps installed as our nightly regen. --- internal/gapicgen/cmd/genbot/Dockerfile | 5 + internal/gapicgen/cmd/genbot/README.md | 86 ++++++++++++--- internal/gapicgen/cmd/genbot/bot.go | 83 ++++++++++++++ internal/gapicgen/cmd/genbot/generate.go | 10 +- internal/gapicgen/cmd/genbot/github.go | 2 +- internal/gapicgen/cmd/genbot/local.go | 104 ++++++++++++++++++ internal/gapicgen/cmd/genbot/main.go | 132 ++++++----------------- internal/gapicgen/cmd/genlocal/README.md | 34 ------ internal/gapicgen/cmd/genlocal/main.go | 113 ------------------- internal/gapicgen/generator/generator.go | 18 ++-- 10 files changed, 312 insertions(+), 275 deletions(-) create mode 100644 internal/gapicgen/cmd/genbot/bot.go create mode 100644 internal/gapicgen/cmd/genbot/local.go delete mode 100644 internal/gapicgen/cmd/genlocal/README.md delete mode 100644 internal/gapicgen/cmd/genlocal/main.go diff --git a/internal/gapicgen/cmd/genbot/Dockerfile b/internal/gapicgen/cmd/genbot/Dockerfile index dfc8067cea1..6ef6e7a2da1 100644 --- a/internal/gapicgen/cmd/genbot/Dockerfile +++ b/internal/gapicgen/cmd/genbot/Dockerfile @@ -41,6 +41,11 @@ RUN mkdir /root/.ssh && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" RUN echo -e '#!/bin/bash\n\ set -ex\n\ + if [[ ${GENBOT_LOCAL_MODE} = "true" ]]; then\n\ + cd internal/gapicgen\n\ + go run cloud.google.com/go/internal/gapicgen/cmd/genbot\n\ + exit 0\n\ + fi\n\ go mod download \n\ go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ ' >> /run.sh diff --git a/internal/gapicgen/cmd/genbot/README.md b/internal/gapicgen/cmd/genbot/README.md index 0328f86744a..455fd5fb76a 100644 --- a/internal/gapicgen/cmd/genbot/README.md +++ b/internal/gapicgen/cmd/genbot/README.md @@ -3,23 +3,38 @@ genbot is a binary for generating gapics and creating PRs with the results. It is intended to be used as a bot, though it can be run locally too. -## Getting certs - -### Github +## Prerequisites for running locally + +Note that only step one, listed below, is required if you plan to run the code +in docker. + +1. Clone this project: `git clone https://github.com/googleapis/google-cloud-go.git` +1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases) +1. Install [Go](http://golang.org/dl) +1. Add `$GOPATH/bin` to `PATH` +1. Create a [GitHub access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). +1. Install Go tools: + +```bash +go get \ + github.com/golang/protobuf/protoc-gen-go \ + golang.org/x/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic +``` -For Github, you need to generate/supply a Personal Access Token. More -information on how that's done can be found here: -[creating a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). +## Generating code and PRs(bot mode) -## Running locally +### Run genbot locally Note: this may change your `~/.gitconfig`, `~/.gitcookies`, and use up non-trivial amounts of space on your computer. 1. Make sure you are on a non-Windows platform. If you are using windows continue on to the docker instructions. -2. Make sure you have all the tools installed listed in genlocal's README.md -3. Run: +1. Make sure you have all the tools installed listed in genlocal's README.md +1. Run: ```shell cd /path/to/internal/gapicgen @@ -30,7 +45,7 @@ go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ --githubEmail=deklerk@google.com \ ``` -## Run with docker +### Run genbot with docker Note: this can be quite slow (~10m). @@ -41,12 +56,51 @@ Note: this may leave a lot of docker resources laying around. Use cd /path/to/internal/gapicgen/cmd/genbot docker build . -t genbot docker run -t --rm --privileged \ - -v `pwd`/../..:/gapicgen \ - -e GITHUB_ACCESS_TOKEN \ - -e GITHUB_USERNAME \ - -e GITHUB_NAME \ - -e GITHUB_EMAIL \ - genbot + -v `pwd`/../..:/gapicgen \ + -e GITHUB_ACCESS_TOKEN \ + -e GITHUB_USERNAME \ + -e GITHUB_NAME \ + -e GITHUB_EMAIL \ + genbot +``` + +## Generating code (local mode) + +Sometimes you may want to just generate gapic sources to test out +new changes, test the generation of a new library, test new generator tweaks, +run generators against googleapis-private, and various other local tasks. The +local mode in genbot allows you to do just that. + +### Run genbot(local mode) locally + +```shell +cd /path/to/internal/gapicgen +go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ + -local \ + -only-gapics \ + -gocloud-dir=/path/to/google-cloud-go \ + -gapic=cloud.google.com/go/foo/apiv1 +``` + +### Run genbot(local mode) with docker + +```shell +cd /path/to/internal/gapicgen +docker build -t genbot -f cmd/genbot/Dockerfile . +docker run --rm \ + -v `pwd`/../..:/gapicgen \ + -e GENBOT_LOCAL_MODE=true \ + -e ONLY_GAPICS=true \ + -e GOCLOUD_DIR=/gapicgen \ + -e GAPIC_TO_GENERATE=cloud.google.com/go/foo/apiv1 \ + genbot \ +``` + +Note you can optionally mount in your Go module cache if you have Go installed. +This will speed up the build a bit: + +```shell +-v `go env GOMODCACHE`:/root/go/pkg/mod ``` ## FAQ diff --git a/internal/gapicgen/cmd/genbot/bot.go b/internal/gapicgen/cmd/genbot/bot.go new file mode 100644 index 00000000000..e5d8b8cbac7 --- /dev/null +++ b/internal/gapicgen/cmd/genbot/bot.go @@ -0,0 +1,83 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" +) + +func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error { + for k, v := range map[string]string{ + "githubAccessToken": githubAccessToken, + "githubUsername": githubUsername, + "githubName": githubName, + "githubEmail": githubEmail, + } { + if v == "" { + log.Printf("missing or empty value for required flag --%s\n", k) + flag.PrintDefaults() + } + } + + // Setup the client and git environment. + githubClient, err := NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken) + if err != nil { + return err + } + + // Check current regen status. + if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil { + return err + } else if pr != nil { + return fmt.Errorf("There is already a re-generation in progress") + } + if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil { + return err + } else if pr != nil { + if err := updateGocloudPR(ctx, githubClient, pr); err != nil { + return err + } + return nil + } + log.Println("checking if a pull request was already opened and merged today") + if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil { + return err + } else if pr != nil && hasCreatedPRToday(pr.Created) { + log.Println("skipping generation, already created and merged a go-genproto PR today") + return nil + } + if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil { + return err + } else if pr != nil && hasCreatedPRToday(pr.Created) { + log.Println("skipping generation, already created and merged a google-cloud-go PR today") + return nil + } + + return generate(ctx, githubClient) +} + +// hasCreatedPRToday checks if the created time of a PR is from today. +func hasCreatedPRToday(created time.Time) bool { + now := time.Now() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created) + return created.After(today) +} diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go index b4b023ca1b8..154db21f4ca 100644 --- a/internal/gapicgen/cmd/genbot/generate.go +++ b/internal/gapicgen/cmd/genbot/generate.go @@ -53,16 +53,16 @@ func generate(ctx context.Context, githubClient *GithubClient) error { grp, _ := errgroup.WithContext(ctx) grp.Go(func() error { - return gitClone("https://github.com/googleapis/googleapis", googleapisDir) + return gitDeepClone("https://github.com/googleapis/googleapis", googleapisDir) }) grp.Go(func() error { - return gitClone("https://github.com/googleapis/go-genproto", genprotoDir) + return gitDeepClone("https://github.com/googleapis/go-genproto", genprotoDir) }) grp.Go(func() error { - return gitClone("https://github.com/googleapis/google-cloud-go", gocloudDir) + return gitDeepClone("https://github.com/googleapis/google-cloud-go", gocloudDir) }) grp.Go(func() error { - return gitClone("https://github.com/protocolbuffers/protobuf", protoDir) + return gitDeepClone("https://github.com/protocolbuffers/protobuf", protoDir) }) if err := grp.Wait(); err != nil { log.Println(err) @@ -140,7 +140,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error { } // gitClone clones a repository in the given directory. -func gitClone(repo, dir string) error { +func gitDeepClone(repo, dir string) error { log.Printf("cloning %s\n", repo) _, err := git.PlainClone(dir, false, &git.CloneOptions{ diff --git a/internal/gapicgen/cmd/genbot/github.go b/internal/gapicgen/cmd/genbot/github.go index c84e04d1833..24b296d5e6c 100644 --- a/internal/gapicgen/cmd/genbot/github.go +++ b/internal/gapicgen/cmd/genbot/github.go @@ -235,7 +235,7 @@ git push origin $BRANCH_NAME // Can't assign the submitter of the PR as a reviewer. var reviewers []string for _, r := range githubReviewers { - if r != *githubUsername { + if r != gc.Username { reviewers = append(reviewers, r) } } diff --git a/internal/gapicgen/cmd/genbot/local.go b/internal/gapicgen/cmd/genbot/local.go new file mode 100644 index 00000000000..a45b5d6656e --- /dev/null +++ b/internal/gapicgen/cmd/genbot/local.go @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package main + +import ( + "context" + "io/ioutil" + "log" + "os" + "path/filepath" + + "cloud.google.com/go/internal/gapicgen/generator" + "golang.org/x/sync/errgroup" + "gopkg.in/src-d/go-git.v4" +) + +type localConfig struct { + googleapisDir string + gocloudDir string + genprotoDir string + protoDir string + gapicToGenerate string + onlyGapics bool +} + +func genLocal(ctx context.Context, c localConfig) error { + log.Println("creating temp dir") + tmpDir, err := ioutil.TempDir("", "update-genproto") + if err != nil { + log.Fatal(err) + } + log.Printf("temp dir created at %s\n", tmpDir) + tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis") + tmpGenprotoDir := filepath.Join(tmpDir, "genproto") + tmpGocloudDir := filepath.Join(tmpDir, "gocloud") + tmpProtoDir := filepath.Join(tmpDir, "proto") + + // Clone repositories if needed. + grp, _ := errgroup.WithContext(ctx) + gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir) + gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir) + gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir) + gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir) + if err := grp.Wait(); err != nil { + log.Println(err) + } + + // Regen. + conf := &generator.Config{ + GoogleapisDir: deafultDir(tmpGoogleapisDir, c.googleapisDir), + GenprotoDir: deafultDir(tmpGenprotoDir, c.genprotoDir), + GapicDir: deafultDir(tmpGocloudDir, c.gocloudDir), + ProtoDir: deafultDir(tmpProtoDir, c.protoDir), + GapicToGenerate: c.gapicToGenerate, + OnlyGenerateGapic: c.onlyGapics, + LocalMode: true, + } + if _, err := generator.Generate(ctx, conf); err != nil { + log.Printf("Generator ran (and failed) in %s\n", tmpDir) + log.Fatal(err) + } + return nil +} + +// gitShallowClone clones a repository into the provided tmpDir if a dir has not +// been specified. +func gitShallowClone(eg *errgroup.Group, repo, dir, tmpDir string) { + if dir != "" { + return + } + eg.Go(func() error { + log.Printf("cloning %s\n", repo) + + _, err := git.PlainClone(tmpDir, false, &git.CloneOptions{ + URL: repo, + Progress: os.Stdout, + Depth: 1, + Tags: git.NoTags, + }) + return err + }) +} + +// deafultDir returns the default option if dir is not set. +func deafultDir(def, dir string) string { + if dir == "" { + return def + } + return dir +} diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go index bf07d2f4449..aa598b77869 100644 --- a/internal/gapicgen/cmd/genbot/main.go +++ b/internal/gapicgen/cmd/genbot/main.go @@ -21,44 +21,13 @@ package main import ( "context" "flag" - "fmt" "log" "os" - "time" + "strconv" "cloud.google.com/go/internal/gapicgen" ) -var ( - toolsNeeded = []string{"git", "go", "protoc"} - - githubAccessToken = flag.String("githubAccessToken", "", "Get an access token at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") - githubUsername = flag.String("githubUsername", "", "ex -githubUsername=jadekler") - githubName = flag.String("githubName", "", "ex -githubName=\"Jean de Klerk\"") - githubEmail = flag.String("githubEmail", "", "ex -githubEmail=deklerk@google.com") - - usage = func() { - fmt.Fprintln(os.Stderr, `genbot \ - -githubAccessToken=11223344556677889900aabbccddeeff11223344 \ - -githubUsername=jadekler \ - -githubEmail=deklerk@google.com \ - -githubName="Jean de Klerk" \ - --githubAccessToken - The access token to authenticate to github. Get this at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line. - --githubUsername - The username to use in the github commit. - --githubName - The name to use in the github commit. - --githubEmail - The email to use in the github commit.`) - os.Exit(2) - } -) - type prStatus uint8 func (ps prStatus) Has(status prStatus) bool { return ps&status != 0 } @@ -71,83 +40,48 @@ const ( func main() { log.SetFlags(0) - - flag.Usage = usage - flag.Parse() - if *githubAccessToken == "" { - *githubAccessToken = os.Getenv("GITHUB_ACCESS_TOKEN") - } - if *githubUsername == "" { - *githubUsername = os.Getenv("GITHUB_USERNAME") - } - if *githubName == "" { - *githubName = os.Getenv("GITHUB_NAME") - } - if *githubEmail == "" { - *githubEmail = os.Getenv("GITHUB_EMAIL") - } - - for k, v := range map[string]string{ - "githubAccessToken": *githubAccessToken, - "githubUsername": *githubUsername, - "githubName": *githubName, - "githubEmail": *githubEmail, - } { - if v == "" { - log.Printf("missing or empty value for required flag --%s\n", k) - usage() - } + if err := gapicgen.VerifyAllToolsExist([]string{"git", "go", "protoc"}); err != nil { + log.Fatal(err) } - ctx := context.Background() - // Setup the client and git environment. - if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil { - log.Fatal(err) - } - githubClient, err := NewGithubClient(ctx, *githubUsername, *githubName, *githubEmail, *githubAccessToken) - if err != nil { - log.Fatal(err) - } + // General bot flags + githubAccessToken := flag.String("githubAccessToken", os.Getenv("GITHUB_ACCESS_TOKEN"), "The token used to open pull requests.") + githubUsername := flag.String("githubUsername", os.Getenv("GITHUB_USERNAME"), "The GitHub user name for the author.") + githubName := flag.String("githubName", os.Getenv("GITHUB_NAME"), "The name of the author for git commits.") + githubEmail := flag.String("githubEmail", os.Getenv("GITHUB_EMAIL"), "The email address of the author.") + localMode := flag.Bool("local", strToBool(os.Getenv("GENBOT_LOCAL_MODE")), "Enables generating sources locally. This mode will not open any pull requests.") + + // flags for local mode + googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + gapicToGenerate := flag.String("gapic", os.Getenv("GAPIC_TO_GENERATE"), `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`) + onlyGapics := flag.Bool("only-gapics", strToBool(os.Getenv("ONLY_GAPICS")), "Enabling stops regenerating genproto.") - // Check current regen status. - if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil { - log.Fatal(err) - } else if pr != nil { - log.Println("There is already a re-generation in progress") - return - } - if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil { - log.Fatal(err) - } else if pr != nil { - if err := updateGocloudPR(ctx, githubClient, pr); err != nil { + flag.Parse() + + if *localMode { + if err := genLocal(ctx, localConfig{ + googleapisDir: *googleapisDir, + gocloudDir: *gocloudDir, + genprotoDir: *genprotoDir, + protoDir: *protoDir, + gapicToGenerate: *gapicToGenerate, + onlyGapics: *onlyGapics, + }); err != nil { log.Fatal(err) } return } - log.Println("checking if a pull request was already opened and merged today") - if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil { - log.Fatal(err) - } else if pr != nil && hasCreatedPRToday(pr.Created) { - log.Println("skipping generation, already created and merged a go-genproto PR today") - return - } - if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil { - log.Fatal(err) - } else if pr != nil && hasCreatedPRToday(pr.Created) { - log.Println("skipping generation, already created and merged a google-cloud-go PR today") - return - } - - if err := generate(ctx, githubClient); err != nil { + if err := genBot(ctx, *githubAccessToken, *githubUsername, *githubName, *githubEmail); err != nil { log.Fatal(err) } } -// hasCreatedPRToday checks if the created time of a PR is from today. -func hasCreatedPRToday(created time.Time) bool { - now := time.Now() - today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created) - return created.After(today) +func strToBool(s string) bool { + // Treat error as false + b, _ := strconv.ParseBool(s) + return b } diff --git a/internal/gapicgen/cmd/genlocal/README.md b/internal/gapicgen/cmd/genlocal/README.md deleted file mode 100644 index 3532ee437ab..00000000000 --- a/internal/gapicgen/cmd/genlocal/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# genlocal - -genlocal is a binary for generating gapics locally. It may be used to test out -new changes, test the generation of a new library, test new generator tweaks, -run generators against googleapis-private, and various other local tasks. - -## Required tools - -*Note*: If you are on a Windows platform you will need to install these tools -in a linux docker container: [Install docker](https://www.docker.com/get-started) - -1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases) -2. Install [Go](http://golang.org/dl) -3. Install python3, pip3 -4. Install virtualenv `pip3 install virtualenv` -5. Install Go tools: - -```bash -go get \ - github.com/golang/protobuf/protoc-gen-go \ - golang.org/x/lint/golint \ - golang.org/x/tools/cmd/goimports \ - honnef.co/go/tools/cmd/staticcheck \ - github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic -``` - -## Running - -`git clone` this project if you don't already have it checked-out locally. - -```bash -cd /path/to/google-cloud-go/internal/gapicgen -go run cloud.google.com/go/internal/gapicgen/cmd/genlocal -``` diff --git a/internal/gapicgen/cmd/genlocal/main.go b/internal/gapicgen/cmd/genlocal/main.go deleted file mode 100644 index 0f004a0e815..00000000000 --- a/internal/gapicgen/cmd/genlocal/main.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !windows - -// genlocal is a binary for generating gapics locally. It may be used to test out -// new changes, test the generation of a new library, test new generator tweaks, -// run generators against googleapis-private, and various other local tasks. -package main - -import ( - "context" - "flag" - "io/ioutil" - "log" - "os" - "path/filepath" - "strings" - - "cloud.google.com/go/internal/gapicgen" - "cloud.google.com/go/internal/gapicgen/generator" - "golang.org/x/sync/errgroup" - "gopkg.in/src-d/go-git.v4" -) - -var ( - toolsNeeded = []string{"go", "protoc"} -) - -func main() { - log.SetFlags(0) - if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil { - log.Fatal(err) - } - - log.Println("creating temp dir") - tmpDir, err := ioutil.TempDir("", "update-genproto") - if err != nil { - log.Fatal(err) - } - log.Printf("temp dir created at %s\n", tmpDir) - - googleapisDir := flag.String("googleapis-dir", filepath.Join(tmpDir, "googleapis"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - gocloudDir := flag.String("gocloud-dir", filepath.Join(tmpDir, "gocloud"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - genprotoDir := flag.String("genproto-dir", filepath.Join(tmpDir, "genproto"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - protoDir := flag.String("proto-dir", filepath.Join(tmpDir, "proto"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - gapicToGenerate := flag.String("gapic", "", `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`) - onlyGapics := flag.Bool("only-gapics", false, "Enabling stops regenerating genproto.") - verbose := flag.Bool("verbose", false, "Enables verbose logging.") - flag.Parse() - - ctx := context.Background() - - // Clone repositories if needed. - grp, _ := errgroup.WithContext(ctx) - gitClone(grp, "https://github.com/googleapis/googleapis.git", *googleapisDir, tmpDir) - gitClone(grp, "https://github.com/googleapis/go-genproto", *genprotoDir, tmpDir) - gitClone(grp, "https://github.com/googleapis/google-cloud-go", *gocloudDir, tmpDir) - gitClone(grp, "https://github.com/protocolbuffers/protobuf", *protoDir, tmpDir) - if err := grp.Wait(); err != nil { - log.Println(err) - } - - // Regen. - conf := &generator.Config{ - GoogleapisDir: *googleapisDir, - GenprotoDir: *genprotoDir, - GapicDir: *gocloudDir, - ProtoDir: *protoDir, - GapicToGenerate: *gapicToGenerate, - OnlyGenerateGapic: *onlyGapics, - } - changes, err := generator.Generate(ctx, conf) - if err != nil { - log.Printf("Generator ran (and failed) in %s\n", tmpDir) - log.Fatal(err) - } - - // Log results. - log.Println(genprotoDir) - log.Println(gocloudDir) - - if *verbose { - log.Println(generator.FormatChanges(changes, false)) - } -} - -// gitClone clones a repository in the given directory if dir is not in tmpDir. -func gitClone(eg *errgroup.Group, repo, dir, tmpDir string) { - if !strings.HasPrefix(dir, tmpDir) { - return - } - eg.Go(func() error { - log.Printf("cloning %s\n", repo) - - _, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: repo, - Progress: os.Stdout, - }) - return err - }) -} diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index f5b7f9707a2..c3831e4b030 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -36,6 +36,7 @@ type Config struct { ProtoDir string GapicToGenerate string OnlyGenerateGapic bool + LocalMode bool } // Generate generates genproto and gapics. @@ -51,13 +52,16 @@ func Generate(ctx context.Context, conf *Config) ([]*ChangeInfo, error) { return nil, fmt.Errorf("error generating gapics (may need to check logs for more errors): %v", err) } - changes, err := gatherChanges(conf.GoogleapisDir, conf.GenprotoDir) - if err != nil { - return nil, fmt.Errorf("error gathering commit info") - } - - if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil { - return nil, err + var changes []*ChangeInfo + if !conf.LocalMode { + var err error + changes, err = gatherChanges(conf.GoogleapisDir, conf.GenprotoDir) + if err != nil { + return nil, fmt.Errorf("error gathering commit info") + } + if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil { + return nil, err + } } return changes, nil