diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index 328de513c4d5..19dcb58cf357 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -53,7 +53,7 @@ deep-remove-regex: - /binaryauthorization/apiv1beta1/ - /certificatemanager/apiv1/ - /channel/apiv1/ - # - /cloudbuild/apiv1/v2/ + - /cloudbuild/apiv1/v2/ - /cloudbuild/apiv2/ - /clouddms/apiv1/ - /cloudtasks/apiv2/ @@ -78,6 +78,7 @@ deep-remove-regex: - /datastore/admin/apiv1/ - /datastream/apiv1/ - /datastream/apiv1alpha1/ + - /debugger/apiv2/ - /deploy/apiv1/ - /dialogflow/apiv2/ - /dialogflow/apiv2beta1/ @@ -89,6 +90,7 @@ deep-remove-regex: - /documentai/apiv1beta3/ - /domains/apiv1beta1/ - /edgecontainer/apiv1/ + - /errorreporting/apiv1beta1/ - /essentialcontacts/apiv1/ - /eventarc/apiv1/ - /eventarc/publishing/apiv1/ @@ -128,6 +130,7 @@ deep-remove-regex: - /metastore/apiv1/ - /metastore/apiv1alpha/ - /metastore/apiv1beta/ + - /monitoring/apiv3/v2/ - /monitoring/dashboard/apiv1/ - /monitoring/metricsscope/apiv1/ - /networkconnectivity/apiv1/ @@ -153,7 +156,8 @@ deep-remove-regex: - /privatecatalog/apiv1beta1/ - /pubsub/apiv1/ - /pubsublite/apiv1/ - # - /recaptchaenterprise/v2/apiv1beta1/ + - /recaptchaenterprise/v2/apiv1/ + - /recaptchaenterprise/v2/apiv1beta1/ - /recommendationengine/apiv1beta1/ - /recommender/apiv1/ - /recommender/apiv1beta1/ @@ -187,6 +191,7 @@ deep-remove-regex: - /speech/apiv1/ - /speech/apiv1p1beta1/ - /speech/apiv2/ + - /storage/internal/apiv2/ - /storagetransfer/apiv1/ - /talent/apiv4/ - /talent/apiv4beta1/ @@ -194,12 +199,15 @@ deep-remove-regex: - /tpu/apiv1/ - /trace/apiv1/ - /trace/apiv2/ + - /translate/apiv3/ - /video/livestream/apiv1/ # - /video/stitcher/apiv1/ - /video/transcoder/apiv1/ - /videointelligence/apiv1/ - /videointelligence/apiv1beta2/ - /videointelligence/apiv1p3beta1/ + - /vision/v2/apiv1/ + - /vision/v2/apiv1p1beta1/ - /vmmigration/apiv1/ - /vmwareengine/apiv1/ - /vpcaccess/apiv1/ @@ -354,8 +362,8 @@ deep-copy-regex: dest: /certificatemanager/apiv1 - source: /google/cloud/channel/v1/cloud.google.com/go/channel/apiv1 dest: /channel/apiv1 - # - source: /google/devtools/cloudbuild/v1/cloud.google.com/go/cloudbuild/apiv1/v2 - # dest: /cloudbuild/apiv1/v2 + - source: /google/devtools/cloudbuild/v1/cloud.google.com/go/cloudbuild/apiv1/v2 + dest: /cloudbuild/apiv1/v2 - source: /google/devtools/cloudbuild/v2/cloud.google.com/go/cloudbuild/apiv2 dest: /cloudbuild/apiv2 - source: /google/cloud/clouddms/v1/cloud.google.com/go/clouddms/apiv1 @@ -404,6 +412,8 @@ deep-copy-regex: dest: /datastream/apiv1 - source: /google/cloud/datastream/v1alpha1/cloud.google.com/go/datastream/apiv1alpha1 dest: /datastream/apiv1alpha1 + - source: /google/devtools/clouddebugger/v2/cloud.google.com/go/debugger/apiv2 + dest: /debugger/apiv2 - source: /google/cloud/deploy/v1/cloud.google.com/go/deploy/apiv1 dest: /deploy/apiv1 - source: /google/cloud/dialogflow/v2/cloud.google.com/go/dialogflow/apiv2 @@ -426,6 +436,8 @@ deep-copy-regex: dest: /domains/apiv1beta1 - source: /google/cloud/edgecontainer/v1/cloud.google.com/go/edgecontainer/apiv1 dest: /edgecontainer/apiv1 + - source: /google/devtools/clouderrorreporting/v1beta1/cloud.google.com/go/errorreporting/apiv1beta1 + dest: /errorreporting/apiv1beta1 - source: /google/cloud/essentialcontacts/v1/cloud.google.com/go/essentialcontacts/apiv1 dest: /essentialcontacts/apiv1 - source: /google/cloud/eventarc/v1/cloud.google.com/go/eventarc/apiv1 @@ -504,6 +516,8 @@ deep-copy-regex: dest: /metastore/apiv1alpha - source: /google/cloud/metastore/v1beta/cloud.google.com/go/metastore/apiv1beta dest: /metastore/apiv1beta + - source: /google/monitoring/v3/cloud.google.com/go/monitoring/apiv3/v2 + dest: /monitoring/apiv3/v2 - source: /google/monitoring/dashboard/v1/cloud.google.com/go/monitoring/dashboard/apiv1 dest: /monitoring/dashboard/apiv1 - source: /google/monitoring/metricsscope/v1/cloud.google.com/go/monitoring/metricsscope/apiv1 @@ -554,8 +568,10 @@ deep-copy-regex: dest: /pubsub/apiv1 - source: /google/cloud/pubsublite/v1/cloud.google.com/go/pubsublite/apiv1 dest: /pubsublite/apiv1 - # - source: /google/cloud/recaptchaenterprise/v1beta1/cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1 - # dest: /recaptchaenterprise/v2/apiv1beta1 + - source: /google/cloud/recaptchaenterprise/v1/cloud.google.com/go/recaptchaenterprise/v2/apiv1 + dest: /recaptchaenterprise/v2/apiv1 + - source: /google/cloud/recaptchaenterprise/v1beta1/cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1 + dest: /recaptchaenterprise/v2/apiv1beta1 - source: /google/cloud/recommendationengine/v1beta1/cloud.google.com/go/recommendationengine/apiv1beta1 dest: /recommendationengine/apiv1beta1 - source: /google/cloud/recommender/v1/cloud.google.com/go/recommender/apiv1 @@ -622,6 +638,8 @@ deep-copy-regex: dest: /speech/apiv1p1beta1 - source: /google/cloud/speech/v2/cloud.google.com/go/speech/apiv2 dest: /speech/apiv2 + - source: /google/storage/v2/cloud.google.com/go/storage/internal/apiv2 + dest: /storage/internal/apiv2 - source: /google/storagetransfer/v1/cloud.google.com/go/storagetransfer/apiv1 dest: /storagetransfer/apiv1 - source: /google/cloud/talent/v4/cloud.google.com/go/talent/apiv4 @@ -636,6 +654,8 @@ deep-copy-regex: dest: /trace/apiv1 - source: /google/devtools/cloudtrace/v2/cloud.google.com/go/trace/apiv2 dest: /trace/apiv2 + - source: /google/cloud/translate/v3/cloud.google.com/go/translate/apiv3 + dest: /translate/apiv3 - source: /google/cloud/video/livestream/v1/cloud.google.com/go/video/livestream/apiv1 dest: /video/livestream/apiv1 # - source: /google/cloud/video/stitcher/v1/cloud.google.com/go/video/stitcher/apiv1 @@ -648,6 +668,10 @@ deep-copy-regex: dest: /videointelligence/apiv1beta2 - source: /google/cloud/videointelligence/v1p3beta1/cloud.google.com/go/videointelligence/apiv1p3beta1 dest: /videointelligence/apiv1p3beta1 + - source: /google/cloud/vision/v1/cloud.google.com/go/vision/v2/apiv1 + dest: /vision/v2/apiv1 + - source: /google/cloud/vision/v1p1beta1/cloud.google.com/go/vision/v2/apiv1p1beta1 + dest: /vision/v2/apiv1p1beta1 - source: /google/cloud/vmmigration/v1/cloud.google.com/go/vmmigration/apiv1 dest: /vmmigration/apiv1 - source: /google/cloud/vmwareengine/v1/cloud.google.com/go/vmwareengine/apiv1 diff --git a/internal/gapicgen/cmd/genbot/bot.go b/internal/gapicgen/cmd/genbot/bot.go index aa854a2dc955..6ea1ed1eae9d 100644 --- a/internal/gapicgen/cmd/genbot/bot.go +++ b/internal/gapicgen/cmd/genbot/bot.go @@ -60,14 +60,7 @@ func genBot(ctx context.Context, c botConfig) error { log.Println("there is already a re-generation in progress") return nil } - if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil { - return err - } else if pr != nil { - if err := updateGocloudPR(ctx, githubClient, pr); err != nil { - return err - } - return nil - } + log.Println("checking if a pull request was already opened and merged today") if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil { return err @@ -75,13 +68,6 @@ func genBot(ctx context.Context, c botConfig) error { log.Println("skipping generation, already created and merged a go-genproto PR today") return nil } - if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil { - return err - } else if pr != nil && hasCreatedPRToday(pr.Created) { - log.Println("skipping generation, already created and merged a google-cloud-go PR today") - return nil - } - return generate(ctx, githubClient, c.forceAll) } diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go index 2e3414f60bd4..96c1d5aa1ff2 100644 --- a/internal/gapicgen/cmd/genbot/generate.go +++ b/internal/gapicgen/cmd/genbot/generate.go @@ -42,12 +42,10 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool log.Printf("working out %s\n", tmpDir) googleapisDir := filepath.Join(tmpDir, "googleapis") - gocloudDir := filepath.Join(tmpDir, "gocloud") genprotoDir := filepath.Join(tmpDir, "genproto") protoDir := filepath.Join(tmpDir, "proto") // Clone repositories. - grp, _ := errgroup.WithContext(ctx) grp.Go(func() error { return git.DeepClone("https://github.com/googleapis/googleapis", googleapisDir) @@ -55,9 +53,7 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool grp.Go(func() error { return git.DeepClone("https://github.com/googleapis/go-genproto", genprotoDir) }) - grp.Go(func() error { - return git.DeepClone("https://github.com/googleapis/google-cloud-go", gocloudDir) - }) + grp.Go(func() error { return git.DeepClone("https://github.com/protocolbuffers/protobuf", protoDir) }) @@ -69,7 +65,6 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool conf := &generator.Config{ GoogleapisDir: googleapisDir, GenprotoDir: genprotoDir, - GapicDir: gocloudDir, ProtoDir: protoDir, ForceAll: forceAll, } @@ -78,61 +73,20 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool return err } - // Create PRs. + // Create PR genprotoHasChanges, err := git.HasChanges(genprotoDir) if err != nil { return err } - gocloudHasChanges, err := git.HasChanges(gocloudDir) - if err != nil { - return err + if !genprotoHasChanges { + log.Println("no changes detected") + return nil } - - switch { - case genprotoHasChanges && gocloudHasChanges: - // Both have changes. - genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, true, changes) - if err != nil { - return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err) - } - - gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, genprotoPRNum, changes) - if err != nil { - return fmt.Errorf("error creating CL for veneers (may need to check logs for more errors): %v", err) - } - - if err := githubClient.AmendGenprotoPR(ctx, genprotoPRNum, genprotoDir, gocloudPRNum, changes); err != nil { - return fmt.Errorf("error amending genproto PR: %v", err) - } - - genprotoPRURL := fmt.Sprintf("https://github.com/googleapis/go-genproto/pull/%d", genprotoPRNum) - gocloudPRURL := fmt.Sprintf("https://github.com/googleapis/google-cloud-go/pull/%d", genprotoPRNum) - log.Println(genprotoPRURL) - log.Println(gocloudPRURL) - case genprotoHasChanges: - // Only genproto has changes. - genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, false, changes) - if err != nil { - return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err) - } - - genprotoPRURL := fmt.Sprintf("https://github.com/googleapis/go-genproto/pull/%d", genprotoPRNum) - log.Println(genprotoPRURL) - log.Println("gocloud had no changes") - case gocloudHasChanges: - // Only gocloud has changes. - gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, -1, changes) - if err != nil { - return fmt.Errorf("error creating CL for veneers (may need to check logs for more errors): %v", err) - } - - gocloudPRURL := fmt.Sprintf("https://github.com/googleapis/google-cloud-go/pull/%d", gocloudPRNum) - log.Println("genproto had no changes") - log.Println(gocloudPRURL) - default: - // Neither have changes. - log.Println("Neither genproto nor gocloud had changes") + genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, false, changes) + if err != nil { + return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err) } + log.Printf("https://github.com/googleapis/go-genproto/pull/%d", genprotoPRNum) return nil } diff --git a/internal/gapicgen/cmd/genbot/local.go b/internal/gapicgen/cmd/genbot/local.go index f54f400389cd..e05fa4ae1ec8 100644 --- a/internal/gapicgen/cmd/genbot/local.go +++ b/internal/gapicgen/cmd/genbot/local.go @@ -30,7 +30,6 @@ import ( type localConfig struct { googleapisDir string - gocloudDir string genprotoDir string protoDir string gapicToGenerate string @@ -49,14 +48,12 @@ func genLocal(ctx context.Context, c localConfig) error { log.Printf("temp dir created at %s\n", tmpDir) tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis") tmpGenprotoDir := filepath.Join(tmpDir, "genproto") - tmpGocloudDir := filepath.Join(tmpDir, "gocloud") tmpProtoDir := filepath.Join(tmpDir, "proto") // Clone repositories if needed. grp, _ := errgroup.WithContext(ctx) gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir) gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir) - gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir) gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir) if err := grp.Wait(); err != nil { log.Println(err) @@ -66,7 +63,6 @@ func genLocal(ctx context.Context, c localConfig) error { conf := &generator.Config{ GoogleapisDir: defaultDir(tmpGoogleapisDir, c.googleapisDir), GenprotoDir: defaultDir(tmpGenprotoDir, c.genprotoDir), - GapicDir: defaultDir(tmpGocloudDir, c.gocloudDir), ProtoDir: defaultDir(tmpProtoDir, c.protoDir), GapicToGenerate: c.gapicToGenerate, OnlyGenerateGapic: c.onlyGapics, diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go index c2df09ad808c..4dfbf27ab880 100644 --- a/internal/gapicgen/cmd/genbot/main.go +++ b/internal/gapicgen/cmd/genbot/main.go @@ -46,7 +46,6 @@ func main() { // flags for local mode googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") gapicToGenerate := flag.String("gapic", os.Getenv("GAPIC_TO_GENERATE"), `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`) @@ -59,7 +58,6 @@ func main() { if *localMode { if err := genLocal(ctx, localConfig{ googleapisDir: *googleapisDir, - gocloudDir: *gocloudDir, genprotoDir: *genprotoDir, protoDir: *protoDir, gapicToGenerate: *gapicToGenerate, diff --git a/internal/gapicgen/cmd/genbot/update.go b/internal/gapicgen/cmd/genbot/update.go deleted file mode 100644 index 47f99c424266..000000000000 --- a/internal/gapicgen/cmd/genbot/update.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !windows -// +build !windows - -package main - -import ( - "context" - "fmt" - "log" - - "cloud.google.com/go/internal/gapicgen/git" -) - -func updateGocloudPR(ctx context.Context, githubClient *git.GithubClient, pr *git.PullRequest) error { - if pr.Author != githubClient.Username { - return fmt.Errorf("pull request author %q does not match authenticated user %q", pr.Author, githubClient.Username) - } - - // Checkout PR and update go.mod - if err := githubClient.UpdateGocloudGoMod(); err != nil { - return err - } - - if pr.IsDraft { - if err := githubClient.MarkPRReadyForReview(ctx, pr.Repo, pr.NodeID); err != nil { - return fmt.Errorf("unable to mark PR %v ready for review: %v", pr.Number, err) - } - } - - // Done! - log.Printf("done updating google-cloud-go PR: %s\n", pr.URL) - return nil -} diff --git a/internal/gapicgen/generator/config.go b/internal/gapicgen/generator/config.go deleted file mode 100644 index 2fbc4ef761c1..000000000000 --- a/internal/gapicgen/generator/config.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generator - -import ( - "strings" - - "cloud.google.com/go/internal/aliasfix" -) - -// MicrogenConfig represents a single microgen target. -type MicrogenConfig struct { - // InputDirectoryPath is the path to the input (.proto, etc) files, relative - // to googleapisDir. - InputDirectoryPath string - - // ImportPath is the path that this library should be imported as. - ImportPath string - - // Pkg is the name that should be used in the package declaration. - Pkg string - - // GRPCServiceConfigPath is the path to the grpc service config for this - // target, relative to googleapisDir/inputDirectoryPath. - GRPCServiceConfigPath string - - // ApiServiceConfigPath is the path to the gapic service config for this - // target, relative to googleapisDir/inputDirectoryPath. - ApiServiceConfigPath string - - // ReleaseLevel is the release level of this target. Values incl ga, - // beta, alpha. - ReleaseLevel string - - // stopGeneration is used to stop generating a given client. This might be - // useful if a client needs to be deprecated, but retained in the repo - // metadata. - stopGeneration bool - - // DisableMetadata is used to toggle generation of the gapic_metadata.json - // file for the client library. - DisableMetadata bool - - // Transports is a list of Transports to generate a client for. Acceptable - // values are 'grpc' and 'rest'. Default is ["grpc", "rest"]. - Transports []string - - // stubsDir indicates that the protobuf/gRPC stubs should be generated - // in the GAPIC module by replacing the go_package option with the value of - // ImportPath plus the specified suffix separated by a "/", and using the - // same Pkg value. - stubsDir string - - // NumericEnumsDisabled indicates, for REST GAPICs, if requests should *not* - // be generated to send the $alt=json;enum-encoding=int system parameter - // with every API call. This should only be disabled for services that are - // *not* up-to-date enough to support such a system parameter. - NumericEnumsDisabled bool -} - -// genprotoImportPath returns the genproto import path for a given config. -func (m *MicrogenConfig) genprotoImportPath() string { - return "google.golang.org/genproto/googleapis/" + strings.TrimPrefix(m.InputDirectoryPath, "google/") -} - -// getStubsDir gets the stubs dir specified in config or returns the -// directory path if the config is either in progress or completed a migration. -func (m *MicrogenConfig) getStubsDir() string { - if m.stubsDir != "" { - return m.stubsDir - } - if pkg, ok := aliasfix.GenprotoPkgMigration[m.genprotoImportPath()]; ok && pkg.Status != aliasfix.StatusNotMigrated { - ss := strings.Split(pkg.ImportPath, "/") - return ss[len(ss)-1] - } - return "" -} - -// isMigrated is a convenience wrapper for calling the function of the same -// name. -func (m *MicrogenConfig) isMigrated() bool { - return isMigrated(m.genprotoImportPath()) -} - -// StopGeneration is used to stop generating a given client. This might be -// useful if a client needs to be deprecated, but retained in the repo -// metadata. -func (m *MicrogenConfig) StopGeneration() bool { - return m.stopGeneration -} - -// isMigrated returns true if the specified genproto import path has been -// migrated. -func isMigrated(importPath string) bool { - if pkg, ok := aliasfix.GenprotoPkgMigration[importPath]; ok && pkg.Status == aliasfix.StatusMigrated { - return true - } - return false -} - -var MicrogenGapicConfigs = []*MicrogenConfig{ - { - InputDirectoryPath: "google/devtools/cloudbuild/v1", - Pkg: "cloudbuild", - ImportPath: "cloud.google.com/go/cloudbuild/apiv1/v2", - GRPCServiceConfigPath: "cloudbuild_grpc_service_config.json", - ApiServiceConfigPath: "cloudbuild_v1.yaml", - ReleaseLevel: "ga", - }, - { - InputDirectoryPath: "google/devtools/clouddebugger/v2", - Pkg: "debugger", - ImportPath: "cloud.google.com/go/debugger/apiv2", - GRPCServiceConfigPath: "clouddebugger_grpc_service_config.json", - ApiServiceConfigPath: "clouddebugger_v2.yaml", - ReleaseLevel: "ga", - }, - { - InputDirectoryPath: "google/devtools/clouderrorreporting/v1beta1", - Pkg: "errorreporting", - ImportPath: "cloud.google.com/go/errorreporting/apiv1beta1", - GRPCServiceConfigPath: "errorreporting_grpc_service_config.json", - ApiServiceConfigPath: "clouderrorreporting_v1beta1.yaml", - ReleaseLevel: "beta", - }, - { - InputDirectoryPath: "google/monitoring/v3", - Pkg: "monitoring", - ImportPath: "cloud.google.com/go/monitoring/apiv3/v2", - GRPCServiceConfigPath: "monitoring_grpc_service_config.json", - ApiServiceConfigPath: "monitoring.yaml", - ReleaseLevel: "ga", - NumericEnumsDisabled: true, - Transports: []string{"grpc"}, - }, - { - InputDirectoryPath: "google/cloud/recaptchaenterprise/v1", - Pkg: "recaptchaenterprise", - ImportPath: "cloud.google.com/go/recaptchaenterprise/v2/apiv1", - GRPCServiceConfigPath: "recaptchaenterprise_grpc_service_config.json", - ApiServiceConfigPath: "recaptchaenterprise_v1.yaml", - ReleaseLevel: "ga", - NumericEnumsDisabled: false, - Transports: []string{"grpc"}, - }, - { - InputDirectoryPath: "google/cloud/recaptchaenterprise/v1beta1", - Pkg: "recaptchaenterprise", - ImportPath: "cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1", - GRPCServiceConfigPath: "recaptchaenterprise_grpc_service_config.json", - ApiServiceConfigPath: "recaptchaenterprise_v1beta1.yaml", - ReleaseLevel: "beta", - }, - { - InputDirectoryPath: "google/cloud/translate/v3", - Pkg: "translate", - ImportPath: "cloud.google.com/go/translate/apiv3", - GRPCServiceConfigPath: "translate_grpc_service_config.json", - ApiServiceConfigPath: "translate_v3.yaml", - ReleaseLevel: "ga", - }, - { - InputDirectoryPath: "google/cloud/vision/v1", - Pkg: "vision", - ImportPath: "cloud.google.com/go/vision/v2/apiv1", - GRPCServiceConfigPath: "vision_grpc_service_config.json", - ApiServiceConfigPath: "vision_v1.yaml", - ReleaseLevel: "ga", - }, - { - InputDirectoryPath: "google/cloud/vision/v1p1beta1", - Pkg: "vision", - ImportPath: "cloud.google.com/go/vision/v2/apiv1p1beta1", - GRPCServiceConfigPath: "vision_grpc_service_config.json", - ApiServiceConfigPath: "vision_v1p1beta1.yaml", - ReleaseLevel: "beta", - }, -} diff --git a/internal/gapicgen/generator/config_test.go b/internal/gapicgen/generator/config_test.go deleted file mode 100644 index 09abbb0e2089..000000000000 --- a/internal/gapicgen/generator/config_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generator - -import ( - "strings" - "testing" -) - -var allowedReleaseLevels = map[string]bool{ - "alpha": true, - "beta": true, - "ga": true, -} - -var apivExceptions = map[string]bool{ - "cloud.google.com/go/longrunning/autogen": true, - "cloud.google.com/go/firestore/apiv1/admin": true, - "cloud.google.com/go/cloudbuild/apiv1/v2": true, - "cloud.google.com/go/monitoring/apiv3/v2": true, -} - -var packagePathExceptions = map[string]bool{ - "cloud.google.com/go/longrunning/autogen": true, - "cloud.google.com/go/firestore/apiv1/admin": true, - "cloud.google.com/go/recaptchaenterprise/v2/apiv1": true, - "cloud.google.com/go/storage/internal/apiv2": true, - "cloud.google.com/go/vision/v2/apiv1": true, - "cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1": true, - "cloud.google.com/go/vision/v2/apiv1p1beta1": true, -} - -// TestMicrogenConfigs validates config entries. -// We expect entries here to have reasonable production settings, whereas -// the low level tooling is more permissive in the face of ambiguity. -// -// TODO: we should be able to do more substantial validation of config entries -// given sufficient setup. Consider fetching https://github.com/googleapis/googleapis -// to ensure referenced entries are present. -func TestMicrogenConfigs(t *testing.T) { - for k, entry := range MicrogenGapicConfigs { - if entry.ImportPath == "" { - t.Errorf("config %q (#%d) expected non-empty importPath", entry.InputDirectoryPath, k) - } - importParts := strings.Split(entry.ImportPath, "/") - v := importParts[len(importParts)-1] - if !strings.HasPrefix(v, "apiv") && !apivExceptions[entry.ImportPath] { - t.Errorf("config %q (#%d) expected the last part of import path %q to start with \"apiv\"", entry.InputDirectoryPath, k, entry.ImportPath) - } - if want := entry.Pkg + "/apiv"; !strings.Contains(entry.ImportPath, want) && !packagePathExceptions[entry.ImportPath] { - t.Errorf("config %q (#%d) want import path to contain %q", entry.ImportPath, k, want) - } - if entry.InputDirectoryPath == "" { - t.Errorf("config %q (#%d) expected non-empty inputDirectoryPath field", entry.ImportPath, k) - } - if entry.Pkg == "" { - t.Errorf("config %q (#%d) expected non-empty pkg field", entry.ImportPath, k) - } - if entry.ApiServiceConfigPath == "" { - t.Errorf("config %q (#%d) expected non-empty apiServiceConfigPath", entry.ImportPath, k) - } - if p := entry.ApiServiceConfigPath; p != "" && strings.HasSuffix(p, "gapic.yaml") { - t.Errorf("config %q (#%d) should not use GAPIC yaml for apiServiceConfigPath", entry.ImportPath, k) - } - // Internally, an empty release level means "ga" to the underlying tool, but we - // want to be explicit in this configuration. - if entry.ReleaseLevel == "" { - t.Errorf("config %q (#%d) expected non-empty releaseLevel field", entry.ImportPath, k) - } else if !allowedReleaseLevels[entry.ReleaseLevel] { - t.Errorf("config %q (#%d) invalid releaseLevel: %q", entry.ImportPath, k, entry.ReleaseLevel) - } - } -} diff --git a/internal/gapicgen/generator/gapics.go b/internal/gapicgen/generator/gapics.go deleted file mode 100644 index 4d5c4c848102..000000000000 --- a/internal/gapicgen/generator/gapics.go +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generator - -import ( - "context" - _ "embed" - "fmt" - "log" - "os" - "path/filepath" - "strings" - - "cloud.google.com/go/internal/gapicgen/execv" - "cloud.google.com/go/internal/gapicgen/execv/gocmd" -) - -// GapicGenerator is used to regenerate gapic libraries. -type GapicGenerator struct { - googleapisDir string - protoDir string - googleCloudDir string - genprotoDir string - gapicToGenerate string - regenOnly bool - onlyGenerateGapic bool - modifiedPkgs []string - forceAll bool - genAlias bool -} - -// NewGapicGenerator creates a GapicGenerator. -func NewGapicGenerator(c *Config, modifiedPkgs []string) *GapicGenerator { - return &GapicGenerator{ - googleapisDir: c.GoogleapisDir, - protoDir: c.ProtoDir, - googleCloudDir: c.GapicDir, - genprotoDir: c.GenprotoDir, - gapicToGenerate: c.GapicToGenerate, - regenOnly: c.RegenOnly, - onlyGenerateGapic: c.OnlyGenerateGapic, - modifiedPkgs: modifiedPkgs, - forceAll: c.ForceAll, - genAlias: c.GenAlias, - } -} - -type modInfo struct { - path string - importPath string - serviceImportPath string -} - -// Regen generates gapics. -func (g *GapicGenerator) Regen(ctx context.Context) error { - log.Println("regenerating gapics") - for _, c := range MicrogenGapicConfigs { - if !g.shouldGenerateConfig(c) { - continue - } - if err := g.microgen(c); err != nil { - return err - } - } - - if err := g.copyMicrogenFiles(); err != nil { - return err - } - - // Get rid of diffs related to bad formatting. - if err := gocmd.Vet(g.googleCloudDir); err != nil { - return err - } - - if g.regenOnly { - return nil - } - - if !g.onlyGenerateGapic { - if err := execv.ForEachMod(g.googleCloudDir, g.addModReplaceGenproto); err != nil { - return err - } - } - - if err := gocmd.Vet(g.googleCloudDir); err != nil { - return err - } - - if err := gocmd.Build(g.googleCloudDir); err != nil { - return err - } - - if !g.onlyGenerateGapic { - if err := execv.ForEachMod(g.googleCloudDir, g.dropModReplaceGenproto); err != nil { - return err - } - } - - return nil -} - -func (g *GapicGenerator) shouldGenerateConfig(c *MicrogenConfig) bool { - if g.forceAll && !c.StopGeneration() { - return true - } - - // Skip generation if generating all of the gapics and the associated - // config has a block on it. Or if generating a single gapic and it does - // not match the specified import path. - if (c.StopGeneration() && g.gapicToGenerate == "") || - (g.gapicToGenerate != "" && !strings.Contains(g.gapicToGenerate, c.ImportPath)) || - (g.forceAll && !c.StopGeneration()) { - return false - } - return true -} - -// addModReplaceGenproto adds a genproto replace statement that points genproto -// to the local copy. This is necessary since the remote genproto may not have -// changes that are necessary for the in-flight regen. -func (g *GapicGenerator) addModReplaceGenproto(dir string) error { - log.Printf("[%s] adding temporary genproto replace statement", dir) - c := execv.Command("bash", "-c", ` -set -ex - -go mod edit -replace "google.golang.org/genproto=$GENPROTO_DIR" -go mod tidy -`) - c.Dir = dir - c.Env = []string{ - "GENPROTO_DIR=" + g.genprotoDir, - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - } - return c.Run() -} - -// dropModReplaceGenproto drops the genproto replace statement. It is intended -// to be run after addModReplaceGenproto. -func (g *GapicGenerator) dropModReplaceGenproto(dir string) error { - log.Printf("[%s] removing genproto replace statement", dir) - c := execv.Command("bash", "-c", ` -set -ex - -git restore go.mod -git restore go.sum -`) - c.Dir = dir - c.Env = []string{ - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - } - return c.Run() -} - -// microgen runs the microgenerator on a single microgen config. -func (g *GapicGenerator) microgen(conf *MicrogenConfig) error { - log.Println("microgen generating", conf.Pkg) - - var protoFiles []string - inputDir := filepath.Join(g.googleapisDir, conf.InputDirectoryPath) - entries, err := os.ReadDir(inputDir) - if err != nil { - return err - } - for _, entry := range entries { - // Ignore compute_small.proto which is just for testing and would cause a collision if used in generation. - // - // TODO(noahdietz): Remove this when it is no longer needed. - if strings.Contains(entry.Name(), ".proto") && !strings.Contains(entry.Name(), "compute_small.proto") { - protoFiles = append(protoFiles, filepath.Join(inputDir, entry.Name())) - } - } - - args := []string{"-I", g.googleapisDir, - "--experimental_allow_proto3_optional", - "-I", g.protoDir, - "--go_gapic_out", g.googleCloudDir, - // TODO(chrisdsmith): Enable snippets by deleting the next line after removing internal/gapicgen/gensnippets. - "--go_gapic_opt", "omit-snippets", - "--go_gapic_opt", fmt.Sprintf("go-gapic-package=%s;%s", conf.ImportPath, conf.Pkg), - "--go_gapic_opt", fmt.Sprintf("api-service-config=%s", filepath.Join(conf.InputDirectoryPath, conf.ApiServiceConfigPath))} - - if conf.ReleaseLevel != "" { - args = append(args, "--go_gapic_opt", fmt.Sprintf("release-level=%s", conf.ReleaseLevel)) - } - if conf.GRPCServiceConfigPath != "" { - args = append(args, "--go_gapic_opt", fmt.Sprintf("grpc-service-config=%s", filepath.Join(conf.InputDirectoryPath, conf.GRPCServiceConfigPath))) - } - if !conf.DisableMetadata { - args = append(args, "--go_gapic_opt", "metadata") - } - if len(conf.Transports) == 0 { - conf.Transports = []string{"grpc", "rest"} - } - if len(conf.Transports) > 0 { - args = append(args, "--go_gapic_opt", fmt.Sprintf("transport=%s", strings.Join(conf.Transports, "+"))) - } - if !conf.NumericEnumsDisabled { - args = append(args, "--go_gapic_opt", "rest-numeric-enums") - } - if stubsDir := conf.getStubsDir(); stubsDir != "" { - // Enable protobuf/gRPC generation in the google-cloud-go directory. - args = append(args, "--go_out=plugins=grpc:"+g.googleCloudDir) - - // For each file to be generated i.e. each file in the proto package, - // override the go_package option. Applied to both the protobuf/gRPC - // generated code, and to notify the GAPIC generator of the new - // import path used to reference those stubs. - stubPkgPath := filepath.Join(conf.ImportPath, stubsDir) - for _, f := range protoFiles { - f = strings.TrimPrefix(f, g.googleapisDir+"/") - // Storage is a special case because it is generating a hidden beta - // proto surface. - if conf.ImportPath == "cloud.google.com/go/storage/internal/apiv2" { - rerouteGoPkg := fmt.Sprintf("M%s=%s;%s", f, stubPkgPath, conf.Pkg) - args = append(args, - "--go_opt="+rerouteGoPkg, - "--go_gapic_opt="+rerouteGoPkg, - ) - } else { - var stubPkg string - if conf.InputDirectoryPath == "google/devtools/containeranalysis/v1beta1/grafeas" { - // grafeas is a special case since protos are not at the root of - // client definition - stubPkgPath = "cloud.google.com/go/containeranalysis/apiv1beta1/grafeas/grafeaspb" - stubPkg = "grafeaspb" - } else if conf.InputDirectoryPath == "google/firestore/admin/v1" { - // firestore/admin is a special case since the gapic is generated - // at a non-standard spot - stubPkgPath = "cloud.google.com/go/firestore/apiv1/admin/adminpb" - stubPkg = "adminpb" - } else { - stubPkg = conf.Pkg + "pb" - } - rerouteGoPkg := fmt.Sprintf("M%s=%s;%s", f, stubPkgPath, stubPkg) - args = append(args, "--go_opt="+rerouteGoPkg) - if conf.isMigrated() { - args = append(args, "--go_gapic_opt="+rerouteGoPkg) - } - } - } - } - - args = append(args, protoFiles...) - c := execv.Command("protoc", args...) - c.Dir = g.googleapisDir - return c.Run() -} - -// copyMicrogenFiles takes microgen files from gocloudDir/cloud.google.com/go -// and places them in gocloudDir. -func (g *GapicGenerator) copyMicrogenFiles() error { - // The period at the end is analagous to * (copy everything in this dir). - c := execv.Command("cp", "-R", g.googleCloudDir+"/cloud.google.com/go/.", ".") - c.Dir = g.googleCloudDir - if err := c.Run(); err != nil { - return err - } - - c = execv.Command("rm", "-rf", "cloud.google.com") - c.Dir = g.googleCloudDir - return c.Run() -} diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index 0cca200137d0..19731ac24f45 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -31,7 +31,6 @@ import ( type Config struct { GoogleapisDir string GenprotoDir string - GapicDir string ProtoDir string GapicToGenerate string OnlyGenerateGapic bool @@ -61,18 +60,6 @@ func Generate(ctx context.Context, conf *Config) ([]*git.ChangeInfo, error) { return nil, err } } - var modifiedPkgs []string - for _, v := range changes { - if v.Package != "" { - modifiedPkgs = append(modifiedPkgs, v.PackageDir) - } - } - - gapicGenerator := NewGapicGenerator(conf, modifiedPkgs) - if err := gapicGenerator.Regen(ctx); err != nil { - return nil, fmt.Errorf("error generating gapics (may need to check logs for more errors): %v", err) - } - return changes, nil } @@ -86,11 +73,7 @@ func gatherChanges(googleapisDir, genprotoDir string) ([]*git.ChangeInfo, error) if err != nil { return nil, err } - gapicPkgs := make(map[string]string) - for _, v := range MicrogenGapicConfigs { - gapicPkgs[v.InputDirectoryPath] = v.ImportPath - } - changes, err := git.ParseChangeInfo(googleapisDir, commits, gapicPkgs) + changes, err := git.ParseChangeInfo(googleapisDir, commits) if err != nil { return nil, err } diff --git a/internal/gapicgen/generator/genproto.go b/internal/gapicgen/generator/genproto.go index b8cd943bfe11..34bcdd9d83b8 100644 --- a/internal/gapicgen/generator/genproto.go +++ b/internal/gapicgen/generator/genproto.go @@ -25,8 +25,6 @@ import ( "strconv" "strings" - "cloud.google.com/go/internal/aliasfix" - "cloud.google.com/go/internal/aliasgen" "cloud.google.com/go/internal/gapicgen/execv" "cloud.google.com/go/internal/gapicgen/execv/gocmd" "cloud.google.com/go/internal/gapicgen/git" @@ -35,26 +33,13 @@ import ( var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`) -// denylist is a set of clients to NOT generate. -var denylist = map[string]bool{ - // Temporarily stop generation of removed protos. Will be manually cleaned - // up with: https://github.com/googleapis/google-cloud-go/issues/4098 - "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1alpha2": true, - - // Not properly configured: - "google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1beta1": true, - "google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1": true, -} - // GenprotoGenerator is used to generate code for googleapis/go-genproto. type GenprotoGenerator struct { genprotoDir string googleapisDir string protoSrcDir string - googleCloudDir string gapicToGenerate string forceAll bool - genAlias bool } // NewGenprotoGenerator creates a new GenprotoGenerator. @@ -63,19 +48,36 @@ func NewGenprotoGenerator(c *Config) *GenprotoGenerator { genprotoDir: c.GenprotoDir, googleapisDir: c.GoogleapisDir, protoSrcDir: filepath.Join(c.ProtoDir, "/src"), - googleCloudDir: c.GapicDir, gapicToGenerate: c.GapicToGenerate, forceAll: c.ForceAll, - genAlias: c.GenAlias, } } +// TODO: consider flipping this to an allowlist var skipPrefixes = []string{ "google.golang.org/genproto/googleapis/ads/", + "google.golang.org/genproto/googleapis/analytics/", + "google.golang.org/genproto/googleapis/api/servicecontrol/", + "google.golang.org/genproto/googleapis/api/servicemanagement/", + "google.golang.org/genproto/googleapis/api/serviceusage/", + "google.golang.org/genproto/googleapis/appengine/", + "google.golang.org/genproto/googleapis/area120/", "google.golang.org/genproto/googleapis/cloud/", + "google.golang.org/genproto/googleapis/dataflow/", + "google.golang.org/genproto/googleapis/datastore/", "google.golang.org/genproto/googleapis/devtools/", + "google.golang.org/genproto/googleapis/firestore/", + "google.golang.org/genproto/googleapis/iam/", + "google.golang.org/genproto/googleapis/identity/", + "google.golang.org/genproto/googleapis/logging/", + "google.golang.org/genproto/googleapis/longrunning/", "google.golang.org/genproto/googleapis/maps/", + "google.golang.org/genproto/googleapis/monitoring/", + "google.golang.org/genproto/googleapis/privacy/", + "google.golang.org/genproto/googleapis/pubsub/", + "google.golang.org/genproto/googleapis/spanner/", "google.golang.org/genproto/googleapis/storage/", + "google.golang.org/genproto/googleapis/storagetransfer/", } func hasPrefix(s string, prefixes []string) bool { @@ -103,11 +105,6 @@ func hasPrefix(s string, prefixes []string) bool { // declaring the same Go package. func (g *GenprotoGenerator) Regen(ctx context.Context) error { log.Println("regenerating genproto") - - if g.genAlias { - return g.generateAliases() - } - // Create space to put generated .pb.go's. c := execv.Command("mkdir", "-p", "generated") c.Dir = g.genprotoDir @@ -136,20 +133,16 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error { log.Println("generating from protos") grp, _ := errgroup.WithContext(ctx) for pkg, fileNames := range pkgFiles { - if !strings.HasPrefix(pkg, "google.golang.org/genproto") || denylist[pkg] || hasPrefix(pkg, skipPrefixes) { + if !strings.HasPrefix(pkg, "google.golang.org/genproto") || hasPrefix(pkg, skipPrefixes) { continue } pk := pkg fn := fileNames - if !isMigrated(pkg) { - grp.Go(func() error { - log.Println("running protoc on", pk) - return g.protoc(fn) - }) - } else { - log.Printf("skipping, %q has been migrated", pkg) - } + grp.Go(func() error { + log.Println("running protoc on", pk) + return g.protoc(fn) + }) } if err := grp.Wait(); err != nil { return err @@ -289,22 +282,3 @@ func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error { return nil } - -func (g *GenprotoGenerator) generateAliases() error { - for genprotoImport, newPkg := range aliasfix.GenprotoPkgMigration { - if !isMigrated(genprotoImport) || g.gapicToGenerate == "" { - continue - } - // remove the stubs dir segment from path - gapicImport := newPkg.ImportPath[:strings.LastIndex(newPkg.ImportPath, "/")] - if !strings.Contains(g.gapicToGenerate, gapicImport) { - continue - } - srdDir := filepath.Join(g.googleCloudDir, strings.TrimPrefix(newPkg.ImportPath, "cloud.google.com/go/")) - destDir := filepath.Join(g.genprotoDir, "googleapis", strings.TrimPrefix(genprotoImport, "google.golang.org/genproto/googleapis/")) - if err := aliasgen.Run(srdDir, destDir); err != nil { - return err - } - } - return nil -} diff --git a/internal/gapicgen/git/git.go b/internal/gapicgen/git/git.go index c011e841fcf4..5fa9d4eafbd7 100644 --- a/internal/gapicgen/git/git.go +++ b/internal/gapicgen/git/git.go @@ -18,7 +18,6 @@ import ( "fmt" "log" "os" - "path/filepath" "strings" "cloud.google.com/go/internal/gapicgen/execv" @@ -29,8 +28,6 @@ import ( type ChangeInfo struct { Body string Title string - Package string - PackageDir string GoogleapisHash string } @@ -44,32 +41,10 @@ func FormatChanges(changes []*ChangeInfo, onlyGapicChanges bool) string { var sb strings.Builder sb.WriteString("\nChanges:\n\n") for _, c := range changes { - if onlyGapicChanges && c.Package == "" { + if onlyGapicChanges { continue } - - title := c.Title - if c.Package != "" { - // Try to add in pkg affected into conventional commit scope. - titleParts := strings.SplitN(c.Title, ":", 2) - if len(titleParts) == 2 { - // If a scope is already provided, remove it. - if i := strings.Index(titleParts[0], "("); i > 0 { - titleParts[0] = titleParts[0][:i] - } - - var breakChangeIndicator string - if strings.HasSuffix(titleParts[0], "!") { - // If the change is marked as breaking we need to move the - // bang to after the added scope. - titleParts[0] = titleParts[0][:len(titleParts[0])-1] - breakChangeIndicator = "!" - } - titleParts[0] = fmt.Sprintf("%s(%s)%s", titleParts[0], c.Package, breakChangeIndicator) - } - title = strings.Join(titleParts, ":") - } - sb.WriteString(fmt.Sprintf("%s\n", title)) + sb.WriteString(fmt.Sprintf("%s\n", c.Title)) // Format the commit body to conventional commit footer standards. splitBody := strings.Split(c.Body, "\n") @@ -88,7 +63,7 @@ func FormatChanges(changes []*ChangeInfo, onlyGapicChanges bool) string { } // ParseChangeInfo gets the ChangeInfo for a given googleapis hash. -func ParseChangeInfo(googleapisDir string, hashes []string, gapicPkgs map[string]string) ([]*ChangeInfo, error) { +func ParseChangeInfo(googleapisDir string, hashes []string) ([]*ChangeInfo, error) { var changes []*ChangeInfo for _, hash := range hashes { // Get commit title and body @@ -108,30 +83,9 @@ func ParseChangeInfo(googleapisDir string, hashes []string, gapicPkgs map[string // Add link so corresponding googleapis commit. body = fmt.Sprintf("%s\nSource-Link: https://github.com/googleapis/googleapis/commit/%s", body, hash) - // Try to map files updated to a package in google-cloud-go. Assumes only - // one servies protos are updated per commit. Multile versions are okay. - files, err := filesChanged(googleapisDir, hash) - if err != nil { - return nil, err - } - var pkg, pkgDir string - for _, file := range files { - if file == "" { - continue - } - importPath := gapicPkgs[filepath.Dir(file)] - if importPath != "" { - pkg = parseConventionalCommitPkg(importPath) - pkgDir = strings.TrimPrefix(importPath, "cloud.google.com/go/") - break - } - } - changes = append(changes, &ChangeInfo{ Title: title, Body: body, - Package: pkg, - PackageDir: pkgDir, GoogleapisHash: hash, }) } diff --git a/internal/gapicgen/git/git_test.go b/internal/gapicgen/git/git_test.go index 169fb666d4d9..dcfecd8a05a8 100644 --- a/internal/gapicgen/git/git_test.go +++ b/internal/gapicgen/git/git_test.go @@ -68,27 +68,6 @@ func TestFormatChanges(t *testing.T) { onlyGapics: true, want: "", }, - { - name: "with package", - changes: []*ChangeInfo{{Title: "fix: foo", Body: "bar", Package: "baz"}}, - want: "\nChanges:\n\nfix(baz): foo\n bar\n\n", - }, - { - name: "with package, breaking change", - changes: []*ChangeInfo{{Title: "feat!: foo", Body: "bar", Package: "baz"}}, - want: "\nChanges:\n\nfeat(baz)!: foo\n bar\n\n", - }, - { - name: "multiple changes", - changes: []*ChangeInfo{{Title: "fix: foo", Body: "bar", Package: "foo"}, {Title: "fix: baz", Body: "bar"}}, - want: "\nChanges:\n\nfix(foo): foo\n bar\n\nfix: baz\n bar\n\n", - }, - { - name: "multiple changes, some filtered", - changes: []*ChangeInfo{{Title: "fix: foo", Body: "bar", Package: "foo"}, {Title: "fix: baz", Body: "bar"}}, - onlyGapics: true, - want: "\nChanges:\n\nfix(foo): foo\n bar\n\n", - }, } for _, tc := range tests { diff --git a/internal/gapicgen/git/github.go b/internal/gapicgen/git/github.go index 54f477dfeb5f..abd0f49d3d29 100644 --- a/internal/gapicgen/git/github.go +++ b/internal/gapicgen/git/github.go @@ -16,40 +16,21 @@ package git import ( "context" - "errors" "fmt" "log" "os" "os/user" "path" - "path/filepath" "regexp" "strings" "time" "cloud.google.com/go/internal/gapicgen/execv" - "cloud.google.com/go/internal/gapicgen/execv/gocmd" "github.com/google/go-github/v50/github" - "github.com/shurcooL/githubv4" "golang.org/x/oauth2" ) const ( - gocloudBranchName = "regen_gocloud" - gocloudCommitTitle = "chore(all): auto-regenerate gapics" - gocloudCommitBody = ` -This is an auto-generated regeneration of the gapic clients by -cloud.google.com/go/internal/gapicgen. Once the corresponding genproto PR is -submitted, genbot will update this PR with a newer dependency to the newer -version of genproto and assign reviewers to this PR. - -If you have been assigned to review this PR, please: - -- Ensure that the version of genproto in go.mod has been updated. -- Ensure that CI is passing. If it's failing, it requires your manual attention. -- Approve and submit this PR if you believe it's ready to ship. -` - genprotoBranchName = "regen_genproto" genprotoCommitTitle = "chore(all): auto-regenerate .pb.go files" genprotoCommitBody = ` @@ -66,8 +47,6 @@ If you have been assigned to review this PR, please: - Approve and submit this PR if you believe it's ready to ship. That will prompt genbot to assign reviewers to the google-cloud-go PR. ` - aliasBranchName = "regen_alias" - aliasCommitTitle = "chore(all): auto-regenerate alias files" ) var conventionalCommitScopeRe = regexp.MustCompile(`.*\((.*)\): .*`) @@ -88,7 +67,6 @@ type PullRequest struct { // GithubClient is a convenience wrapper around Github clients. type GithubClient struct { cV3 *github.Client - cV4 *githubv4.Client // Username is the GitHub username. Read-only. Username string } @@ -103,7 +81,7 @@ func NewGithubClient(ctx context.Context, username, name, email, accessToken str &oauth2.Token{AccessToken: accessToken}, ) tc := oauth2.NewClient(ctx, ts) - return &GithubClient{cV3: github.NewClient(tc), cV4: githubv4.NewClient(tc), Username: username}, nil + return &GithubClient{cV3: github.NewClient(tc), Username: username}, nil } // setGitCreds configures credentials for GitHub. @@ -232,302 +210,6 @@ git push origin $BRANCH_NAME return pr.GetNumber(), nil } -// CreateGocloudPR creates a PR for a given google-cloud-go change. -func (gc *GithubClient) CreateGocloudPR(ctx context.Context, gocloudDir string, genprotoPRNum int, changes []*ChangeInfo) (prNumber int, _ error) { - log.Println("creating google-cloud-go PR") - - var sb strings.Builder - var draft bool - sb.WriteString(gocloudCommitBody) - if genprotoPRNum > 0 { - sb.WriteString(fmt.Sprintf("\n\nCorresponding genproto PR: https://github.com/googleapis/go-genproto/pull/%d\n", genprotoPRNum)) - draft = true - } else { - sb.WriteString("\n\nThere is no corresponding genproto PR.\n") - } - sb.WriteString(FormatChanges(changes, true)) - body := sb.String() - - c := execv.Command("/bin/bash", "-c", ` -set -ex - -git config credential.helper store # cache creds from ~/.git-credentials - -git branch -D $BRANCH_NAME || true -git push -d origin $BRANCH_NAME || true - -git add -A -git checkout -b $BRANCH_NAME -git commit -m "$COMMIT_TITLE" -m "$COMMIT_BODY" -git push origin $BRANCH_NAME -`) - c.Env = []string{ - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("COMMIT_TITLE=%s", gocloudCommitTitle), - fmt.Sprintf("COMMIT_BODY=%s", body), - fmt.Sprintf("BRANCH_NAME=%s", gocloudBranchName), - } - c.Dir = gocloudDir - if err := c.Run(); err != nil { - return 0, err - } - - t := gocloudCommitTitle // Because we have to take the address. - pr, _, err := gc.cV3.PullRequests.Create(ctx, "googleapis", "google-cloud-go", &github.NewPullRequest{ - Title: &t, - Body: &body, - Head: github.String(fmt.Sprintf("googleapis:" + gocloudBranchName)), - Base: github.String("main"), - Draft: github.Bool(draft), - }) - if err != nil { - return 0, err - } - - log.Printf("creating google-cloud-go PR... done %s\n", pr.GetHTMLURL()) - - return pr.GetNumber(), nil -} - -// CreateAliasPR creates a PR for a given genproto change. -func (gc *GithubClient) CreateAliasPR(ctx context.Context, genprotoDir string) error { - log.Println("creating genproto PR") - - c := execv.Command("/bin/bash", "-c", ` -set -ex - -git config credential.helper store # cache creds from ~/.git-credentials - -git branch -D $BRANCH_NAME || true -git push -d origin $BRANCH_NAME || true - -git add -A -git checkout -b $BRANCH_NAME -git commit -m "$COMMIT_TITLE" -git push origin $BRANCH_NAME -`) - c.Env = []string{ - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), - fmt.Sprintf("HOME=%s", os.Getenv("HOME")), - fmt.Sprintf("COMMIT_TITLE=%s", aliasCommitTitle), - fmt.Sprintf("BRANCH_NAME=%s", aliasBranchName), - } - c.Dir = genprotoDir - if err := c.Run(); err != nil { - return err - } - pr, _, err := gc.cV3.PullRequests.Create(ctx, "googleapis", "go-genproto", &github.NewPullRequest{ - Title: github.String(aliasCommitTitle), - Head: github.String(fmt.Sprintf("googleapis:" + aliasBranchName)), - Base: github.String("main"), - }) - if err != nil { - return err - } - log.Printf("creating alias PR: %d\n", pr.GetNumber()) - - return nil -} - -// AmendGenprotoPR amends the given genproto PR with a link to the given -// google-cloud-go PR. -func (gc *GithubClient) AmendGenprotoPR(ctx context.Context, genprotoPRNum int, genprotoDir string, gocloudPRNum int, changes []*ChangeInfo) error { - var body strings.Builder - body.WriteString(genprotoCommitBody) - body.WriteString(fmt.Sprintf("\n\nCorresponding google-cloud-go PR: googleapis/google-cloud-go#%d\n", gocloudPRNum)) - body.WriteString(FormatChanges(changes, false)) - sBody := body.String() - c := execv.Command("/bin/bash", "-c", ` -set -ex - -git config credential.helper store # cache creds from ~/.git-credentials - -git checkout $BRANCH_NAME -git commit --amend -m "$COMMIT_TITLE" -m "$COMMIT_BODY" -git push -f origin $BRANCH_NAME -`) - c.Env = []string{ - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("COMMIT_TITLE=%s", genprotoCommitTitle), - fmt.Sprintf("COMMIT_BODY=%s", sBody), - fmt.Sprintf("BRANCH_NAME=%s", genprotoBranchName), - } - c.Dir = genprotoDir - if err := c.Run(); err != nil { - return err - } - _, _, err := gc.cV3.PullRequests.Edit(ctx, "googleapis", "go-genproto", genprotoPRNum, &github.PullRequest{ - Body: &sBody, - }) - return err -} - -// MarkPRReadyForReview switches a draft pull request to a reviewable pull -// request. -func (gc *GithubClient) MarkPRReadyForReview(ctx context.Context, repo string, nodeID string) error { - var m struct { - MarkPullRequestReadyForReview struct { - PullRequest struct { - ID githubv4.ID - } - } `graphql:"markPullRequestReadyForReview(input: $input)"` - } - input := githubv4.MarkPullRequestReadyForReviewInput{ - PullRequestID: nodeID, - } - if err := gc.cV4.Mutate(ctx, &m, input, nil); err != nil { - return err - } - return nil -} - -// UpdateGocloudGoMod updates the go.mod to include latest version of genproto -// for the given gocloud ref. -func (gc *GithubClient) UpdateGocloudGoMod() error { - tmpDir, err := os.MkdirTemp("", "finalize-github-pr") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - - if err := checkoutCode(tmpDir); err != nil { - return err - } - if err := updateDeps(tmpDir); err != nil { - return err - } - if err := addAndPushCode(tmpDir); err != nil { - return err - } - - return nil -} - -func checkoutCode(tmpDir string) error { - c := execv.Command("/bin/bash", "-c", ` -set -ex - -git init -git remote add origin https://github.com/googleapis/google-cloud-go -git fetch --all -git checkout $BRANCH_NAME -`) - c.Env = []string{ - fmt.Sprintf("BRANCH_NAME=%s", gocloudBranchName), - } - c.Dir = tmpDir - return c.Run() -} - -func updateDeps(tmpDir string) error { - // Find directories that had code changes. - c := execv.Command("git", "diff", "--name-only", "HEAD", "HEAD~1") - c.Dir = tmpDir - out, err := c.Output() - if err != nil { - return err - } - files := strings.Split(string(out), "\n") - dirs := map[string]bool{} - for _, file := range files { - if strings.HasPrefix(file, "internal") { - continue - } - dir := filepath.Dir(file) - dirs[filepath.Join(tmpDir, dir)] = true - } - - // Find which modules had code changes. - updatedModDirs := map[string]bool{} - for dir := range dirs { - modDir, err := gocmd.ListModDirName(dir) - if err != nil { - if errors.Is(err, gocmd.ErrBuildConstraint) { - continue - } - return err - } - updatedModDirs[modDir] = true - } - - // Find modules based on conventional commit messages. - commitModDirs, err := processCommitMessage(tmpDir) - if err != nil { - return err - } - for _, v := range commitModDirs { - updatedModDirs[v] = true - } - - // Update required modules. - for modDir := range updatedModDirs { - log.Printf("Updating module dir %q", modDir) - c := execv.Command("/bin/bash", "-c", ` -set -ex - -go get -d google.golang.org/api | true # We don't care that there's no files at root. -go get -d google.golang.org/genproto | true # We don't care that there's no files at root. -`) - c.Dir = modDir - if err := c.Run(); err != nil { - return err - } - } - - // Tidy all modules - return gocmd.ModTidyAll(tmpDir) -} - -func addAndPushCode(tmpDir string) error { - c := execv.Command("/bin/bash", "-c", ` -set -ex - -git add -A -filesUpdated=$( git status --short | wc -l ) -if [ $filesUpdated -gt 0 ]; -then - git config credential.helper store # cache creds from ~/.git-credentials - git commit --amend --no-edit - git push -f origin $BRANCH_NAME -fi -`) - c.Env = []string{ - fmt.Sprintf("BRANCH_NAME=%s", gocloudBranchName), - fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands. - } - c.Dir = tmpDir - return c.Run() -} - -// processCommitMessage process the context aware commits mentioned in the PR -// body to determine what modules need to have dependency bumps. -func processCommitMessage(dir string) ([]string, error) { - c := execv.Command("git", "log", "-1", "--pretty=%B") - c.Dir = dir - out, err := c.Output() - if err != nil { - return nil, err - } - outStr := string(out) - ss := strings.Split(outStr, "Changes:\n\n") - if len(ss) != 2 { - return nil, fmt.Errorf("unable to process commit msg") - } - commits := strings.Split(ss[1], "\n\n") - var modDirs []string - for _, v := range commits { - pkg := parsePackage(v) - if pkg == "" { - continue - } - modDirs = append(modDirs, filepath.Join(dir, pkg)) - } - return modDirs, nil -} - // parsePackage parses a package name from the conventional commit scope of a // commit message. func parsePackage(msg string) string { diff --git a/internal/gapicgen/go.mod b/internal/gapicgen/go.mod index 5221db1387e6..e0c55f89a529 100644 --- a/internal/gapicgen/go.mod +++ b/internal/gapicgen/go.mod @@ -3,11 +3,8 @@ module cloud.google.com/go/internal/gapicgen go 1.20 require ( - cloud.google.com/go/internal/aliasfix v0.0.0-20230404205613-6e536a548996 - cloud.google.com/go/internal/aliasgen v0.0.0-20230404205613-6e536a548996 github.com/go-git/go-git/v5 v5.6.1 github.com/google/go-github/v50 v50.2.0 - github.com/shurcooL/githubv4 v0.0.0-20230215024106-420ad0987b9b golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 ) @@ -29,18 +26,13 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect - github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect github.com/skeema/knownhosts v1.1.0 // indirect github.com/stretchr/testify v1.8.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect golang.org/x/crypto v0.7.0 // indirect - golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.54.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/internal/gapicgen/go.sum b/internal/gapicgen/go.sum index 9452fcb352be..a8074fa47f65 100644 --- a/internal/gapicgen/go.sum +++ b/internal/gapicgen/go.sum @@ -1,7 +1,3 @@ -cloud.google.com/go/internal/aliasfix v0.0.0-20230404205613-6e536a548996 h1:xTzBraLhNJZ2d5nrdw3uZWfQIB0dKVhwNttgDjCY75Y= -cloud.google.com/go/internal/aliasfix v0.0.0-20230404205613-6e536a548996/go.mod h1:D+dJ6okFu10AX545irTA9Ts/1QmCESAV7XdTZ2VHmss= -cloud.google.com/go/internal/aliasgen v0.0.0-20230404205613-6e536a548996 h1:NB0J7HMb10gIqDv6mIHi+ZovHVp0o86MyFDhVZfA2lY= -cloud.google.com/go/internal/aliasgen v0.0.0-20230404205613-6e536a548996/go.mod h1:H1y5YASWOlqI7A8GaNgWg+eaJVi5LaeLW9Ho2Q7nhFU= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= @@ -74,10 +70,6 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/githubv4 v0.0.0-20230215024106-420ad0987b9b h1:i0X9yN+P00PrKM2VZ4pc7K9m4bAQWpYnQDCk+VH4BA0= -github.com/shurcooL/githubv4 v0.0.0-20230215024106-420ad0987b9b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 h1:B1PEwpArrNp4dkQrfxh/abbBAOZBVp0ds+fBEOUOqOc= -github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= @@ -106,8 +98,6 @@ golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -162,16 +152,10 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= diff --git a/internal/postprocessor/config.yaml b/internal/postprocessor/config.yaml index 1563bbc4b8f0..cda2506aa2d7 100644 --- a/internal/postprocessor/config.yaml +++ b/internal/postprocessor/config.yaml @@ -112,6 +112,7 @@ modules: - shell - spanner - speech + - storage - storagetransfer - talent - texttospeech @@ -340,12 +341,8 @@ service-configs: service-config: cloudchannel_v1.yaml - input-directory: google/cloud/clouddms/v1 service-config: datamigration_v1.yaml - - input-directory: google/cloud/tasks/v2 - service-config: cloudtasks_v2.yaml - - input-directory: google/cloud/tasks/v2beta2 - service-config: cloudtasks_v2beta2.yaml - - input-directory: google/cloud/tasks/v2beta3 - service-config: cloudtasks_v2beta3.yaml + - input-directory: google/cloud/compute/v1 + service-config: compute_v1.yaml - input-directory: google/cloud/contactcenterinsights/v1 service-config: contactcenterinsights_v1.yaml - input-directory: google/cloud/datacatalog/lineage/v1 @@ -494,6 +491,10 @@ service-configs: service-config: cloudprivatecatalog_v1beta1.yaml - input-directory: google/cloud/pubsublite/v1 service-config: pubsublite_v1.yaml + - input-directory: google/cloud/recaptchaenterprise/v1 + service-config: recaptchaenterprise_v1.yaml + - input-directory: google/cloud/recaptchaenterprise/v1beta1 + service-config: recaptchaenterprise_v1beta1.yaml - input-directory: google/cloud/recommendationengine/v1beta1 service-config: recommendationengine_v1beta1.yaml - input-directory: google/cloud/recommender/v1 @@ -554,10 +555,22 @@ service-configs: service-config: jobs_v4.yaml - input-directory: google/cloud/talent/v4beta1 service-config: jobs_v4beta1.yaml +- input-directory: google/cloud/tasks/v2 + service-config: cloudtasks_v2.yaml + - input-directory: google/cloud/tasks/v2beta2 + service-config: cloudtasks_v2beta2.yaml + - input-directory: google/cloud/tasks/v2beta3 + service-config: cloudtasks_v2beta3.yaml - input-directory: google/cloud/texttospeech/v1 service-config: texttospeech_v1.yaml - input-directory: google/cloud/tpu/v1 service-config: tpu_v1.yaml + - input-directory: google/cloud/translate/v3 + service-config: translate_v3.yaml + - input-directory: google/devtools/clouddebugger/v2 + service-config: clouddebugger_v2.yaml + - input-directory: google/devtools/clouderrorreporting/v1beta1 + service-config: clouderrorreporting_v1beta1.yaml - input-directory: google/devtools/cloudtrace/v1 service-config: cloudtrace_v1.yaml - input-directory: google/devtools/cloudtrace/v2 @@ -572,6 +585,10 @@ service-configs: service-config: ../videointelligence_v1beta2.yaml - input-directory: google/cloud/videointelligence/v1p3beta1 service-config: videointelligence_v1p3beta1.yaml + - input-directory: google/cloud/vision/v1 + service-config: vision_v1.yaml + - input-directory: google/cloud/vision/v1p1beta1 + service-config: vision_v1p1beta1.yaml - input-directory: google/cloud/vmmigration/v1 service-config: vmmigration_v1.yaml - input-directory: google/cloud/vmwareengine/v1 @@ -592,6 +609,10 @@ service-configs: service-config: workflows_v1.yaml - input-directory: google/cloud/workflows/v1beta service-config: workflows_v1beta.yaml + - input-directory: google/cloud/workstations/v1beta + service-config: workstations_v1beta.yaml + - input-directory: google/cloud/workstations/v1 + service-config: workstations_v1.yaml - input-directory: google/container/v1 service-config: container_v1.yaml - input-directory: google/dataflow/v1beta3 @@ -638,6 +659,8 @@ service-configs: service-config: monitoring.yaml - input-directory: google/monitoring/metricsscope/v1 service-config: monitoring.yaml + - input-directory: google/monitoring/v3 + service-config: monitoring.yaml - input-directory: google/privacy/dlp/v2 service-config: dlp_v2.yaml - input-directory: google/pubsub/v1 @@ -648,44 +671,16 @@ service-configs: service-config: spanner_admin_instance.yaml - input-directory: google/spanner/v1 service-config: spanner.yaml + - input-directory: google/storage/v2 + service-config: storage_v2.yaml + import-path: cloud.google.com/go/storage/internal/apiv2 - input-directory: google/storagetransfer/v1 service-config: storagetransfer_v1.yaml - - input-directory: google/cloud/workstations/v1beta - service-config: workstations_v1beta.yaml - - input-directory: google/cloud/workstations/v1 - service-config: workstations_v1.yaml - # All entries below are unmigrated clients. We need to store the import-path - # info for them since they don't have an entry in owlbot.yaml yet. + # All entries below are clients that are not currently copied by owlbot. They + # explicitly express thier import path for some code gen. - input-directory: google/devtools/containeranalysis/v1beta1/grafeas service-config: ../containeranalysis_v1beta1.yaml import-path: cloud.google.com/go/containeranalysis/apiv1beta1 - - input-directory: google/devtools/clouddebugger/v2 - service-config: clouddebugger_v2.yaml - import-path: cloud.google.com/go/debugger/apiv2 - - input-directory: google/devtools/clouderrorreporting/v1beta1 - service-config: clouderrorreporting_v1beta1.yaml - import-path: cloud.google.com/go/errorreporting/apiv1beta1 - - input-directory: google/monitoring/v3 - service-config: monitoring.yaml - import-path: cloud.google.com/go/monitoring/apiv3/v2 - - input-directory: google/cloud/recaptchaenterprise/v1 - service-config: recaptchaenterprise_v1.yaml - import-path: cloud.google.com/go/recaptchaenterprise/v2/apiv1 - - input-directory: google/cloud/recaptchaenterprise/v1beta1 - service-config: recaptchaenterprise_v1beta1.yaml - import-path: cloud.google.com/go/recaptchaenterprise/v2/apiv1beta1 - - input-directory: google/storage/v2 - service-config: storage_v2.yaml - import-path: cloud.google.com/go/storage/internal/apiv2 - - input-directory: google/cloud/translate/v3 - service-config: translate_v3.yaml - import-path: cloud.google.com/go/translate/apiv3 - - input-directory: google/cloud/vision/v1 - service-config: vision_v1.yaml - import-path: cloud.google.com/go/vision/v2/apiv1 - - input-directory: google/cloud/vision/v1p1beta1 - service-config: vision_v1p1beta1.yaml - import-path: cloud.google.com/go/vision/v2/apiv1p1beta1 - input-directory: google/cloud/video/stitcher/v1 service-config: videostitcher_v1.yaml import-path: cloud.google.com/go/video/stitcher/apiv1