Skip to content

Commit b70e0fc

Browse files
authored
feat(internal): auto-generate snippets (#3949)
I created a new package to make it easier to run than calling the binary. I included a binary for local development. I needed to install build-base in the Docker image to get gcc, which was needed to Load all packages. I modified gapics.go to replace genproto in every go.mod because the changes need to be visible everywhere to Load every module. I ran the generator locally and it seems to work. I only included one directory in this PR because there are a lot of files. I can upload the rest in this PR after initial review. Updates #3916.
1 parent 8b4adbf commit b70e0fc

File tree

16 files changed

+802
-49
lines changed

16 files changed

+802
-49
lines changed

internal/gapicgen/cmd/genbot/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ENV GO111MODULE on
55
RUN apk update && \
66
apk add ca-certificates wget git unzip
77
# Install bash and ssh tools (needed to run regen.sh etc).
8-
RUN apk add bash openssh openssh-client
8+
RUN apk add bash openssh openssh-client build-base
99
RUN which bash
1010

1111
# Install libc compatibility (required by protoc and go)

internal/gapicgen/generator/gapics.go

Lines changed: 85 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@ import (
1818
"context"
1919
"encoding/json"
2020
"fmt"
21+
"io/fs"
2122
"log"
2223
"os"
2324
"path/filepath"
2425
"strings"
2526

27+
"cloud.google.com/go/internal/gensnippets"
28+
"golang.org/x/sys/execabs"
2629
"gopkg.in/yaml.v2"
2730
)
2831

@@ -74,7 +77,18 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
7477
return err
7578
}
7679

77-
if err := g.addModReplaceGenproto(); err != nil {
80+
if err := forEachMod(g.googleCloudDir, g.addModReplaceGenproto); err != nil {
81+
return err
82+
}
83+
84+
snippetDir := filepath.Join(g.googleCloudDir, "internal", "generated", "snippets")
85+
if err := gensnippets.Generate(g.googleCloudDir, snippetDir); err != nil {
86+
return fmt.Errorf("error generating snippets: %v", err)
87+
}
88+
if err := replaceAllForSnippets(g.googleCloudDir, snippetDir); err != nil {
89+
return err
90+
}
91+
if err := goModTidy(snippetDir); err != nil {
7892
return err
7993
}
8094

@@ -86,25 +100,85 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
86100
return err
87101
}
88102

89-
if err := g.dropModReplaceGenproto(); err != nil {
103+
if err := forEachMod(g.googleCloudDir, g.dropModReplaceGenproto); err != nil {
90104
return err
91105
}
92106

93107
return nil
94108
}
95109

110+
// forEachMod runs the given function with the directory of
111+
// every non-internal module.
112+
func forEachMod(rootDir string, fn func(dir string) error) error {
113+
return filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error {
114+
if err != nil {
115+
return err
116+
}
117+
if strings.Contains(path, "internal") {
118+
return filepath.SkipDir
119+
}
120+
if d.Name() == "go.mod" {
121+
if err := fn(filepath.Dir(path)); err != nil {
122+
return err
123+
}
124+
}
125+
return nil
126+
})
127+
}
128+
129+
func goModTidy(dir string) error {
130+
log.Printf("[%s] running go mod tidy", dir)
131+
c := command("go", "mod", "tidy")
132+
c.Dir = dir
133+
c.Env = []string{
134+
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
135+
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
136+
}
137+
return c.Run()
138+
}
139+
140+
func replaceAllForSnippets(googleCloudDir, snippetDir string) error {
141+
return forEachMod(googleCloudDir, func(dir string) error {
142+
if dir == snippetDir {
143+
return nil
144+
}
145+
146+
// Get the module name in this dir.
147+
modC := execabs.Command("go", "list", "-m")
148+
modC.Dir = dir
149+
mod, err := modC.Output()
150+
if err != nil {
151+
return err
152+
}
153+
154+
// Replace it. Use a relative path to avoid issues on different systems.
155+
rel, err := filepath.Rel(snippetDir, dir)
156+
if err != nil {
157+
return err
158+
}
159+
c := command("bash", "-c", `go mod edit -replace "$MODULE=$MODULE_PATH"`)
160+
c.Dir = snippetDir
161+
c.Env = []string{
162+
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
163+
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
164+
fmt.Sprintf("MODULE=%s", mod),
165+
fmt.Sprintf("MODULE_PATH=%s", rel),
166+
}
167+
return c.Run()
168+
})
169+
}
170+
96171
// addModReplaceGenproto adds a genproto replace statement that points genproto
97172
// to the local copy. This is necessary since the remote genproto may not have
98173
// changes that are necessary for the in-flight regen.
99-
func (g *GapicGenerator) addModReplaceGenproto() error {
100-
log.Println("adding temporary genproto replace statement")
174+
func (g *GapicGenerator) addModReplaceGenproto(dir string) error {
175+
log.Printf("[%s] adding temporary genproto replace statement", dir)
101176
c := command("bash", "-c", `
102177
set -ex
103178
104-
GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | awk '{print $2}')
105-
go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR"
179+
go mod edit -replace "google.golang.org/genproto=$GENPROTO_DIR"
106180
`)
107-
c.Dir = g.googleCloudDir
181+
c.Dir = dir
108182
c.Env = []string{
109183
"GENPROTO_DIR=" + g.genprotoDir,
110184
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
@@ -115,15 +189,14 @@ go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR
115189

116190
// dropModReplaceGenproto drops the genproto replace statement. It is intended
117191
// to be run after addModReplaceGenproto.
118-
func (g *GapicGenerator) dropModReplaceGenproto() error {
119-
log.Println("removing genproto replace statement")
192+
func (g *GapicGenerator) dropModReplaceGenproto(dir string) error {
193+
log.Printf("[%s] removing genproto replace statement", dir)
120194
c := command("bash", "-c", `
121195
set -ex
122196
123-
GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | grep -v replace | awk '{print $2}')
124-
go mod edit -dropreplace "google.golang.org/genproto@$GENPROTO_VERSION"
197+
go mod edit -dropreplace "google.golang.org/genproto"
125198
`)
126-
c.Dir = g.googleCloudDir
199+
c.Dir = dir
127200
c.Env = []string{
128201
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
129202
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.

internal/gapicgen/go.mod

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
module cloud.google.com/go/internal/gapicgen
22

3-
go 1.13
3+
go 1.16
44

55
require (
6+
cloud.google.com/go/internal/gensnippets v0.0.0-00010101000000-000000000000
67
github.com/golang/protobuf v1.5.2 // indirect
78
github.com/google/go-github/v34 v34.0.0
89
github.com/kr/text v0.2.0 // indirect
910
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
1011
github.com/shurcooL/githubv4 v0.0.0-20201206200315-234843c633fa
1112
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect
12-
github.com/stretchr/testify v1.6.1 // indirect
13-
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 // indirect
1413
golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78
1514
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
16-
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
17-
golang.org/x/text v0.3.5 // indirect
18-
google.golang.org/appengine v1.6.7 // indirect
15+
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
1916
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
2017
gopkg.in/src-d/go-git.v4 v4.13.1
2118
gopkg.in/yaml.v2 v2.4.0
2219
)
20+
21+
replace cloud.google.com/go/internal/gensnippets => ../gensnippets
22+
23+
replace cloud.google.com/go/internal/godocfx => ../godocfx

0 commit comments

Comments
 (0)