feat(internal): auto-generate snippets (#3949)

I created a new package to make it easier to run than calling the binary. I included a binary for local development.

I needed to install build-base in the Docker image to get gcc, which was needed to Load all packages.

I modified gapics.go to replace genproto in every go.mod because the changes need to be visible everywhere to Load every module.

I ran the generator locally and it seems to work. I only included one directory in this PR because there are a lot of files. I can upload the rest in this PR after initial review.

Updates #3916.

internal/gapicgen/cmd/genbot/Dockerfile

@@ -5,7 +5,7 @@ ENV GO111MODULE on
 RUN apk update && \
     apk add ca-certificates wget git unzip
 # Install bash and ssh tools (needed to run regen.sh etc).
-RUN apk add bash openssh openssh-client
+RUN apk add bash openssh openssh-client build-base
 RUN which bash
 
 # Install libc compatibility (required by protoc and go)

internal/gapicgen/generator/gapics.go

@@ -18,11 +18,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"cloud.google.com/go/internal/gensnippets"
+	"golang.org/x/sys/execabs"
 	"gopkg.in/yaml.v2"
 )
 
@@ -74,7 +77,18 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
 		return err
 	}
 
-	if err := g.addModReplaceGenproto(); err != nil {
+	if err := forEachMod(g.googleCloudDir, g.addModReplaceGenproto); err != nil {
+		return err
+	}
+
+	snippetDir := filepath.Join(g.googleCloudDir, "internal", "generated", "snippets")
+	if err := gensnippets.Generate(g.googleCloudDir, snippetDir); err != nil {
+		return fmt.Errorf("error generating snippets: %v", err)
+	}
+	if err := replaceAllForSnippets(g.googleCloudDir, snippetDir); err != nil {
+		return err
+	}
+	if err := goModTidy(snippetDir); err != nil {
 		return err
 	}
 
@@ -86,25 +100,85 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
 		return err
 	}
 
-	if err := g.dropModReplaceGenproto(); err != nil {
+	if err := forEachMod(g.googleCloudDir, g.dropModReplaceGenproto); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// forEachMod runs the given function with the directory of
+// every non-internal module.
+func forEachMod(rootDir string, fn func(dir string) error) error {
+	return filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if strings.Contains(path, "internal") {
+			return filepath.SkipDir
+		}
+		if d.Name() == "go.mod" {
+			if err := fn(filepath.Dir(path)); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func goModTidy(dir string) error {
+	log.Printf("[%s] running go mod tidy", dir)
+	c := command("go", "mod", "tidy")
+	c.Dir = dir
+	c.Env = []string{
+		fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+		fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+	}
+	return c.Run()
+}
+
+func replaceAllForSnippets(googleCloudDir, snippetDir string) error {
+	return forEachMod(googleCloudDir, func(dir string) error {
+		if dir == snippetDir {
+			return nil
+		}
+
+		// Get the module name in this dir.
+		modC := execabs.Command("go", "list", "-m")
+		modC.Dir = dir
+		mod, err := modC.Output()
+		if err != nil {
+			return err
+		}
+
+		// Replace it. Use a relative path to avoid issues on different systems.
+		rel, err := filepath.Rel(snippetDir, dir)
+		if err != nil {
+			return err
+		}
+		c := command("bash", "-c", `go mod edit -replace "$MODULE=$MODULE_PATH"`)
+		c.Dir = snippetDir
+		c.Env = []string{
+			fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+			fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+			fmt.Sprintf("MODULE=%s", mod),
+			fmt.Sprintf("MODULE_PATH=%s", rel),
+		}
+		return c.Run()
+	})
+}
+
 // addModReplaceGenproto adds a genproto replace statement that points genproto
 // to the local copy. This is necessary since the remote genproto may not have
 // changes that are necessary for the in-flight regen.
-func (g *GapicGenerator) addModReplaceGenproto() error {
-	log.Println("adding temporary genproto replace statement")
+func (g *GapicGenerator) addModReplaceGenproto(dir string) error {
+	log.Printf("[%s] adding temporary genproto replace statement", dir)
 	c := command("bash", "-c", `
 set -ex
 
-GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | awk '{print $2}')
-go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR"
+go mod edit -replace "google.golang.org/genproto=$GENPROTO_DIR"
 `)
-	c.Dir = g.googleCloudDir
+	c.Dir = dir
 	c.Env = []string{
 		"GENPROTO_DIR=" + g.genprotoDir,
 		fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
@@ -115,15 +189,14 @@ go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR
 
 // dropModReplaceGenproto drops the genproto replace statement. It is intended
 // to be run after addModReplaceGenproto.
-func (g *GapicGenerator) dropModReplaceGenproto() error {
-	log.Println("removing genproto replace statement")
+func (g *GapicGenerator) dropModReplaceGenproto(dir string) error {
+	log.Printf("[%s] removing genproto replace statement", dir)
 	c := command("bash", "-c", `
 set -ex
 
-GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | grep -v replace | awk '{print $2}')
-go mod edit -dropreplace "google.golang.org/genproto@$GENPROTO_VERSION"
+go mod edit -dropreplace "google.golang.org/genproto"
 `)
-	c.Dir = g.googleCloudDir
+	c.Dir = dir
 	c.Env = []string{
 		fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
 		fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.

internal/gapicgen/go.mod

@@ -1,22 +1,23 @@
 module cloud.google.com/go/internal/gapicgen
 
-go 1.13
+go 1.16
 
 require (
+	cloud.google.com/go/internal/gensnippets v0.0.0-00010101000000-000000000000
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-github/v34 v34.0.0
 	github.com/kr/text v0.2.0 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/shurcooL/githubv4 v0.0.0-20201206200315-234843c633fa
 	github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect
-	github.com/stretchr/testify v1.6.1 // indirect
-	golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 // indirect
 	golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
-	golang.org/x/text v0.3.5 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+replace cloud.google.com/go/internal/gensnippets => ../gensnippets
+
+replace cloud.google.com/go/internal/godocfx => ../godocfx