feat(internal): generate region tags for snippets (#3962)

I found a couple "manual" clients that needed repo-metadata entries, so
I added them and included the updated JSON.

There are many packages where it doesn't make sense to generate snippets
with any sort of correct region tag. So, I just skipped them. We can
come back if needed.

The region tags end up slightly repetitive (e.g.
accessapproval_generated_accessapproval_apiv1_Client_UpdateAccessApprovalSettings),
but I think they're more predictable as-is because the package path may not
always line up with the API shortname.

More in depth configuration of the region tag would probably make things
considerably more complex (need to associate every example with the underlying
RPC?)

internal/.repo-metadata-full.json

@@ -263,6 +263,14 @@
     "docs_url": "https://pkg.go.dev/cloud.google.com/go/cloudtasks/apiv2beta3",
     "release_level": "beta"
   },
+  "cloud.google.com/go/compute/metadata": {
+    "distribution_name": "cloud.google.com/go/compute/metadata",
+    "description": "Service Metadata API",
+    "language": "Go",
+    "client_library_type": "manual",
+    "docs_url": "https://pkg.go.dev/cloud.google.com/go/compute/metadata",
+    "release_level": "ga"
+  },
   "cloud.google.com/go/container/apiv1": {
     "distribution_name": "cloud.google.com/go/container/apiv1",
     "description": "Kubernetes Engine API",
@@ -455,6 +463,14 @@
     "docs_url": "https://pkg.go.dev/cloud.google.com/go/functions/apiv1",
     "release_level": "ga"
   },
+  "cloud.google.com/go/functions/metadata": {
+    "distribution_name": "cloud.google.com/go/functions/metadata",
+    "description": "Cloud Functions",
+    "language": "Go",
+    "client_library_type": "manual",
+    "docs_url": "https://pkg.go.dev/cloud.google.com/go/functions/metadata",
+    "release_level": "alpha"
+  },
   "cloud.google.com/go/gaming/apiv1": {
     "distribution_name": "cloud.google.com/go/gaming/apiv1",
     "description": "Game Services API",

internal/gapicgen/generator/gapics.go

@@ -82,7 +82,11 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
 	}
 
 	snippetDir := filepath.Join(g.googleCloudDir, "internal", "generated", "snippets")
-	if err := gensnippets.Generate(g.googleCloudDir, snippetDir); err != nil {
+	apiShortnames, err := g.parseAPIShortnames(microgenGapicConfigs, manualEntries)
+	if err != nil {
+		return err
+	}
+	if err := gensnippets.Generate(g.googleCloudDir, snippetDir, apiShortnames); err != nil {
 		return fmt.Errorf("error generating snippets: %v", err)
 	}
 	if err := replaceAllForSnippets(g.googleCloudDir, snippetDir); err != nil {
@@ -329,6 +333,22 @@ var manualEntries = []manifestEntry{
 		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/profiler",
 		ReleaseLevel:      "ga",
 	},
+	{
+		DistributionName:  "cloud.google.com/go/compute/metadata",
+		Description:       "Service Metadata API",
+		Language:          "Go",
+		ClientLibraryType: "manual",
+		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/compute/metadata",
+		ReleaseLevel:      "ga",
+	},
+	{
+		DistributionName:  "cloud.google.com/go/functions/metadata",
+		Description:       "Cloud Functions",
+		Language:          "Go",
+		ClientLibraryType: "manual",
+		DocsURL:           "https://pkg.go.dev/cloud.google.com/go/functions/metadata",
+		ReleaseLevel:      "alpha",
+	},
 	// Manuals with a GAPIC.
 	{
 		DistributionName:  "cloud.google.com/go/errorreporting",
@@ -425,3 +445,32 @@ func (g *GapicGenerator) copyMicrogenFiles() error {
 	c.Dir = g.googleCloudDir
 	return c.Run()
 }
+
+func (g *GapicGenerator) parseAPIShortnames(confs []*microgenConfig, manualEntries []manifestEntry) (map[string]string, error) {
+	shortnames := map[string]string{}
+	for _, conf := range confs {
+		yamlPath := filepath.Join(g.googleapisDir, conf.apiServiceConfigPath)
+		yamlFile, err := os.Open(yamlPath)
+		if err != nil {
+			return nil, err
+		}
+		config := struct {
+			Name string `yaml:"name"`
+		}{}
+		if err := yaml.NewDecoder(yamlFile).Decode(&config); err != nil {
+			return nil, fmt.Errorf("Decode: %v", err)
+		}
+		shortname := strings.TrimSuffix(config.Name, ".googleapis.com")
+		shortnames[conf.importPath] = shortname
+	}
+
+	// Do our best for manuals.
+	for _, manual := range manualEntries {
+		p := strings.TrimPrefix(manual.DistributionName, "cloud.google.com/go/")
+		if strings.Contains(p, "/") {
+			p = p[0:strings.Index(p, "/")]
+		}
+		shortnames[manual.DistributionName] = p
+	}
+	return shortnames, nil
+}

internal/gapicgen/gensnippets/gensnippets.go

@@ -34,7 +34,7 @@ import (
 )
 
 // Generate reads all modules in rootDir and outputs their examples in outDir.
-func Generate(rootDir, outDir string) error {
+func Generate(rootDir, outDir string, apiShortnames map[string]string) error {
 	if rootDir == "" {
 		rootDir = "."
 	}
@@ -60,18 +60,22 @@ func Generate(rootDir, outDir string) error {
 	log.Printf("Processing examples in %v directories: %q\n", len(dirs), dirs)
 
 	trimPrefix := "cloud.google.com/go"
+	errs := []error{}
 	for _, dir := range dirs {
 		// Load does not look at nested modules.
 		pis, err := pkgload.Load("./...", dir, nil)
 		if err != nil {
 			return fmt.Errorf("failed to load packages: %v", err)
 		}
 		for _, pi := range pis {
-			if err := processExamples(pi.Doc, pi.Fset, trimPrefix, outDir); err != nil {
-				return fmt.Errorf("failed to process examples: %v", err)
+			if err := processExamples(pi.Doc, pi.Fset, trimPrefix, outDir, apiShortnames); err != nil {
+				errs = append(errs, fmt.Errorf("failed to process examples: %v", err))
 			}
 		}
 	}
+	if len(errs) > 0 {
+		log.Fatal(errs)
+	}
 
 	if len(dirs) > 0 {
 		cmd := execabs.Command("goimports", "-w", ".")
@@ -84,11 +88,48 @@ func Generate(rootDir, outDir string) error {
 	return nil
 }
 
-func processExamples(pkg *doc.Package, fset *token.FileSet, trimPrefix, outDir string) error {
+var skip = map[string]bool{
+	"cloud.google.com/go":                          true, // No product for root package.
+	"cloud.google.com/go/civil":                    true, // General time/date package.
+	"cloud.google.com/go/cloudbuild/apiv1":         true, // Has v2.
+	"cloud.google.com/go/cmd/go-cloud-debug-agent": true, // Command line tool.
+	"cloud.google.com/go/container":                true, // Deprecated.
+	"cloud.google.com/go/containeranalysis/apiv1":  true, // Accidental beta at wrong path?
+	"cloud.google.com/go/grafeas/apiv1":            true, // With containeranalysis.
+	"cloud.google.com/go/httpreplay":               true, // Helper.
+	"cloud.google.com/go/httpreplay/cmd/httpr":     true, // Helper.
+	"cloud.google.com/go/longrunning":              true, // Helper.
+	"cloud.google.com/go/monitoring/apiv3":         true, // Has v2.
+	"cloud.google.com/go/translate":                true, // Has newer version.
+}
+
+func processExamples(pkg *doc.Package, fset *token.FileSet, trimPrefix, outDir string, apiShortnames map[string]string) error {
+	if skip[pkg.ImportPath] {
+		return nil
+	}
 	trimmed := strings.TrimPrefix(pkg.ImportPath, trimPrefix)
 	outDir = filepath.Join(outDir, trimmed)
 
-	regionTag := "generated" + strings.ReplaceAll(trimmed, "/", "_")
+	shortname, ok := apiShortnames[pkg.ImportPath]
+	if !ok {
+		// Do our best to find a shortname. For example,
+		// cloud.google.com/go/bigtable/bttest should lead to
+		// cloud.google.com/go/bigtable.
+		bestMatch := ""
+		for path := range apiShortnames {
+			if strings.HasPrefix(pkg.ImportPath, path) {
+				if len(path) > len(bestMatch) {
+					bestMatch = path
+				}
+			}
+		}
+		if bestMatch == "" {
+			return fmt.Errorf("could not find API shortname for %v", pkg.ImportPath)
+		}
+		log.Printf("The best match for %q is %q", pkg.ImportPath, bestMatch)
+		shortname = apiShortnames[bestMatch]
+	}
+	regionTag := shortname + "_generated" + strings.ReplaceAll(trimmed, "/", "_")
 
 	// Note: variables and constants don't have examples.
 
@@ -166,18 +207,17 @@ func writeExamples(outDir string, exs []*doc.Example, fset *token.FileSet, regio
 		if _, err := f.WriteString(header()); err != nil {
 			return err
 		}
-		// TODO(tbpg): print the right region tag.
-		// tag := regionTag + "_" + ex.Name
+		tag := regionTag + "_" + ex.Name
 		// Include an extra newline to keep separate from the package declaration.
-		// if _, err := fmt.Fprintf(f, "// [START %v]\n\n", tag); err != nil {
-		// 	return err
-		// }
+		if _, err := fmt.Fprintf(f, "// [START %v]\n\n", tag); err != nil {
+			return err
+		}
 		if _, err := f.WriteString(s); err != nil {
 			return err
 		}
-		// if _, err := fmt.Fprintf(f, "// [END %v]\n", tag); err != nil {
-		// 	return err
-		// }
+		if _, err := fmt.Fprintf(f, "// [END %v]\n", tag); err != nil {
+			return err
+		}
 	}
 	return nil
 }

internal/generated/snippets/accessapproval/apiv1/Client/ApproveApprovalRequest/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_ApproveApprovalRequest]
+
 package main
 
 import (
@@ -40,3 +42,5 @@ func main() {
 	// TODO: Use resp.
 	_ = resp
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_ApproveApprovalRequest]

internal/generated/snippets/accessapproval/apiv1/Client/DeleteAccessApprovalSettings/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_DeleteAccessApprovalSettings]
+
 package main
 
 import (
@@ -36,3 +38,5 @@ func main() {
 		// TODO: Handle error.
 	}
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_DeleteAccessApprovalSettings]

internal/generated/snippets/accessapproval/apiv1/Client/DismissApprovalRequest/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_DismissApprovalRequest]
+
 package main
 
 import (
@@ -40,3 +42,5 @@ func main() {
 	// TODO: Use resp.
 	_ = resp
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_DismissApprovalRequest]

internal/generated/snippets/accessapproval/apiv1/Client/GetAccessApprovalSettings/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_GetAccessApprovalSettings]
+
 package main
 
 import (
@@ -40,3 +42,5 @@ func main() {
 	// TODO: Use resp.
 	_ = resp
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_GetAccessApprovalSettings]

internal/generated/snippets/accessapproval/apiv1/Client/GetApprovalRequest/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_GetApprovalRequest]
+
 package main
 
 import (
@@ -40,3 +42,5 @@ func main() {
 	// TODO: Use resp.
 	_ = resp
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_GetApprovalRequest]

internal/generated/snippets/accessapproval/apiv1/Client/ListApprovalRequests/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_ListApprovalRequests]
+
 package main
 
 import (
@@ -48,3 +50,5 @@ func main() {
 		_ = resp
 	}
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_ListApprovalRequests]

internal/generated/snippets/accessapproval/apiv1/Client/NewClient/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_NewClient]
+
 package main
 
 import (
@@ -29,3 +31,5 @@ func main() {
 	// TODO: Use client.
 	_ = c
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_NewClient]

internal/generated/snippets/accessapproval/apiv1/Client/UpdateAccessApprovalSettings/main.go

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// [START accessapproval_generated_accessapproval_apiv1_Client_UpdateAccessApprovalSettings]
+
 package main
 
 import (
@@ -40,3 +42,5 @@ func main() {
 	// TODO: Use resp.
 	_ = resp
 }
+
+// [END accessapproval_generated_accessapproval_apiv1_Client_UpdateAccessApprovalSettings]