feat(godocfx): create Go DocFX YAML generator (#2854)

Author: tbpg

internal/godocfx/.gitignore

@@ -0,0 +1,3 @@
+obj/
+site/
+docfxgen

internal/godocfx/README.md

@@ -0,0 +1,19 @@
+# Go DocFX YAML Generator
+
+This tool generates DocFX YAML for Go modules.
+
+Only a single module will be processed at once.
+
+By default, the output files are stored at `./obj/api`. You can convert them to
+HTML using [doc-templates](https://github.com/googleapis/doc-templates) and/or
+[doc-pipeline](https://github.com/googleapis/doc-pipeline).
+
+Example usage:
+
+```
+cd module && godocfx ./...
+godocfx cloud.google.com/go/...
+godocfx -print cloud.google.com/go/storage/...
+godocfx -out custom/output/dir cloud.google.com/go/...
+godocfx -rm custom/output/dir cloud.google.com/go/...
+```

internal/godocfx/go.mod

@@ -0,0 +1,12 @@
+module cloud.google.com/go/internal/godocfx
+
+go 1.15
+
+require (
+	cloud.google.com/go v0.65.0
+	cloud.google.com/go/storage v1.11.0
+	golang.org/x/tools v0.0.0-20200913032122-97363e29fc9b
+	gopkg.in/yaml.v2 v2.3.0
+)
+
+replace cloud.google.com/go => ../..

internal/godocfx/go.sum

@@ -0,0 +1,69 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.15
+
+package main
+
+import (
+	"testing"
+
+	_ "cloud.google.com/go/storage" // Implicitly required by test.
+)
+
+func TestParse(t *testing.T) {
+	testPath := "cloud.google.com/go/storage"
+	pages, toc, module, err := parse(testPath)
+	if err != nil {
+		t.Fatalf("Parse: %v", err)
+	}
+	if got, want := len(toc), 1; got != want {
+		t.Fatalf("Parse got len(toc) = %d, want %d", got, want)
+	}
+	if got, want := len(pages), 1; got != want {
+		t.Errorf("Parse got len(pages) = %d, want %d", got, want)
+	}
+	if got := module.Path; got != testPath {
+		t.Fatalf("Parse got module = %q, want %q", got, testPath)
+	}
+
+	page := pages[testPath]
+
+	// Check invariants for every item.
+	for _, item := range page.Items {
+		if got := item.UID; got == "" {
+			t.Errorf("Parse found missing UID: %v", item)
+		}
+
+		if got, want := item.Langs, []string{"go"}; len(got) != 1 || got[0] != want[0] {
+			t.Errorf("Parse %v got langs = %v, want %v", item.UID, got, want)
+		}
+	}
+
+	// Check there is at least one type, const, variable, and function.
+	// Note: no method because they aren't printed for Namespaces yet.
+	wants := []string{"type", "const", "variable", "function"}
+	for _, want := range wants {
+		found := false
+		for _, c := range page.Items {
+			if c.Type == want {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Errorf("Parse got no %q, want at least one", want)
+		}
+	}
+}

internal/godocfx/godocfx_test.go

@@ -0,0 +1,141 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.15
+
+/*Command godocfx generates DocFX YAML for Go code.
+
+Usage:
+
+    godocfx [flags] path
+
+    cd module && godocfx ./...
+    godocfx cloud.google.com/go/...
+    godocfx -print cloud.google.com/go/storage/...
+    godocfx -out custom/output/dir cloud.google.com/go/...
+    godocfx -rm cloud.google.com/go/...
+
+See:
+* https://dotnet.github.io/docfx/spec/metadata_format_spec.html
+* https://github.com/googleapis/doc-templates
+* https://github.com/googleapis/doc-pipeline
+
+TODO:
+  * Cross link referenced packages.
+*/
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v2"
+)
+
+func main() {
+	print := flag.Bool("print", false, "Print instead of save (default false)")
+	rm := flag.Bool("rm", false, "Delete out directory before generating")
+	outDir := flag.String("out", "obj/api", "Output directory (default obj/api)")
+	flag.Parse()
+	if flag.NArg() != 1 {
+		log.Fatalf("%s missing required argument: module path", os.Args[0])
+	}
+
+	pages, toc, module, err := parse(flag.Arg(0))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if *print {
+		if err := yaml.NewEncoder(os.Stdout).Encode(pages); err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println("----- toc.yaml")
+		if err := yaml.NewEncoder(os.Stdout).Encode(toc); err != nil {
+			log.Fatal(err)
+		}
+		return
+	}
+
+	if *rm {
+		os.RemoveAll(*outDir)
+	}
+	if err := os.MkdirAll(*outDir, os.ModePerm); err != nil {
+		log.Fatalf("os.MkdirAll: %v", err)
+	}
+	for path, p := range pages {
+		// Make the module root page the index.
+		if path == module.Path {
+			path = "index"
+		}
+		// Trim the module path from all other paths.
+		path = strings.TrimPrefix(path, module.Path+"/")
+		path = filepath.Join(*outDir, path+".yml")
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+			log.Fatalf("os.MkdirAll: %v", err)
+		}
+		f, err := os.Create(path)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+		fmt.Fprintln(f, "### YamlMime:UniversalReference")
+		if err := yaml.NewEncoder(f).Encode(p); err != nil {
+			log.Fatal(err)
+		}
+
+		path = filepath.Join(*outDir, "toc.yml")
+		f, err = os.Create(path)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+		fmt.Fprintln(f, "### YamlMime:TableOfContent")
+		if err := yaml.NewEncoder(f).Encode(toc); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// Write the docuploader docs.metadata file. Not for DocFX.
+	// See https://github.com/googleapis/docuploader/issues/11.
+	// Example:
+	/*
+		update_time {
+		  seconds: 1600048103
+		  nanos: 183052000
+		}
+		name: "cloud.google.com/go"
+		version: "v0.65.0"
+		language: "go"
+	*/
+	path := filepath.Join(*outDir, "docs.metadata")
+	f, err := os.Create(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	now := time.Now().UTC()
+	fmt.Fprintf(f, `update_time {
+  seconds: %d
+  nanos: %d
+}
+name: %q
+version: %q
+language: "go"`, now.Unix(), now.Nanosecond(), module.Path, module.Version)
+}