Skip to content
Permalink
Browse files
feat(internal/godocfx): add support for other modules (#4290)
I chose to process all modules at the same time, rather than in separate
invocations of godocfx, so we wouldn't have to make duplicate requests
to the module index.

The docs.metadata generation will probably need to be updated in a
future PR to support different final URL destinations.
  • Loading branch information
tbpg committed Jun 22, 2021
1 parent 4550c9e commit d52bae6cd77474174192c46236d309bf967dfa00
Showing with 39 additions and 40 deletions.
  1. +5 −5 internal/godocfx/index.go
  2. +2 −2 internal/godocfx/index_test.go
  3. +31 −31 internal/godocfx/main.go
  4. +1 −2 internal/godocfx/parse.go
@@ -28,7 +28,7 @@ import (

// indexer gets a limited list of entries from index.golang.org.
type indexer interface {
get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error)
get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error)
}

// indexClient is used to access index.golang.org.
@@ -49,7 +49,7 @@ type indexEntry struct {
// versions since the given timestamp.
//
// newModules stores the timestamp of the last successful run with tSaver.
func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) {
func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefixes []string) ([]indexEntry, error) {
since, err := tSaver.get(ctx)
if err != nil {
return nil, err
@@ -61,7 +61,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string)
for {
count++
var cur []indexEntry
cur, since, err = i.get(prefix, since)
cur, since, err = i.get(prefixes, since)
if err != nil {
return nil, err
}
@@ -80,7 +80,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string)

// get fetches a single chronological page of modules from
// index.golang.org/index.
func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) {
func (indexClient) get(prefixes []string, since time.Time) ([]indexEntry, time.Time, error) {
entries := []indexEntry{}
sinceString := since.Format(time.RFC3339)
resp, err := http.Get("https://index.golang.org/index?since=" + sinceString)
@@ -96,7 +96,7 @@ func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time,
return nil, time.Time{}, err
}
last = e.Timestamp // Always update the last timestamp.
if !strings.HasPrefix(e.Path, prefix) ||
if !hasPrefix(e.Path, prefixes) ||
strings.Contains(e.Path, "internal") ||
strings.Contains(e.Path, "third_party") ||
strings.Contains(e.Version, "-") { // Filter out pseudo-versions.
@@ -26,7 +26,7 @@ const wantEntries = 5

type fakeIC struct{}

func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) {
func (f fakeIC) get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error) {
e := indexEntry{Timestamp: since.Add(24 * time.Hour)}
return []indexEntry{e}, e.Timestamp, nil
}
@@ -49,7 +49,7 @@ func (f *fakeTS) put(context.Context, time.Time) error {
func TestNewModules(t *testing.T) {
ic := fakeIC{}
ts := &fakeTS{}
entries, err := newModules(context.Background(), ic, ts, "cloud.google.com")
entries, err := newModules(context.Background(), ic, ts, []string{"cloud.google.com"})
if err != nil {
t.Fatalf("newModules got err: %v", err)
}
@@ -58,42 +58,43 @@ func main() {
outDir := flag.String("out", "obj/api", "Output directory (default obj/api)")
projectID := flag.String("project", "", "Project ID to use. Required when using -new-modules.")
newMods := flag.Bool("new-modules", false, "Process all new modules with the given prefix. Uses timestamp in Datastore. Stores results in $out/$mod.")
// TODO: flag to set output URL path

log.SetPrefix("[godocfx] ")

flag.Parse()
if flag.NArg() != 1 {
if flag.NArg() == 0 {
log.Fatalf("%s missing required argument: module path/prefix", os.Args[0])
}

mod := flag.Arg(0)
modNames := flag.Args()
var mods []indexEntry
if *newMods {
if *projectID == "" {
log.Fatal("Must set -project when using -new-modules")
}
var err error
mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, mod)
mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, modNames)
if err != nil {
log.Fatal(err)
}
} else {
modPath := mod
version := "latest"
if strings.Contains(mod, "@") {
parts := strings.Split(mod, "@")
if len(parts) != 2 {
log.Fatal("module arg expected only one '@'")
for _, mod := range modNames {
modPath := mod
version := "latest"
if strings.Contains(mod, "@") {
parts := strings.Split(mod, "@")
if len(parts) != 2 {
log.Fatal("module arg expected only one '@'")
}
modPath = parts[0]
version = parts[1]
}
modPath = parts[0]
version = parts[1]
}
modPath = strings.TrimSuffix(modPath, "/...") // No /... needed.
mods = []indexEntry{
{
modPath = strings.TrimSuffix(modPath, "/...") // No /... needed.
mods = append(mods, indexEntry{
Path: modPath,
Version: version,
},
})
}
}

@@ -105,21 +106,19 @@ func main() {
return
}
// Create a temp module so we can get the exact version asked for.
tempDir, err := ioutil.TempDir("", "godocfx-*")
workingDir, err := ioutil.TempDir("", "godocfx-*")
if err != nil {
log.Fatalf("ioutil.TempDir: %v", err)
}
runCmd(tempDir, "go", "mod", "init", "cloud.google.com/go/lets-build-some-docs")
// Use a fake module that doesn't start with cloud.google.com/go.
runCmd(workingDir, "go", "mod", "init", "cloud.google.com/lets-build-some-docs")
for _, m := range mods {
log.Printf("Processing %s@%s", m.Path, m.Version)

path := *outDir
// If we have more than one module, we need a more specific out path.
if len(mods) > 1 {
path = filepath.Join(path, fmt.Sprintf("%s@%s", m.Path, m.Version))
}
if err := process(m, tempDir, path, *print); err != nil {
log.Printf("Failed to process %v", m)
// Always output to specific directory.
path := filepath.Join(*outDir, fmt.Sprintf("%s@%s", m.Path, m.Version))
if err := process(m, workingDir, path, *print); err != nil {
log.Printf("Failed to process %v: %v", m, err)
}
log.Printf("Done with %s@%s", m.Path, m.Version)
}
@@ -138,22 +137,23 @@ func runCmd(dir, name string, args ...string) error {
return nil
}

func process(mod indexEntry, tempDir, outDir string, print bool) error {
func process(mod indexEntry, workingDir, outDir string, print bool) error {
// Be sure to get the module and run the module loader in the tempDir.
if err := runCmd(tempDir, "go", "mod", "tidy"); err != nil {
return err
if err := runCmd(workingDir, "go", "mod", "tidy"); err != nil {
return fmt.Errorf("go mod tidy error: %v", err)
}
if err := runCmd(tempDir, "go", "get", mod.Path+"@"+mod.Version); err != nil {
return err
if err := runCmd(workingDir, "go", "get", "-d", "-t", mod.Path+"/...@"+mod.Version); err != nil {
return fmt.Errorf("go get %s@%s: %v", mod.Path, mod.Version, err)
}

log.Println("Starting to parse")
optionalExtraFiles := []string{}
filter := []string{
"cloud.google.com/go/analytics",
"cloud.google.com/go/area120",
"cloud.google.com/go/gsuiteaddons",
}
r, err := parse(mod.Path+"/...", tempDir, optionalExtraFiles, filter)
r, err := parse(mod.Path+"/...", workingDir, optionalExtraFiles, filter)
if err != nil {
return fmt.Errorf("parse: %v", err)
}
@@ -452,8 +452,6 @@ func (l *linker) linkify(s string) string {
return fmt.Sprintf("%s%s.%s", prefix, href(l.toURL(pkgPath, ""), pkg), href(l.toURL(pkgPath, name), name))
}

// TODO: link to the right baseURL, with the right module name and version
// pattern.
func (l *linker) toURL(pkg, name string) string {
if pkg == "" {
if anchor := l.idToAnchor[""][name]; anchor != "" {
@@ -467,6 +465,7 @@ func (l *linker) toURL(pkg, name string) string {
pkgRemainder = pkg[len(mod.Path)+1:] // +1 to skip slash.
}
// Note: we always link to latest. One day, we'll link to mod.Version.
// Also, other packages may have different paths.
baseURL := fmt.Sprintf("/go/docs/reference/%v/latest/%v", mod.Path, pkgRemainder)
if anchor := l.idToAnchor[pkg][name]; anchor != "" {
return fmt.Sprintf("%s#%s", baseURL, anchor)

0 comments on commit d52bae6

Please sign in to comment.