Skip to content

Commit d52bae6

Browse files
tbpgtbpg
authored
feat(internal/godocfx): add support for other modules (#4290)
I chose to process all modules at the same time, rather than in separate invocations of godocfx, so we wouldn't have to make duplicate requests to the module index. The docs.metadata generation will probably need to be updated in a future PR to support different final URL destinations.
1 parent 4550c9e commit d52bae6

File tree

4 files changed

+39
-40
lines changed

4 files changed

+39
-40
lines changed

‎internal/godocfx/index.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828

2929
// indexer gets a limited list of entries from index.golang.org.
3030
type indexer interface {
31-
get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error)
31+
get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error)
3232
}
3333

3434
// indexClient is used to access index.golang.org.
@@ -49,7 +49,7 @@ type indexEntry struct {
4949
// versions since the given timestamp.
5050
//
5151
// newModules stores the timestamp of the last successful run with tSaver.
52-
func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) {
52+
func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefixes []string) ([]indexEntry, error) {
5353
since, err := tSaver.get(ctx)
5454
if err != nil {
5555
return nil, err
@@ -61,7 +61,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string)
6161
for {
6262
count++
6363
var cur []indexEntry
64-
cur, since, err = i.get(prefix, since)
64+
cur, since, err = i.get(prefixes, since)
6565
if err != nil {
6666
return nil, err
6767
}
@@ -80,7 +80,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string)
8080

8181
// get fetches a single chronological page of modules from
8282
// index.golang.org/index.
83-
func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) {
83+
func (indexClient) get(prefixes []string, since time.Time) ([]indexEntry, time.Time, error) {
8484
entries := []indexEntry{}
8585
sinceString := since.Format(time.RFC3339)
8686
resp, err := http.Get("https://index.golang.org/index?since=" + sinceString)
@@ -96,7 +96,7 @@ func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time,
9696
return nil, time.Time{}, err
9797
}
9898
last = e.Timestamp // Always update the last timestamp.
99-
if !strings.HasPrefix(e.Path, prefix) ||
99+
if !hasPrefix(e.Path, prefixes) ||
100100
strings.Contains(e.Path, "internal") ||
101101
strings.Contains(e.Path, "third_party") ||
102102
strings.Contains(e.Version, "-") { // Filter out pseudo-versions.

‎internal/godocfx/index_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ const wantEntries = 5
2626

2727
type fakeIC struct{}
2828

29-
func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) {
29+
func (f fakeIC) get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error) {
3030
e := indexEntry{Timestamp: since.Add(24 * time.Hour)}
3131
return []indexEntry{e}, e.Timestamp, nil
3232
}
@@ -49,7 +49,7 @@ func (f *fakeTS) put(context.Context, time.Time) error {
4949
func TestNewModules(t *testing.T) {
5050
ic := fakeIC{}
5151
ts := &fakeTS{}
52-
entries, err := newModules(context.Background(), ic, ts, "cloud.google.com")
52+
entries, err := newModules(context.Background(), ic, ts, []string{"cloud.google.com"})
5353
if err != nil {
5454
t.Fatalf("newModules got err: %v", err)
5555
}

‎internal/godocfx/main.go‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -58,42 +58,43 @@ func main() {
5858
outDir := flag.String("out", "obj/api", "Output directory (default obj/api)")
5959
projectID := flag.String("project", "", "Project ID to use. Required when using -new-modules.")
6060
newMods := flag.Bool("new-modules", false, "Process all new modules with the given prefix. Uses timestamp in Datastore. Stores results in $out/$mod.")
61+
// TODO: flag to set output URL path
6162

6263
log.SetPrefix("[godocfx] ")
6364

6465
flag.Parse()
65-
if flag.NArg() != 1 {
66+
if flag.NArg() == 0 {
6667
log.Fatalf("%s missing required argument: module path/prefix", os.Args[0])
6768
}
6869

69-
mod := flag.Arg(0)
70+
modNames := flag.Args()
7071
var mods []indexEntry
7172
if *newMods {
7273
if *projectID == "" {
7374
log.Fatal("Must set -project when using -new-modules")
7475
}
7576
var err error
76-
mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, mod)
77+
mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, modNames)
7778
if err != nil {
7879
log.Fatal(err)
7980
}
8081
} else {
81-
modPath := mod
82-
version := "latest"
83-
if strings.Contains(mod, "@") {
84-
parts := strings.Split(mod, "@")
85-
if len(parts) != 2 {
86-
log.Fatal("module arg expected only one '@'")
82+
for _, mod := range modNames {
83+
modPath := mod
84+
version := "latest"
85+
if strings.Contains(mod, "@") {
86+
parts := strings.Split(mod, "@")
87+
if len(parts) != 2 {
88+
log.Fatal("module arg expected only one '@'")
89+
}
90+
modPath = parts[0]
91+
version = parts[1]
8792
}
88-
modPath = parts[0]
89-
version = parts[1]
90-
}
91-
modPath = strings.TrimSuffix(modPath, "/...") // No /... needed.
92-
mods = []indexEntry{
93-
{
93+
modPath = strings.TrimSuffix(modPath, "/...") // No /... needed.
94+
mods = append(mods, indexEntry{
9495
Path: modPath,
9596
Version: version,
96-
},
97+
})
9798
}
9899
}
99100

@@ -105,21 +106,19 @@ func main() {
105106
return
106107
}
107108
// Create a temp module so we can get the exact version asked for.
108-
tempDir, err := ioutil.TempDir("", "godocfx-*")
109+
workingDir, err := ioutil.TempDir("", "godocfx-*")
109110
if err != nil {
110111
log.Fatalf("ioutil.TempDir: %v", err)
111112
}
112-
runCmd(tempDir, "go", "mod", "init", "cloud.google.com/go/lets-build-some-docs")
113+
// Use a fake module that doesn't start with cloud.google.com/go.
114+
runCmd(workingDir, "go", "mod", "init", "cloud.google.com/lets-build-some-docs")
113115
for _, m := range mods {
114116
log.Printf("Processing %s@%s", m.Path, m.Version)
115117

116-
path := *outDir
117-
// If we have more than one module, we need a more specific out path.
118-
if len(mods) > 1 {
119-
path = filepath.Join(path, fmt.Sprintf("%s@%s", m.Path, m.Version))
120-
}
121-
if err := process(m, tempDir, path, *print); err != nil {
122-
log.Printf("Failed to process %v", m)
118+
// Always output to specific directory.
119+
path := filepath.Join(*outDir, fmt.Sprintf("%s@%s", m.Path, m.Version))
120+
if err := process(m, workingDir, path, *print); err != nil {
121+
log.Printf("Failed to process %v: %v", m, err)
123122
}
124123
log.Printf("Done with %s@%s", m.Path, m.Version)
125124
}
@@ -138,22 +137,23 @@ func runCmd(dir, name string, args ...string) error {
138137
return nil
139138
}
140139

141-
func process(mod indexEntry, tempDir, outDir string, print bool) error {
140+
func process(mod indexEntry, workingDir, outDir string, print bool) error {
142141
// Be sure to get the module and run the module loader in the tempDir.
143-
if err := runCmd(tempDir, "go", "mod", "tidy"); err != nil {
144-
return err
142+
if err := runCmd(workingDir, "go", "mod", "tidy"); err != nil {
143+
return fmt.Errorf("go mod tidy error: %v", err)
145144
}
146-
if err := runCmd(tempDir, "go", "get", mod.Path+"@"+mod.Version); err != nil {
147-
return err
145+
if err := runCmd(workingDir, "go", "get", "-d", "-t", mod.Path+"/...@"+mod.Version); err != nil {
146+
return fmt.Errorf("go get %s@%s: %v", mod.Path, mod.Version, err)
148147
}
149148

149+
log.Println("Starting to parse")
150150
optionalExtraFiles := []string{}
151151
filter := []string{
152152
"cloud.google.com/go/analytics",
153153
"cloud.google.com/go/area120",
154154
"cloud.google.com/go/gsuiteaddons",
155155
}
156-
r, err := parse(mod.Path+"/...", tempDir, optionalExtraFiles, filter)
156+
r, err := parse(mod.Path+"/...", workingDir, optionalExtraFiles, filter)
157157
if err != nil {
158158
return fmt.Errorf("parse: %v", err)
159159
}

‎internal/godocfx/parse.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -452,8 +452,6 @@ func (l *linker) linkify(s string) string {
452452
return fmt.Sprintf("%s%s.%s", prefix, href(l.toURL(pkgPath, ""), pkg), href(l.toURL(pkgPath, name), name))
453453
}
454454

455-
// TODO: link to the right baseURL, with the right module name and version
456-
// pattern.
457455
func (l *linker) toURL(pkg, name string) string {
458456
if pkg == "" {
459457
if anchor := l.idToAnchor[""][name]; anchor != "" {
@@ -467,6 +465,7 @@ func (l *linker) toURL(pkg, name string) string {
467465
pkgRemainder = pkg[len(mod.Path)+1:] // +1 to skip slash.
468466
}
469467
// Note: we always link to latest. One day, we'll link to mod.Version.
468+
// Also, other packages may have different paths.
470469
baseURL := fmt.Sprintf("/go/docs/reference/%v/latest/%v", mod.Path, pkgRemainder)
471470
if anchor := l.idToAnchor[pkg][name]; anchor != "" {
472471
return fmt.Sprintf("%s#%s", baseURL, anchor)

0 commit comments

Comments
 (0)