-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(internal): auto-run godocfx on new mods (#3069)
This uses index.golang.org to list all modules since a given time. For example, see https://index.golang.org/index?since=2019-04-10T19:08:52.997264Z. Each page is listed in chronological order and is limited to 2000 entries (as of today). This stores the last successful time in Datastore. If there is no time in Datastore, it defaults to 10 days ago. There is a bit of indirection with interfaces to enable testing. In the future, we should add the ability to regenerate the YAML for the latest version of all modules. cc @julieqiu @katiehockman
- Loading branch information
Showing
10 changed files
with
373 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| // Copyright 2020 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // +build go1.15 | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "context" | ||
| "encoding/json" | ||
| "log" | ||
| "net/http" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| // indexer gets a limited list of entries from index.golang.org. | ||
| type indexer interface { | ||
| get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) | ||
| } | ||
|
|
||
| // indexClient is used to access index.golang.org. | ||
| type indexClient struct{} | ||
|
|
||
| var _ indexer = indexClient{} | ||
|
|
||
| // indexEntry represents a line in the output of index.golang.org/index. | ||
| type indexEntry struct { | ||
| Path string | ||
| Version string | ||
| Timestamp time.Time | ||
| } | ||
|
|
||
| // newModules returns the new modules with the given prefix. | ||
| // | ||
| // newModules uses index.golang.org/index?since=timestamp to find new module | ||
| // versions since the given timestamp. | ||
| // | ||
| // newModules stores the timestamp of the last successful run with tSaver. | ||
| func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) { | ||
| since, err := tSaver.get(ctx) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| fiveMinAgo := time.Now().Add(-5 * time.Minute).UTC() // When to stop processing. | ||
| entries := []indexEntry{} | ||
| log.Printf("Fetching index.golang.org entries since %s", since.Format(time.RFC3339)) | ||
| count := 0 | ||
| for { | ||
| count++ | ||
| var cur []indexEntry | ||
| cur, since, err = i.get(prefix, since) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| entries = append(entries, cur...) | ||
| if since.After(fiveMinAgo) { | ||
| break | ||
| } | ||
| } | ||
| log.Printf("Parsed %d index.golang.org pages up to %s", count, since.Format(time.RFC3339)) | ||
| if err := tSaver.put(ctx, since); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return entries, nil | ||
| } | ||
|
|
||
| // get fetches a single chronological page of modules from | ||
| // index.golang.org/index. | ||
| func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) { | ||
| entries := []indexEntry{} | ||
| sinceString := since.Format(time.RFC3339) | ||
| resp, err := http.Get("https://index.golang.org/index?since=" + sinceString) | ||
| if err != nil { | ||
| return nil, time.Time{}, err | ||
| } | ||
|
|
||
| s := bufio.NewScanner(resp.Body) | ||
| last := time.Time{} | ||
| for s.Scan() { | ||
| e := indexEntry{} | ||
| if err := json.Unmarshal(s.Bytes(), &e); err != nil { | ||
| return nil, time.Time{}, err | ||
| } | ||
| last = e.Timestamp // Always update the last timestamp. | ||
| if !strings.HasPrefix(e.Path, prefix) || | ||
| strings.Contains(e.Path, "internal") || | ||
| strings.Contains(e.Path, "third_party") || | ||
| strings.Contains(e.Version, "-") { // Filter out pseudo-versions. | ||
| continue | ||
| } | ||
| entries = append(entries, e) | ||
| } | ||
| return entries, last, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // Copyright 2020 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // +build go1.15 | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| const wantEntries = 5 | ||
|
|
||
| type fakeIC struct{} | ||
|
|
||
| func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) { | ||
| e := indexEntry{Timestamp: since.Add(24 * time.Hour)} | ||
| return []indexEntry{e}, e.Timestamp, nil | ||
| } | ||
|
|
||
| type fakeTS struct { | ||
| getCalled, putCalled bool | ||
| } | ||
|
|
||
| func (f *fakeTS) get(context.Context) (time.Time, error) { | ||
| f.getCalled = true | ||
| t := time.Now().Add(-wantEntries * 24 * time.Hour).UTC() | ||
| return t, nil | ||
| } | ||
|
|
||
| func (f *fakeTS) put(context.Context, time.Time) error { | ||
| f.putCalled = true | ||
| return nil | ||
| } | ||
|
|
||
| func TestNewModules(t *testing.T) { | ||
| ic := fakeIC{} | ||
| ts := &fakeTS{} | ||
| entries, err := newModules(context.Background(), ic, ts, "cloud.google.com") | ||
| if err != nil { | ||
| t.Fatalf("newModules got err: %v", err) | ||
| } | ||
| if got, want := len(entries), wantEntries; got != want { | ||
| t.Errorf("newModules got %d entries, want %d", got, want) | ||
| } | ||
| if !ts.getCalled { | ||
| t.Errorf("fakeTS.get was never called") | ||
| } | ||
| if !ts.putCalled { | ||
| t.Errorf("fakeTS.put was never called") | ||
| } | ||
| } |
Oops, something went wrong.