Skip to content
Permalink
Browse files

feat(internal): auto-run godocfx on new mods (#3069)

This uses index.golang.org to list all modules since a given time. For
example, see
https://index.golang.org/index?since=2019-04-10T19:08:52.997264Z. Each
page is listed in chronological order and is limited to 2000 entries (as
of today).

This stores the last successful time in Datastore. If there is no time in
Datastore, it defaults to 10 days ago.

There is a bit of indirection with interfaces to enable testing.

In the future, we should add the ability to regenerate the YAML for the
latest version of all modules.

cc @julieqiu @katiehockman
  • Loading branch information
tbpg committed Oct 22, 2020
1 parent 4c538f4 commit 49f497eab80ce34dfb4ca41f033a5c0429ff5e42
@@ -88,13 +88,6 @@ the failures have been resolved.
`git push origin $NV`
1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases)
with the new release, copying the contents of `CHANGES.md`.
1. Run `go get cloud.google.com/go@vX.Y.Z` then wait for the release
to show up on http://pkg.go.dev/cloud.google.com/go (a few minutes).
1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and
trigger the job with the following environment variables:
`MODULE=cloud.google.com,VERSION=vX.Y.Z`.
Replace the version with the value for the module you're
releasing. See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh).

# How to release a submodule

@@ -131,15 +124,6 @@ To release a submodule:
`git push origin $NV`
1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases)
with the new release, copying the contents of `datastore/CHANGES.md`.
1. Run `go get cloud.google.com/go/datastore@vX.Y.Z` then wait for the release
to show up on http://pkg.go.dev/cloud.google.com/go/datastore (a few
minutes).
1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and
trigger the job with the following environment variables:
`MODULE=cloud.google.com/go/datastore,VERSION=vX.Y.Z`.
Replace the module path and version with the values for the module you're
releasing. You can leave all of the other fields blank.
See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh).

# Appendix

@@ -5,6 +5,7 @@ go 1.15
require (
cloud.google.com/go v0.70.0
cloud.google.com/go/bigquery v1.8.0
cloud.google.com/go/datastore v1.1.0
cloud.google.com/go/storage v1.11.0
github.com/kr/pretty v0.2.1 // indirect
golang.org/x/tools v0.0.0-20201021122455-2be66b663cb6
@@ -283,7 +283,6 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952 h1:y857ZwFJ60XFsJ00vOc7ouVMLOZp7C+7h03pESkILFY=
google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201021134325-0d71844de594 h1:JZWUHUjZJojCHxs9ZZLFsnRGKVBXBoOHGxeTSt6OE+Q=
@@ -37,7 +37,7 @@ func TestMain(m *testing.M) {

func TestParse(t *testing.T) {
mod := "cloud.google.com/go/bigquery"
r, err := parse(mod+"/...", []string{"README.md"})
r, err := parse(mod+"/...", ".", []string{"README.md"})
if err != nil {
t.Fatalf("Parse: %v", err)
}
@@ -103,7 +103,7 @@ func TestGoldens(t *testing.T) {
extraFiles := []string{"README.md"}

testPath := "cloud.google.com/go/storage"
r, err := parse(testPath, extraFiles)
r, err := parse(testPath, ".", extraFiles)
if err != nil {
t.Fatalf("parse: %v", err)
}
@@ -0,0 +1,108 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.15

package main

import (
"bufio"
"context"
"encoding/json"
"log"
"net/http"
"strings"
"time"
)

// indexer gets a limited list of entries from index.golang.org.
type indexer interface {
get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error)
}

// indexClient is used to access index.golang.org.
type indexClient struct{}

var _ indexer = indexClient{}

// indexEntry represents a line in the output of index.golang.org/index.
type indexEntry struct {
Path string
Version string
Timestamp time.Time
}

// newModules returns the new modules with the given prefix.
//
// newModules uses index.golang.org/index?since=timestamp to find new module
// versions since the given timestamp.
//
// newModules stores the timestamp of the last successful run with tSaver.
func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) {
since, err := tSaver.get(ctx)
if err != nil {
return nil, err
}
fiveMinAgo := time.Now().Add(-5 * time.Minute).UTC() // When to stop processing.
entries := []indexEntry{}
log.Printf("Fetching index.golang.org entries since %s", since.Format(time.RFC3339))
count := 0
for {
count++
var cur []indexEntry
cur, since, err = i.get(prefix, since)
if err != nil {
return nil, err
}
entries = append(entries, cur...)
if since.After(fiveMinAgo) {
break
}
}
log.Printf("Parsed %d index.golang.org pages up to %s", count, since.Format(time.RFC3339))
if err := tSaver.put(ctx, since); err != nil {
return nil, err
}

return entries, nil
}

// get fetches a single chronological page of modules from
// index.golang.org/index.
func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) {
entries := []indexEntry{}
sinceString := since.Format(time.RFC3339)
resp, err := http.Get("https://index.golang.org/index?since=" + sinceString)
if err != nil {
return nil, time.Time{}, err
}

s := bufio.NewScanner(resp.Body)
last := time.Time{}
for s.Scan() {
e := indexEntry{}
if err := json.Unmarshal(s.Bytes(), &e); err != nil {
return nil, time.Time{}, err
}
last = e.Timestamp // Always update the last timestamp.
if !strings.HasPrefix(e.Path, prefix) ||
strings.Contains(e.Path, "internal") ||
strings.Contains(e.Path, "third_party") ||
strings.Contains(e.Version, "-") { // Filter out pseudo-versions.
continue
}
entries = append(entries, e)
}
return entries, last, nil
}
@@ -0,0 +1,65 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.15

package main

import (
"context"
"testing"
"time"
)

const wantEntries = 5

type fakeIC struct{}

func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) {
e := indexEntry{Timestamp: since.Add(24 * time.Hour)}
return []indexEntry{e}, e.Timestamp, nil
}

type fakeTS struct {
getCalled, putCalled bool
}

func (f *fakeTS) get(context.Context) (time.Time, error) {
f.getCalled = true
t := time.Now().Add(-wantEntries * 24 * time.Hour).UTC()
return t, nil
}

func (f *fakeTS) put(context.Context, time.Time) error {
f.putCalled = true
return nil
}

func TestNewModules(t *testing.T) {
ic := fakeIC{}
ts := &fakeTS{}
entries, err := newModules(context.Background(), ic, ts, "cloud.google.com")
if err != nil {
t.Fatalf("newModules got err: %v", err)
}
if got, want := len(entries), wantEntries; got != want {
t.Errorf("newModules got %d entries, want %d", got, want)
}
if !ts.getCalled {
t.Errorf("fakeTS.get was never called")
}
if !ts.putCalled {
t.Errorf("fakeTS.put was never called")
}
}

0 comments on commit 49f497e

Please sign in to comment.