From e7573ff6bf83a73af6f8bd3bd31a7941ec3501a8 Mon Sep 17 00:00:00 2001 From: Heschi Kreinick Date: Fri, 12 Aug 2022 17:25:05 -0400 Subject: [PATCH] internal/task: start x repo tagging tasks First step: list all our projects/repositories, and select those that have a go.mod labeling them as golang.org/x as candidates to release as the next version, or v0.1.0 for untagged repositories. For golang/go#48523. Change-Id: Ice92319a0726daf3bf5f94581582d8802640dffc Reviewed-on: https://go-review.googlesource.com/c/build/+/425088 Reviewed-by: Dmitri Shuralyov Auto-Submit: Heschi Kreinick Reviewed-by: Dmitri Shuralyov Run-TryBot: Heschi Kreinick TryBot-Result: Gopher Robot --- gerrit/gerrit.go | 63 ++++++++++++++--- go.mod | 1 + go.sum | 3 + internal/relui/buildrelease_test.go | 1 + internal/task/gerrit.go | 26 +++++++ internal/task/tagx.go | 101 ++++++++++++++++++++++++++++ internal/task/tagx_test.go | 35 ++++++++++ 7 files changed, 219 insertions(+), 11 deletions(-) create mode 100644 internal/task/tagx.go create mode 100644 internal/task/tagx_test.go diff --git a/gerrit/gerrit.go b/gerrit/gerrit.go index 810f0d7f5..0fcc3c57a 100644 --- a/gerrit/gerrit.go +++ b/gerrit/gerrit.go @@ -12,11 +12,11 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "sort" @@ -116,11 +116,17 @@ type urlValues url.Values func (urlValues) isDoArg() {} +// respBodyRaw returns the body of the response. If set, dst is ignored. +type respBodyRaw struct{ rc *io.ReadCloser } + +func (respBodyRaw) isDoArg() {} + func (c *Client) do(ctx context.Context, dst interface{}, method, path string, opts ...doArg) error { var arg url.Values - var body io.Reader + var requestBody io.Reader var contentType string var wantStatus = http.StatusOK + var responseBody *io.ReadCloser for _, opt := range opts { switch opt := opt.(type) { case wantResStatus: @@ -130,13 +136,15 @@ func (c *Client) do(ctx context.Context, dst interface{}, method, path string, o if err != nil { return err } - body = bytes.NewReader(b) + requestBody = bytes.NewReader(b) contentType = "application/json" case reqBodyRaw: - body = opt.r + requestBody = opt.r contentType = "application/octet-stream" case urlValues: arg = url.Values(opt) + case respBodyRaw: + responseBody = opt.rc default: panic(fmt.Sprintf("internal error; unsupported type %T", opt)) } @@ -148,12 +156,11 @@ func (c *Client) do(ctx context.Context, dst interface{}, method, path string, o if _, ok := c.auth.(noAuth); ok { slashA = "" } - var err error u := c.url + slashA + path if arg != nil { u += "?" + arg.Encode() } - req, err := http.NewRequestWithContext(ctx, method, u, body) + req, err := http.NewRequestWithContext(ctx, method, u, requestBody) if err != nil { return err } @@ -167,16 +174,27 @@ func (c *Client) do(ctx context.Context, dst interface{}, method, path string, o if err != nil { return err } - defer res.Body.Close() + defer func() { + if responseBody != nil && *responseBody != nil { + // We've handed off the body to the user. + return + } + res.Body.Close() + }() if res.StatusCode != wantStatus { - body, err := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10)) + body, err := io.ReadAll(io.LimitReader(res.Body, 4<<10)) return &HTTPError{res, body, err} } + if responseBody != nil { + *responseBody = res.Body + return nil + } + if dst == nil { // Drain the response body, return an error if it's anything but empty. - body, err := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10)) + body, err := io.ReadAll(io.LimitReader(res.Body, 4<<10)) if err != nil || len(body) != 0 { return &HTTPError{res, body, err} } @@ -711,7 +729,7 @@ type ProjectInfo struct { // ListProjects returns the server's active projects. // -// The returned slice is sorted by project ID and excludes the "All-Projects" project. +// The returned slice is sorted by project ID and excludes the "All-Projects" and "All-Users" projects. // // See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#list-projects func (c *Client) ListProjects(ctx context.Context) ([]ProjectInfo, error) { @@ -722,12 +740,15 @@ func (c *Client) ListProjects(ctx context.Context) ([]ProjectInfo, error) { } var ret []ProjectInfo for name, pi := range res { - if name == "All-Projects" { + if name == "All-Projects" || name == "All-Users" { continue } if pi.State != "ACTIVE" { continue } + // https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#project-info: + // "name not set if returned in a map where the project name is used as map key" + pi.Name = name ret = append(ret, pi) } sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) @@ -828,6 +849,26 @@ func (c *Client) GetBranch(ctx context.Context, project, branch string) (BranchI return res, err } +// GetFileContent gets a file's contents at a particular commit. +// +// See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-content-from-commit. +func (c *Client) GetFileContent(ctx context.Context, project, commit, path string) (io.ReadCloser, error) { + var body io.ReadCloser + err := c.do(ctx, nil, "GET", fmt.Sprintf("/projects/%s/commits/%s/files/%s/content", project, commit, url.QueryEscape(path)), respBodyRaw{&body}) + if err != nil { + return nil, err + } + return readCloser{ + Reader: base64.NewDecoder(base64.StdEncoding, body), + Closer: body, + }, nil +} + +type readCloser struct { + io.Reader + io.Closer +} + // WebLinkInfo is information about a web link. // See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#web-link-info type WebLinkInfo struct { diff --git a/go.mod b/go.mod index bf8807a8c..f03c2b4e5 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( go4.org v0.0.0-20180809161055-417644f6feb5 golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d + golang.org/x/mod v0.5.1 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 golang.org/x/perf v0.0.0-20220913151710-7c6e287988f3 diff --git a/go.sum b/go.sum index 859c6bae2..cb639f156 100644 --- a/go.sum +++ b/go.sum @@ -946,7 +946,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/relui/buildrelease_test.go b/internal/relui/buildrelease_test.go index 1eb606ceb..dddd85ff5 100644 --- a/internal/relui/buildrelease_test.go +++ b/internal/relui/buildrelease_test.go @@ -562,6 +562,7 @@ type fakeGerrit struct { changesCreated int createdTags map[string]string wantReviewers []string + task.GerritClient } func (g *fakeGerrit) CreateAutoSubmitChange(ctx context.Context, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) { diff --git a/internal/task/gerrit.go b/internal/task/gerrit.go index 8e22eb113..691b5cc41 100644 --- a/internal/task/gerrit.go +++ b/internal/task/gerrit.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "strings" "golang.org/x/build/gerrit" @@ -27,6 +28,10 @@ type GerritClient interface { ListTags(ctx context.Context, project string) ([]string, error) // ReadBranchHead returns the head of a branch in project. ReadBranchHead(ctx context.Context, project, branch string) (string, error) + // ListProjects lists all the projects on the server. + ListProjects(ctx context.Context) ([]string, error) + // ReadFile reads a file from project at the specified commit. + ReadFile(ctx context.Context, project, commit, file string) ([]byte, error) } type RealGerritClient struct { @@ -149,6 +154,27 @@ func (c *RealGerritClient) ReadBranchHead(ctx context.Context, project, branch s return branchInfo.Revision, nil } +func (c *RealGerritClient) ListProjects(ctx context.Context) ([]string, error) { + projects, err := c.Client.ListProjects(ctx) + if err != nil { + return nil, err + } + var names []string + for _, p := range projects { + names = append(names, p.Name) + } + return names, nil +} + +func (c *RealGerritClient) ReadFile(ctx context.Context, project, commit, file string) ([]byte, error) { + body, err := c.Client.GetFileContent(ctx, project, commit, file) + if err != nil { + return nil, err + } + defer body.Close() + return io.ReadAll(body) +} + // ChangeLink returns a link to the review page for the CL with the specified // change ID. The change ID must be in the project~cl# form. func ChangeLink(changeID string) string { diff --git a/internal/task/tagx.go b/internal/task/tagx.go new file mode 100644 index 000000000..07f55d34b --- /dev/null +++ b/internal/task/tagx.go @@ -0,0 +1,101 @@ +package task + +import ( + "errors" + "fmt" + "strings" + + "golang.org/x/build/gerrit" + wf "golang.org/x/build/internal/workflow" + "golang.org/x/mod/modfile" + "golang.org/x/mod/semver" +) + +type TagXReposTasks struct { + Gerrit GerritClient +} + +type TagRepo struct { + Name string + Next string + Deps []string +} + +func (x *TagXReposTasks) SelectRepos(ctx *wf.TaskContext) ([]*TagRepo, error) { + projects, err := x.Gerrit.ListProjects(ctx) + if err != nil { + return nil, err + } + + ctx.Printf("Examining repositories %v", projects) + var repos []*TagRepo + for _, p := range projects { + repo, err := x.readRepo(ctx, p) + if err != nil { + return nil, err + } + if repo != nil { + repos = append(repos, repo) + } + } + return repos, nil +} + +func (x *TagXReposTasks) readRepo(ctx *wf.TaskContext, project string) (*TagRepo, error) { + head, err := x.Gerrit.ReadBranchHead(ctx, project, "master") + if errors.Is(err, gerrit.ErrResourceNotExist) { + ctx.Printf("ignoring %v: no master branch: %v", project, err) + return nil, nil + } + if err != nil { + return nil, err + } + + gomod, err := x.Gerrit.ReadFile(ctx, project, head, "go.mod") + if errors.Is(err, gerrit.ErrResourceNotExist) { + ctx.Printf("ignoring %v: no go.mod: %v", project, err) + return nil, nil + } + if err != nil { + return nil, err + } + mf, err := modfile.ParseLax("go.mod", gomod, nil) + if err != nil { + return nil, err + } + if !strings.HasPrefix(mf.Module.Mod.Path, "golang.org/x") { + ctx.Printf("ignoring %v: not golang.org/x", project) + return nil, nil + } + + tags, err := x.Gerrit.ListTags(ctx, project) + if err != nil { + return nil, err + } + highestRelease := "" + for _, tag := range tags { + if semver.IsValid(tag) && semver.Prerelease(tag) == "" && + (highestRelease == "" || semver.Compare(highestRelease, tag) < 0) { + highestRelease = tag + } + } + nextTag := "v0.1.0" + if highestRelease != "" { + var err error + nextTag, err = nextVersion(highestRelease) + if err != nil { + return nil, fmt.Errorf("couldn't pick next version for %v: %v", project, err) + } + } + + result := &TagRepo{ + Name: project, + Next: nextTag, + } + for _, req := range mf.Require { + if strings.HasPrefix(req.Mod.Path, "golang.org/x") { + result.Deps = append(result.Deps, req.Mod.Path) + } + } + return result, nil +} diff --git a/internal/task/tagx_test.go b/internal/task/tagx_test.go new file mode 100644 index 000000000..28185fb1f --- /dev/null +++ b/internal/task/tagx_test.go @@ -0,0 +1,35 @@ +package task + +import ( + "context" + "flag" + "testing" + + "golang.org/x/build/gerrit" + "golang.org/x/build/internal/workflow" +) + +var flagRunTagXTest = flag.Bool("run-tagx-test", false, "run tag x/ repo test, which is read-only and safe. Must have a Gerrit cookie in gitcookies.") + +func TestSelectReposLive(t *testing.T) { + if !*flagRunTagXTest { + t.Skip("Not enabled by flags") + } + + tasks := &TagXReposTasks{ + Gerrit: &RealGerritClient{ + Client: gerrit.NewClient("https://go-review.googlesource.com", gerrit.GitCookiesAuth()), + }, + } + ctx := &workflow.TaskContext{ + Context: context.Background(), + Logger: &testLogger{t}, + } + repos, err := tasks.SelectRepos(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range repos { + t.Logf("%#v", r) + } +}