From a126e88e7a3d1c2786c588f6881c740f627d346b Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Tue, 14 Nov 2017 11:04:45 -0800 Subject: [PATCH] Golang implementation of release note generating tool. This is a duplication of the shell script "relnotes". Commits have been squashed. Original commits can be found at https://github.com/roycaihw/release/tree/relnotes-dev --- .bazelrc | 5 + BUILD.bazel | 8 + WORKSPACE | 11 + toolbox/relnotes/BUILD.bazel | 27 ++ toolbox/relnotes/README.md | 45 ++ toolbox/relnotes/build.sh | 16 + toolbox/relnotes/main.go | 796 ++++++++++++++++++++++++++++++++++ toolbox/relnotes/main_test.go | 175 ++++++++ toolbox/util/BUILD.bazel | 27 ++ toolbox/util/common.go | 70 +++ toolbox/util/common_test.go | 27 ++ toolbox/util/github.go | 359 +++++++++++++++ toolbox/util/github_test.go | 229 ++++++++++ toolbox/util/gitlib.go | 52 +++ toolbox/util/gitlib_test.go | 25 ++ 15 files changed, 1872 insertions(+) create mode 100644 .bazelrc create mode 100644 BUILD.bazel create mode 100644 toolbox/relnotes/BUILD.bazel create mode 100644 toolbox/relnotes/README.md create mode 100755 toolbox/relnotes/build.sh create mode 100644 toolbox/relnotes/main.go create mode 100644 toolbox/relnotes/main_test.go create mode 100644 toolbox/util/BUILD.bazel create mode 100644 toolbox/util/common.go create mode 100644 toolbox/util/common_test.go create mode 100644 toolbox/util/github.go create mode 100644 toolbox/util/github_test.go create mode 100644 toolbox/util/gitlib.go create mode 100644 toolbox/util/gitlib_test.go diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000000..b571784c507 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,5 @@ +# TODO: Remove this default once rules_go is bumped. +# Ref kubernetes/kubernetes#52677 +build --incompatible_comprehension_variables_do_not_leak=false +# TODO: remove the following once repo-infra is bumped. +build --incompatible_disallow_set_constructor=false diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 00000000000..6b44223e0c1 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,8 @@ +load("@io_bazel_rules_go//go:def.bzl", "gazelle") + +gazelle( + name = "gazelle", + prefix = "k8s.io/release", + external = "vendored", +) + diff --git a/WORKSPACE b/WORKSPACE index f0a53cae470..27060be6a76 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1 +1,12 @@ workspace(name = "io_kubernetes_build") +http_archive( + name = "io_bazel_rules_go", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.6.0/rules_go-0.6.0.tar.gz", + sha256 = "ba6feabc94a5d205013e70792accb6cce989169476668fbaf98ea9b342e13b59", +) +load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains") +go_rules_dependencies() +go_register_toolchains() + +load("@io_bazel_rules_go//proto:def.bzl", "proto_register_toolchains") +proto_register_toolchains() diff --git a/toolbox/relnotes/BUILD.bazel b/toolbox/relnotes/BUILD.bazel new file mode 100644 index 00000000000..b974f0c107c --- /dev/null +++ b/toolbox/relnotes/BUILD.bazel @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "k8s.io/release/toolbox/relnotes", + visibility = ["//visibility:private"], + deps = [ + "//toolbox/util:go_default_library", + "//vendor/github.com/google/go-github/github:go_default_library", + ], +) + +go_binary( + name = "relnotes", + importpath = "k8s.io/release/toolbox/relnotes", + library = ":go_default_library", + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["main_test.go"], + importpath = "k8s.io/release/toolbox/relnotes", + library = ":go_default_library", + deps = ["//toolbox/util:go_default_library"], +) diff --git a/toolbox/relnotes/README.md b/toolbox/relnotes/README.md new file mode 100644 index 00000000000..660555977ee --- /dev/null +++ b/toolbox/relnotes/README.md @@ -0,0 +1,45 @@ +# Release Notes Collector + +This is a Golang implementation of existing release note collector +[relnotes](https://github.com/kubernetes/release/blob/master/relnotes). + +Golang requires this repo being placed within $GOPATH, explicitly at +$GOPATH/src/k8s.io/release. + +This tool also uses [dep](https://github.com/golang/dep) to manage Golang +dependencies. To install dep, follow the instructions in [dep's Github +page](https://github.com/golang/dep). + +**To bulid, run the build script:** + +`./build.sh` + +**Or do it manually:** + +`cd $GOPATH/src/k8s.io/release` + +`dep ensure` + +`bazel run //:gazelle` + +`bazel build toolbox/relnotes:relnotes` + +**Some example command gathering release notes for Kubernetes (assume currently in +a kubernetes repo):** + +* (On branch release-1.7:) + +`../release/bazel-bin/toolbox/relnotes/relnotes --preview --htmlize-md +--html-file /tmp/release-note-html-testfile +--release-tars=_output/release-tars v1.7.0..v1.7.2` + +* (On branch release-1.7:) + +`../release/bazel-bin/toolbox/relnotes/relnotes --preview --html-file +/tmp/release-note-html-testfile --release-tars=_output/release-tars +v1.7.0..v1.7.0` + +* (On branch release-1.6.3:) + +`../release/bazel-bin/toolbox/relnotes/relnotes --html-file +/tmp/release-note-html-testfile --full` diff --git a/toolbox/relnotes/build.sh b/toolbox/relnotes/build.sh new file mode 100755 index 00000000000..912c9919d14 --- /dev/null +++ b/toolbox/relnotes/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# This is a build script for Golang relnotes (release note collector), see +# README.md for more information. + +# At the root directory +cd $GOPATH/src/k8s.io/release + +# Run dep to get the dependencies +dep ensure + +# Run gazelle to auto-generate BUILD files +bazel run //:gazelle + +# Build the program +bazel build toolbox/relnotes:relnotes diff --git a/toolbox/relnotes/main.go b/toolbox/relnotes/main.go new file mode 100644 index 00000000000..abc55a90173 --- /dev/null +++ b/toolbox/relnotes/main.go @@ -0,0 +1,796 @@ +// Copyright 2017 The Kubernetes Authors All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/google/go-github/github" + u "k8s.io/release/toolbox/util" +) + +const ( + k8sReleaseURLPrefix = "https://dl.k8s.io" + verDotzero = "dotzero" +) + +var ( + // Flags + // TODO: golang flags and parameters syntax + branch = flag.String("branch", "", "Specify a branch other than the current one") + documentURL = flag.String("doc-url", "https://docs.k8s.io", "Documentation URL displayed in release notes") + exampleURLPrefix = flag.String("example-url-prefix", "https://releases.k8s.io/", "Example URL prefix displayed in release notes") + full = flag.Bool("full", false, "Force 'full' release format to show all sections of release notes. "+ + "(This is the *default* for new branch X.Y.0 notes)") + githubToken = flag.String("github-token", "", "The file that contains Github token. Must be specified, or set the GITHUB_TOKEN environment variable.") + htmlFileName = flag.String("html-file", "", "Produce a html version of the notes") + htmlizeMD = flag.Bool("htmlize-md", false, "Output markdown with html for PRs and contributors (for use in CHANGELOG.md)") + mdFileName = flag.String("markdown-file", "", "Specify an alt file to use to store notes") + owner = flag.String("owner", "kubernetes", "Github owner or organization") + preview = flag.Bool("preview", false, "Report additional branch statistics (used for reporting outside of releases)") + quiet = flag.Bool("quiet", false, "Don't display the notes when done") + releaseBucket = flag.String("release-bucket", "kubernetes-release", "Specify Google Storage bucket to point to in generated notes (informational only)") + releaseTars = flag.String("release-tars", "", "Directory of tars to sha256 sum for display") + repo = flag.String("repo", "kubernetes", "Github repository") + + // Global + branchHead = "" + branchVerSuffix = "" // e.g. branch: "release-1.8", branchVerSuffix: "-1.8" +) + +// ReleaseInfo contains release related information to generate a release note. +// NOTE: the prMap only includes PRs with "release-note" label. +type ReleaseInfo struct { + startTag, releaseTag string + prMap map[int]*github.Issue + releasePRs []int + releaseActionRequiredPRs []int +} + +func main() { + // Initialization + flag.Parse() + branchRange := flag.Arg(0) + startingTime := time.Now().Round(time.Second) + + log.Printf("Boolean flags: full: %v, htmlize-md: %v, preview: %v, quiet: %v", *full, *htmlizeMD, *preview, *quiet) + log.Printf("Input branch range: %s", branchRange) + + if *branch == "" { + // If branch isn't specified in flag, use current branch + var err error + *branch, err = u.GetCurrentBranch() + if err != nil { + log.Printf("failed to get current branch: %v", err) + os.Exit(1) + } + } + branchVerSuffix = strings.TrimPrefix(*branch, "release") + log.Printf("Working branch: %s. Branch version suffix: %s.", *branch, branchVerSuffix) + + prFileName := fmt.Sprintf("/tmp/release-notes-%s-prnotes", *branch) + if *mdFileName == "" { + *mdFileName = fmt.Sprintf("/tmp/release-notes-%s.md", *branch) + } + log.Printf("Output markdown file path: %s", *mdFileName) + if *htmlFileName != "" { + log.Printf("Output HTML file path: %s", *htmlFileName) + } + + if *githubToken == "" { + // If githubToken isn't specified in flag, use the GITHUB_TOKEN environment variable + *githubToken = os.Getenv("GITHUB_TOKEN") + } else { + token, err := u.ReadToken(*githubToken) + if err != nil { + log.Printf("failed to read Github token: %v", err) + os.Exit(1) + } + *githubToken = token + } + // Github token must be provided to ensure great rate limit experience + if *githubToken == "" { + log.Print("Github token not provided. Exiting now...") + os.Exit(1) + } + client := u.NewClient(*githubToken) + + // End of initialization + + // Gather release related information including startTag, releaseTag, prMap and releasePRs + releaseInfo, err := gatherReleaseInfo(client, branchRange) + if err != nil { + log.Printf("failed to gather release related information: %v", err) + os.Exit(1) + } + + // Generating release note... + log.Print("Generating release notes...") + err = gatherPRNotes(prFileName, releaseInfo) + if err != nil { + log.Printf("failed to gather PR notes: %v", err) + os.Exit(1) + } + + // Start generating markdown file + log.Print("Preparing layout...") + err = generateMDFile(client, releaseInfo.releaseTag, prFileName) + if err != nil { + log.Printf("failed to generate markdown file: %v", err) + os.Exit(1) + } + + if *htmlizeMD && !u.IsVer(releaseInfo.releaseTag, verDotzero) { + // HTML-ize markdown file + // Make users and PRs linkable + // Also, expand anchors (needed for email announce()) + projectGithubURL := fmt.Sprintf("https://github.com/%s/%s", *owner, *repo) + _, err = u.Shell("sed", "-i", "-e", "s,#\\([0-9]\\{5\\,\\}\\),[#\\1]("+projectGithubURL+"/pull/\\1),g", + "-e", "s,\\(#v[0-9]\\{3\\}-\\),"+projectGithubURL+"/blob/master/CHANGELOG"+branchVerSuffix+".md\\1,g", + "-e", "s,@\\([a-zA-Z0-9-]*\\),[@\\1](https://github.com/\\1),g", *mdFileName) + + if err != nil { + log.Printf("failed to htmlize markdown file: %v", err) + os.Exit(1) + } + } + + if *preview && *owner == "kubernetes" && *repo == "kubernetes" { + // If in preview mode, get the current CI job status + // We do this after htmlizing because we don't want to update the + // issues in the block of this section + // + // NOTE: this function is Kubernetes-specified and runs the find_green_build script under + // kubernetes/release. Make sure you have the dependencies installed for find_green_build + // before running this function. + err = getCIJobStatus(*mdFileName, *branch, *htmlizeMD) + if err != nil { + log.Printf("failed to get CI status: %v", err) + os.Exit(1) + } + } + + if *htmlFileName != "" { + // If HTML file name is given, generate HTML release note + err = createHTMLNote(*htmlFileName, *mdFileName) + if err != nil { + log.Printf("failed to generate HTML release note: %v", err) + os.Exit(1) + } + } + + if !*quiet { + // If --quiet flag is not specified, print the markdown release note to stdout + log.Print("Displaying the markdown release note to stdout...") + dat, err := ioutil.ReadFile(*mdFileName) + if err != nil { + log.Printf("failed to read markdown release note: %v", err) + os.Exit(1) + } + fmt.Print(string(dat)) + } + + log.Printf("Successfully generated release note. Total running time: %s", time.Now().Round(time.Second).Sub(startingTime).String()) + + return +} + +func gatherReleaseInfo(g *u.GithubClient, branchRange string) (*ReleaseInfo, error) { + var info ReleaseInfo + log.Print("Gathering release commits from Github...") + // Get release related commits on the release branch within release range + releaseCommits, startTag, releaseTag, err := getReleaseCommits(g, *owner, *repo, *branch, branchRange) + if err != nil { + return nil, fmt.Errorf("failed to get release commits for %s: %v", branchRange, err) + } + info.startTag = startTag + info.releaseTag = releaseTag + + // Parse release related PR ids from the release commits + commitPRs, err := parsePRFromCommit(releaseCommits) + if err != nil { + return nil, fmt.Errorf("failed to parse release commits: %v", err) + } + + log.Print("Gathering \"release-note\" labelled PRs using Github search API. This may take a while...") + var query []string + query = u.AddQuery(query, "repo", *owner, "/", *repo) + query = u.AddQuery(query, "type", "pr") + query = u.AddQuery(query, "label", "release-note") + releaseNotePRs, err := g.SearchIssues(strings.Join(query, " ")) + if err != nil { + return nil, fmt.Errorf("failed to search release-note labelled PRs: %v", err) + } + log.Print("\"release-note\" labelled PRs gathered.") + + log.Print("Gathering \"release-note-action-required\" labelled PRs using Github search API.") + query = nil + query = u.AddQuery(query, "repo", *owner, "/", *repo) + query = u.AddQuery(query, "type", "pr") + query = u.AddQuery(query, "label", "release-note-action-required") + releaseNoteActionRequiredPRs, err := g.SearchIssues(strings.Join(query, " ")) + if err != nil { + return nil, fmt.Errorf("failed to search release-note-action-required labelled PRs: %v", err) + } + log.Print("\"release-note-action-required\" labelled PRs gathered.") + + info.prMap = make(map[int]*github.Issue) + for _, i := range releaseNotePRs { + ptr := new(github.Issue) + *ptr = i + info.prMap[*ptr.Number] = ptr + } + + actionRequiredPRMap := make(map[int]*github.Issue) + for _, i := range releaseNoteActionRequiredPRs { + ptr := new(github.Issue) + *ptr = i + actionRequiredPRMap[*ptr.Number] = ptr + } + + // Get release note PRs by examining release-note label on commit PRs + info.releasePRs = make([]int, 0) + for _, pr := range commitPRs { + if info.prMap[pr] != nil { + info.releasePRs = append(info.releasePRs, pr) + } + if actionRequiredPRMap[pr] != nil { + info.releaseActionRequiredPRs = append(info.releaseActionRequiredPRs, pr) + } + } + + for k, v := range actionRequiredPRMap { + info.prMap[k] = v + } + + return &info, nil +} + +func gatherPRNotes(prFileName string, info *ReleaseInfo) error { + var result error + prFile, err := os.Create(prFileName) + if err != nil { + return fmt.Errorf("failed to create release note file %s: %v", prFileName, err) + } + defer func() { + if err = prFile.Close(); err != nil { + result = fmt.Errorf("failed to close file %s, %v", prFileName, err) + } + }() + + // Bootstrap notes for minor (new branch) releases + if *full || u.IsVer(info.releaseTag, verDotzero) { + draftURL := fmt.Sprintf("%s%s/features/master/%s/release-notes-draft.md", u.GithubRawURL, *owner, *branch) + changelogURL := fmt.Sprintf("%s%s/%s/master/CHANGELOG%s.md", u.GithubRawURL, *owner, *repo, branchVerSuffix) + minorRelease(prFile, info.releaseTag, draftURL, changelogURL) + } else { + patchRelease(prFile, info) + } + return result +} + +func generateMDFile(g *u.GithubClient, releaseTag, prFileName string) error { + var result error + mdFile, err := os.Create(*mdFileName) + if err != nil { + return fmt.Errorf("failed to create release note markdown file %s: %v", *mdFileName, err) + } + defer func() { + if err = mdFile.Close(); err != nil { + result = fmt.Errorf("failed to close file %s, %v", *mdFileName, err) + } + }() + + // Create markdown file body with documentation and example URLs from program flags + exampleURL := fmt.Sprintf("%s%s/examples", *exampleURLPrefix, *branch) + err = createBody(mdFile, releaseTag, *branch, *documentURL, exampleURL, *releaseTars) + if err != nil { + return fmt.Errorf("failed to create file body: %v", err) + } + + // Copy (append) the pull request notes into the output markdown file + dat, err := ioutil.ReadFile(prFileName) + if err != nil { + return fmt.Errorf("failed to copy from PR file to release note markdown file: %v", err) + } + mdFile.WriteString(string(dat)) + + if *preview { + // If in preview mode, get the pending PRs + err = getPendingPRs(g, mdFile, *owner, *repo, *branch) + if err != nil { + return fmt.Errorf("failed to get pending PRs: %v", err) + } + } + return result +} + +// getPendingPRs gets pending PRs on given branch in the repo. +func getPendingPRs(g *u.GithubClient, f *os.File, owner, repo, branch string) error { + log.Print("Getting pending PR status...") + f.WriteString("-------\n") + f.WriteString(fmt.Sprintf("## PENDING PRs on the %s branch\n", branch)) + + if *htmlizeMD { + f.WriteString("PR | Milestone | User | Date | Commit Message\n") + f.WriteString("-- | --------- | ---- | ---- | --------------\n") + } + + var query []string + query = u.AddQuery(query, "repo", owner, "/", repo) + query = u.AddQuery(query, "is", "open") + query = u.AddQuery(query, "type", "pr") + query = u.AddQuery(query, "base", branch) + pendingPRs, err := g.SearchIssues(strings.Join(query, " ")) + if err != nil { + return fmt.Errorf("failed to search pending PRs: %v", err) + } + + for _, pr := range pendingPRs { + var str string + // escape '*' in commit messages so they don't mess up formatting + msg := strings.Replace(*pr.Title, "*", "", -1) + milestone := "null" + if pr.Milestone != nil { + milestone = *pr.Milestone.Title + } + if *htmlizeMD { + str = fmt.Sprintf("#%-8d | %-4s | @%-10s| %s | %s\n", *pr.Number, milestone, *pr.User.Login, pr.UpdatedAt.Format("Mon Jan 2 15:04:05 MST 2006"), msg) + } else { + str = fmt.Sprintf("#%-8d %-4s @%-10s %s %s\n", *pr.Number, milestone, *pr.User.Login, pr.UpdatedAt.Format("Mon Jan 2 15:04:05 MST 2006"), msg) + } + f.WriteString(str) + } + f.WriteString("\n\n") + return nil +} + +// createHTMLNote generates HTML release note based on the input markdown release note. +func createHTMLNote(htmlFileName, mdFileName string) error { + var result error + log.Print("Generating HTML release note...") + cssFileName := "/tmp/release_note_cssfile" + cssFile, err := os.Create(cssFileName) + if err != nil { + return fmt.Errorf("failed to create css file %s: %v", cssFileName, err) + } + + cssFile.WriteString("") + // Here we manually close the css file instead of defer the close function, + // because we need to use the css file for pandoc command below. + // Writing to css file is a clear small logic so we don't separate it into + // another function. + if err = cssFile.Close(); err != nil { + return fmt.Errorf("failed to close file %s, %v", cssFileName, err) + } + + htmlStr, err := u.Shell("pandoc", "-H", cssFileName, "--from", "markdown_github", "--to", "html", mdFileName) + if err != nil { + return fmt.Errorf("failed to generate html content: %v", err) + } + + htmlFile, err := os.Create(htmlFileName) + if err != nil { + return fmt.Errorf("failed to create html file: %v", err) + } + defer func() { + if err = htmlFile.Close(); err != nil { + result = fmt.Errorf("failed to close file %s, %v", htmlFileName, err) + } + }() + + htmlFile.WriteString(htmlStr) + return result +} + +// getCIJobStatus runs the script find_green_build and append CI job status to outputFile. +// NOTE: this function is Kubernetes-specified and runs the find_green_build script under +// kubernetes/release. Make sure you have the dependencies installed for find_green_build +// before running this function. +func getCIJobStatus(outputFile, branch string, htmlize bool) error { + var result error + log.Print("Getting CI job status (this may take a while)...") + + red := "" + green := "" + off := "" + + if htmlize { + red = "" + green = "" + off = "" + } + + var extraFlag string + + if strings.Contains(branch, "release-") { + // If working on a release branch assume --official for the purpose of displaying + // find_green_build output + extraFlag = "--official" + } else { + // For master branch, limit the analysis to 30 primary ci jobs. This is necessary + // due to the recently expanded blocking test list for master. The expanded test + // list is often unable to find a complete passing set and find_green_build runs + // unbounded for hours + extraFlag = "--limit=30" + } + + f, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer func() { + if err = f.Close(); err != nil { + result = fmt.Errorf("failed to close file %s, %v", outputFile, err) + } + }() + + f.WriteString(fmt.Sprintf("## State of %s branch\n", branch)) + + // Call script find_green_build to get CI job status + content, err := u.Shell(os.Getenv("GOPATH")+"/src/k8s.io/release/find_green_build", "-v", extraFlag, branch) + if err == nil { + f.WriteString(fmt.Sprintf("%sGOOD TO GO!%s\n\n", green, off)) + } else { + f.WriteString(fmt.Sprintf("%sNOT READY%s\n\n", red, off)) + } + + f.WriteString("### Details\n```\n") + f.WriteString(content) + f.WriteString("```\n") + + log.Print("CI job status fetched.") + return result +} + +// createBody creates the general documentation, example and downloads table body for the +// markdown file. +func createBody(f *os.File, releaseTag, branch, docURL, exampleURL, releaseTars string) error { + var title string + if *preview { + title = "Branch " + } + + if releaseTag == "HEAD" || releaseTag == branchHead { + title += branch + } else { + title += releaseTag + } + + if *preview { + f.WriteString(fmt.Sprintf("**Release Note Preview - generated on %s**\n", time.Now().Format("Mon Jan 2 15:04:05 MST 2006"))) + } + + f.WriteString(fmt.Sprintf("\n# %s\n\n", title)) + f.WriteString(fmt.Sprintf("[Documentation](%s) & [Examples](%s)\n\n", docURL, exampleURL)) + + if releaseTars != "" { + f.WriteString(fmt.Sprintf("## Downloads for %s\n\n", title)) + tables := []struct { + heading string + filename []string + }{ + {"", []string{releaseTars + "/kubernetes.tar.gz", releaseTars + "/kubernetes-src.tar.gz"}}, + {"Client Binaries", []string{releaseTars + "/kubernetes-client*.tar.gz"}}, + {"Server Binaries", []string{releaseTars + "/kubernetes-server*.tar.gz"}}, + {"Node Binaries", []string{releaseTars + "/kubernetes-node*.tar.gz"}}, + } + + for _, table := range tables { + err := createDownloadsTable(f, releaseTag, table.heading, table.filename...) + if err != nil { + return fmt.Errorf("failed to create downloads table: %v", err) + } + } + f.WriteString("\n") + } + return nil +} + +// createDownloadTable creates table of download link and sha256 hash for given file. +func createDownloadsTable(f *os.File, releaseTag, heading string, filename ...string) error { + var urlPrefix string + + if *releaseBucket == "kubernetes-release" { + urlPrefix = k8sReleaseURLPrefix + } else { + urlPrefix = fmt.Sprintf("https://storage.googleapis.com/%s/release", *releaseBucket) + } + + if *releaseBucket == "" { + log.Print("NOTE: empty Google Storage bucket specified. Please specify valid bucket using \"release-bucket\" flag.") + } + + if heading != "" { + f.WriteString(fmt.Sprintf("\n### %s\n", heading)) + } + + f.WriteString("\n") + f.WriteString("filename | sha256 hash\n") + f.WriteString("-------- | -----------\n") + + files := make([]string, 0) + for _, name := range filename { + fs, _ := filepath.Glob(name) + for _, v := range fs { + files = append(files, v) + } + } + + for _, file := range files { + fn := filepath.Base(file) + sha, err := u.GetSha256(file) + if err != nil { + return fmt.Errorf("failed to calc SHA256 of file %s: %v", file, err) + } + f.WriteString(fmt.Sprintf("[%s](%s/%s/%s) | `%s`\n", fn, urlPrefix, releaseTag, fn, sha)) + } + return nil +} + +// minorReleases performs a minor (vX.Y.0) release by fetching the release template and aggregate +// previous release in series. +func minorRelease(f *os.File, release, draftURL, changelogURL string) { + // Check for draft and use it if available + log.Printf("Checking if draft release notes exist for %s...", release) + + resp, err := http.Get(draftURL) + if err == nil { + defer resp.Body.Close() + } + + if err == nil && resp.StatusCode == 200 { + log.Print("Draft found - using for release notes...") + _, err = io.Copy(f, resp.Body) + if err != nil { + log.Printf("error during copy to file: %v", err) + return + } + f.WriteString("\n") + } else { + log.Print("Failed to find draft - creating generic template... (error message/status code printed below)") + if err != nil { + log.Printf("Error message: %v", err) + } else { + log.Printf("Response status code: %d", resp.StatusCode) + } + f.WriteString("## Major Themes\n\n* TBD\n\n## Other notable improvements\n\n* TBD\n\n## Known Issues\n\n* TBD\n\n## Provider-specific Notes\n\n* TBD\n\n") + } + + // Aggregate all previous release in series + f.WriteString(fmt.Sprintf("### Previous Release Included in %s\n\n", release)) + + // Regexp Example: + // Assume the release tag is v1.7.0, this regexp matches "- [v1.7.0-" in + // "- [v1.7.0-rc.1](#v170-rc1)" + // "- [v1.7.0-beta.2](#v170-beta2)" + // "- [v1.7.0-alpha.3](#v170-alpha3)" + reAnchor, _ := regexp.Compile(fmt.Sprintf("- \\[%s-", release)) + + resp, err = http.Get(changelogURL) + if err == nil { + defer resp.Body.Close() + } + + if err == nil && resp.StatusCode == 200 { + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + for _, line := range strings.Split(buf.String(), "\n") { + if anchor := reAnchor.FindStringSubmatch(line); anchor != nil { + f.WriteString(line + "\n") + } + } + f.WriteString("\n") + } else { + log.Print("Failed to fetch past changelog for minor release - continuing... (error message/status code printed below)") + if err != nil { + log.Printf("Error message: %v", err) + } else { + log.Printf("Response status code: %d", resp.StatusCode) + } + } +} + +// patchRelease performs a patch (vX.Y.Z) release by printing out all the related changes. +func patchRelease(f *os.File, info *ReleaseInfo) { + // Release note for different labels + f.WriteString(fmt.Sprintf("## Changelog since %s\n\n", info.startTag)) + + if len(info.releaseActionRequiredPRs) > 0 { + f.WriteString("### Action Required\n\n") + for _, pr := range info.releaseActionRequiredPRs { + f.WriteString(fmt.Sprintf("* %s (#%d, @%s)\n", extractReleaseNoteFromPR(info.prMap[pr]), pr, *info.prMap[pr].User.Login)) + } + f.WriteString("\n") + } + + if len(info.releasePRs) > 0 { + f.WriteString("### Other notable changes\n\n") + for _, pr := range info.releasePRs { + f.WriteString(fmt.Sprintf("* %s (#%d, @%s)\n", extractReleaseNoteFromPR(info.prMap[pr]), pr, *info.prMap[pr].User.Login)) + } + f.WriteString("\n") + } else { + f.WriteString("**No notable changes for this release**\n\n") + } +} + +// extractReleaseNoteFromPR tries to fetch release note from PR body, otherwise uses PR title. +func extractReleaseNoteFromPR(pr *github.Issue) string { + // Regexp Example: + // This regexp matches the release note section in Kubernetes pull request template: + // https://github.com/kubernetes/kubernetes/blob/master/.github/PULL_REQUEST_TEMPLATE.md + re, _ := regexp.Compile("```release-note\r\n(.+)\r\n```") + if note := re.FindStringSubmatch(*pr.Body); note != nil { + return note[1] + } + return *pr.Title +} + +// determineRange examines a Git branch range in the format of [[startTag..]endTag], and +// determines a valid range. For example: +// +// "" - last release to HEAD on the branch +// "v1.1.4.." - v1.1.4 to HEAD +// "v1.1.4..v1.1.7" - v1.1.4 to v1.1.7 +// "v1.1.7" - last release on the branch to v1.1.7 +// +// NOTE: the input branch must be the corresponding release branch w.r.t. input range. For example: +// +// Getting "v1.1.4..v1.1.7" on branch "release-1.1" makes sense +// Getting "v1.1.4..v1.1.7" on branch "release-1.2" doesn't +func determineRange(g *u.GithubClient, owner, repo, branch, branchRange string) (startTag, releaseTag string, err error) { + b, _, err := g.GetBranch(context.Background(), owner, repo, branch) + if err != nil { + return "", "", err + } + branchHead = *b.Commit.SHA + + lastRelease, err := g.LastReleases(owner, repo) + if err != nil { + return "", "", err + } + + // If lastRelease[branch] is unset, attempt to get the last release from the parent branch + // and then master + if i := strings.LastIndex(branch, "."); lastRelease[branch] == "" && i != -1 { + lastRelease[branch] = lastRelease[branch[:i]] + } + if lastRelease[branch] == "" { + lastRelease[branch] = lastRelease["master"] + } + + // Regexp Example: + // This regexp matches the Git branch range in the format of [[startTag..]endTag]. For example: + // + // "" + // "v1.1.4.." + // "v1.1.4..v1.1.7" + // "v1.1.7" + re, _ := regexp.Compile("([v0-9.]*-*(alpha|beta|rc)*\\.*[0-9]*)\\.\\.([v0-9.]*-*(alpha|beta|rc)*\\.*[0-9]*)$") + tags := re.FindStringSubmatch(branchRange) + if tags != nil { + startTag = tags[1] + releaseTag = tags[3] + } else { + startTag = lastRelease[branch] + releaseTag = branchHead + } + + if startTag == "" { + return "", "", fmt.Errorf("unable to set beginning of range automatically") + } + if releaseTag == "" { + releaseTag = branchHead + } + + return startTag, releaseTag, nil +} + +// getReleaseCommits given a Git branch range in the format of [[startTag..]endTag], determines +// a valid range and returns all the commits on the branch in that range. +func getReleaseCommits(g *u.GithubClient, owner, repo, branch, branchRange string) ([]*github.RepositoryCommit, string, string, error) { + // Get start and release tag/commit based on input branch range + startTag, releaseTag, err := determineRange(g, owner, repo, branch, branchRange) + if err != nil { + return nil, "", "", fmt.Errorf("failed to determine branch range: %v", err) + } + + // Get all tags in the repository + tags, err := g.ListAllTags(owner, repo) + if err != nil { + return nil, "", "", fmt.Errorf("failed to fetch repo tags: %v", err) + } + + // Get commits for specified branch and range + tStart, err := g.GetCommitDate(owner, repo, startTag, tags) + if err != nil { + return nil, "", "", fmt.Errorf("failed to get start commit date for %s: %v", startTag, err) + } + tEnd, err := g.GetCommitDate(owner, repo, releaseTag, tags) + if err != nil { + return nil, "", "", fmt.Errorf("failed to get release commit date for %s: %v", releaseTag, err) + } + + releaseCommits, err := g.ListAllCommits(owner, repo, branch, tStart, tEnd) + if err != nil { + return nil, "", "", fmt.Errorf("failed to fetch release repo commits: %v", err) + } + + return releaseCommits, startTag, releaseTag, nil +} + +// parsePRFromCommit goes through commit messages, and parse PR IDs for normal pull requests as +// well as cherry picks. +func parsePRFromCommit(commits []*github.RepositoryCommit) ([]int, error) { + prs := make([]int, 0) + prsMap := make(map[int]bool) + + // Regexp example: + // This regexp matches (Note that it supports multiple-source cherry pick) + // + // "automated-cherry-pick-of-#12345-#23412-" + // "automated-cherry-pick-of-#23791-" + reCherry, _ := regexp.Compile("automated-cherry-pick-of-(#[0-9]+-){1,}") + reCherryID, _ := regexp.Compile("#([0-9]+)-") + reMerge, _ := regexp.Compile("^Merge pull request #([0-9]+) from") + + for _, c := range commits { + // Deref all PRs back to master + // Match cherry pick PRs first and then normal pull requests + // Paying special attention to automated cherrypicks that could have multiple + // sources + if cpStr := reCherry.FindStringSubmatch(*c.Commit.Message); cpStr != nil { + cpPRs := reCherryID.FindAllStringSubmatch(cpStr[0], -1) + for _, pr := range cpPRs { + id, err := strconv.Atoi(pr[1]) + if err != nil { + return nil, err + } + if prsMap[id] == false { + prs = append(prs, id) + prsMap[id] = true + } + } + } else if pr := reMerge.FindStringSubmatch(*c.Commit.Message); pr != nil { + id, err := strconv.Atoi(pr[1]) + if err != nil { + return nil, err + } + if prsMap[id] == false { + prs = append(prs, id) + prsMap[id] = true + } + } + } + + return prs, nil +} diff --git a/toolbox/relnotes/main_test.go b/toolbox/relnotes/main_test.go new file mode 100644 index 00000000000..0dd9d5fede3 --- /dev/null +++ b/toolbox/relnotes/main_test.go @@ -0,0 +1,175 @@ +package main + +import ( + "os" + "regexp" + "strconv" + "testing" + + u "k8s.io/release/toolbox/util" +) + +func TestDetermineRange(t *testing.T) { + tables := []struct { + owner string + repo string + branch string + branchRange string + start string + end string + }{ + {"kubernetes", "kubernetes", "release-1.7", "v1.7.0..v1.7.2", "v1.7.0", "v1.7.2"}, + // TODO: bug fix in original script + // {"kubernetes", "kubernetes", "release-1.7", "v1.7.20", "v1.7.8", "v1.7.20"}, + {"kubernetes", "kubernetes", "release-1.7", "v1.7.5..", "v1.7.5", "5adaee21de0c5ed1286a00468e09d866605f85f4"}, + {"kubernetes", "kubernetes", "release-1.7", "", "v1.7.8", "5adaee21de0c5ed1286a00468e09d866605f85f4"}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := u.NewClient(githubToken) + + for _, table := range tables { + s, e, err := determineRange(c, table.owner, table.repo, table.branch, table.branchRange) + if err != nil { + t.Errorf("%v %v: Unexpected error: %v", table.branch, table.branchRange, err) + } + if s != table.start { + t.Errorf("%v %v: Start tag was incorrect, want: %s, got: %s", table.branch, table.branchRange, table.start, s) + } + if e != table.end { + t.Errorf("%v %v: End tag was incorrect, want: %s, got: %s", table.branch, table.branchRange, table.end, e) + } + } +} + +func TestRegExp(t *testing.T) { + tables := []struct { + s string + cp bool + pr bool + numCP []int + numPR int + }{ + {"Merge pull request #53422 from liggitt/automated-cherry-pick-of-#53233-upstream-release-1.8\n\nAutomatic merge from submit-queue.\n\nAutomated cherry pick of #53233\n\nFixes #51899\r\n\r\nCherry pick of #53233 on release-1.8.\r\n\r\n#53233: remove containers of deleted pods once all containers have\r\n\r\n```release-note\r\nFixes a performance issue (#51899) identified in large-scale clusters when deleting thousands of pods simultaneously across hundreds of nodes, by actively removing containers of deleted pods, rather than waiting for periodic garbage collection and batching resulting pod API deletion requests.\r\n```", true, true, []int{53233}, 53422}, + + {"Merge pull request #53448 from liggitt/automated-cherry-pick-of-#53317-upstream-release-1.8\n\nAutomatic merge from submit-queue.\n\nAutomated cherry pick of #53317\n\nCherry pick of #53317 on release-1.8.\n\n#53317: Change default --cert-dir for kubelet to a non-transient", true, true, []int{53317}, 53448}, + + {"Merge pull request #53097 from m1093782566/ipvs-test\n\nAutomatic merge from submit-queue (batch tested with PRs 52768, 51898, 53510, 53097, 53058). If you want to cherry-pick this change to another branch, please follow the instructions here.\n\nRun IPVS proxier UTs everywhere - include !linux platfrom\n\n**What this PR does / why we need it**:\r\n\r\nIPVS proxier UTs should run everywhere, including !linux platfrom, which will help a lot when developing in windows platfrom.\r\n\r\n**Which issue this PR fixes**: \r\n\r\nfixes #53099\r\n\r\n**Special notes for your reviewer**:\r\n\r\n**Release note**:\r\n\r\n```release-note\r\nNONE\r\n```", false, true, []int{0}, 53097}, + + {"Merge pull request #52602 from liggitt/automated-cherry-pick-of-#48394-#43152-upstream-release-1.7\n\nAutomatic merge from submit-queue.\n\nAutomated cherry pick of #48394 #43152\n\nCherry pick of #48394 #43152 on release-1.7.\r\n\r\n#48394: GuaranteedUpdate must write if stored data is not canonical\r\n#43152: etcd3 store: retry w/live object on conflict", true, true, []int{48394, 43152}, 52602}, + } + + reCherry, _ := regexp.Compile("automated-cherry-pick-of-(#[0-9]+-){1,}") + reCherryID, _ := regexp.Compile("#([0-9]+)-") + reMerge, _ := regexp.Compile("^Merge pull request #([0-9]+) from") + + for _, table := range tables { + // Check cherry pick regexp + vCP := reCherry.FindStringSubmatch(table.s) + + if table.cp && vCP == nil { + t.Errorf("Cherry pick message not matched: \n\n%s\n\n", table.s) + } + if !table.cp && vCP != nil { + t.Errorf("Non cherry pick message matched: \n\n%s\n\n", table.s) + } + if table.cp && vCP != nil { + vID := reCherryID.FindAllStringSubmatch(vCP[0], -1) + if vID == nil { + t.Errorf("Unexpected empty cherry pick") + } else { + if len(vID) != len(table.numCP) { + t.Errorf("Number of cherry pick PRs mismatch: want: %d, got: %d", len(table.numCP), len(vID)) + } + for idx, i := range vID { + id, err := strconv.Atoi(i[1]) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } else if table.numCP[idx] != id { + t.Errorf("Cherry pick id was incorrect, want: %d, got: %d", table.numCP[idx], id) + } + } + } + } + // Check normal PR regexp + vPR := reMerge.FindStringSubmatch(table.s) + if table.pr && vPR == nil { + t.Errorf("Normal PR message not matched: \n\n%s\n\n", table.s) + } + if !table.pr && vPR != nil { + t.Errorf("Non normal PR message matched: \n\n%s\n\n", table.s) + } + if table.pr && vPR != nil { + id, err := strconv.Atoi(vPR[1]) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } else if table.numPR != id { + t.Errorf("Normal PR id was incorrect, want: %d, got: %d", table.numPR, id) + } + } + } +} + +// NOTE: the following tests are for file content generation tests, and require taking looks at +// the generated files to verify the correctness. +func TestGetCIJobStatus(t *testing.T) { + filename := "/tmp/release_note_CI_status_testfile" + filenameHTML := "/tmp/release_note_CI_status_testfile_html" + + // Remove existing file, because getCIJobStatus() will append on existing file + os.Remove(filename) + os.Remove(filenameHTML) + + err := getCIJobStatus(filename, "release-1.7", false) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + err = getCIJobStatus(filenameHTML, "release-1.7", true) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} + +func TestCreateBody(t *testing.T) { + filename := "/tmp/release_note_body_testfile" + releaseTag := "v1.7.2" + branch := "release-1.7" + docURL := "https://testdoc.com" + exampleURL := "https://testexample.com" + releaseTars := "../../../public_kubernetes/_output/release-tars" + f, err := os.Create(filename) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + createBody(f, releaseTag, branch, docURL, exampleURL, releaseTars) +} + +func TestCreateHTMLNote(t *testing.T) { + htmlFileName := "/tmp/release_note_tests_html_testfile" + mdFileName := "/tmp/relnotes-release-1.7.md" + err := createHTMLNote(htmlFileName, mdFileName) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} + +func TestGetPendingPRs(t *testing.T) { + filename := "/tmp/release_note_pending_pr_testfile" + owner := "kubernetes" + repo := "kubernetes" + branch := "release-1.7" + + f, err := os.Create(filename) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := u.NewClient(githubToken) + + err = getPendingPRs(c, f, owner, repo, branch) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} diff --git a/toolbox/util/BUILD.bazel b/toolbox/util/BUILD.bazel new file mode 100644 index 00000000000..8bf97661a52 --- /dev/null +++ b/toolbox/util/BUILD.bazel @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "common.go", + "github.go", + "gitlib.go", + ], + importpath = "k8s.io/release/toolbox/util", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/google/go-github/github:go_default_library", + "//vendor/golang.org/x/oauth2:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "common_test.go", + "github_test.go", + "gitlib_test.go", + ], + importpath = "k8s.io/release/toolbox/util", + library = ":go_default_library", +) diff --git a/toolbox/util/common.go b/toolbox/util/common.go new file mode 100644 index 00000000000..53632e62c79 --- /dev/null +++ b/toolbox/util/common.go @@ -0,0 +1,70 @@ +// Copyright 2017 The Kubernetes Authors All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "strings" +) + +// Shell runs a command and returns the result as a string. +func Shell(name string, arg ...string) (string, error) { + c := exec.Command(name, arg...) + bytes, err := c.CombinedOutput() + return string(bytes), err +} + +// GetSha256 calculates SHA256 for input file. +func GetSha256(filename string) (string, error) { + f, err := os.Open(filename) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// RenderProgressBar renders a progress bar by rewriting the current (assuming +// stdout outputs to a terminal console). If initRender is true, the function +// writes a new line instead of rewrite the current line. +func RenderProgressBar(progress, total int, duration string, initRender bool) { + barLen := 80 + var progressLen, arrowLen, remainLen int + var rewrite string + + percentage := float64(progress) / float64(total) + + progressLen = int(percentage * float64(barLen)) + if progressLen < barLen { + arrowLen = 1 + } + remainLen = barLen - progressLen - arrowLen + + if !initRender { + rewrite = "\r" + } + + fmt.Printf("%s%12s [%s%s%s] %7.2f%%", rewrite, duration, strings.Repeat("=", progressLen), strings.Repeat(">", arrowLen), strings.Repeat("-", remainLen), percentage*100.0) +} diff --git a/toolbox/util/common_test.go b/toolbox/util/common_test.go new file mode 100644 index 00000000000..27e09c0ca1b --- /dev/null +++ b/toolbox/util/common_test.go @@ -0,0 +1,27 @@ +package util + +import ( + "os" + "testing" +) + +func TestGetSha256(t *testing.T) { + f, err := os.Create("/tmp/sha256_calc_testfile") + if err != nil { + t.Errorf("Unexpected error during creating test file: %v", err) + } + f.WriteString("Hello world.\n") + f.Close() + + result, err := GetSha256("/tmp/sha256_calc_testfile") + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + answer := "6472bf692aaf270d5f9dc40c5ecab8f826ecc92425c8bac4d1ea69bcbbddaea4" + + if result != answer { + t.Errorf("Sha256sum was incorrect, want: %s, got: %s", answer, result) + } + +} diff --git a/toolbox/util/github.go b/toolbox/util/github.go new file mode 100644 index 00000000000..ddf30283faa --- /dev/null +++ b/toolbox/util/github.go @@ -0,0 +1,359 @@ +// Copyright 2017 The Kubernetes Authors All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "regexp" + "strings" + "time" + + "golang.org/x/oauth2" + + "github.com/google/go-github/github" +) + +const ( + // GithubRawURL is the url prefix for getting raw github user content. + GithubRawURL = "https://raw.githubusercontent.com/" +) + +// GithubClient wraps github client with methods in this file. +type GithubClient struct { + client *github.Client + token string +} + +// ReadToken reads Github token from input file. +func ReadToken(filename string) (string, error) { + dat, err := ioutil.ReadFile(filename) + if err != nil { + return "", err + } + return strings.TrimSpace(string(dat)), nil +} + +// NewClient sets up a new github client with input assess token. +func NewClient(githubToken string) *GithubClient { + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: githubToken}, + ) + tc := oauth2.NewClient(ctx, ts) + + return &GithubClient{github.NewClient(tc), githubToken} +} + +// LastReleases looks up the list of releases on github and puts the last release per branch +// into a branch-indexed dictionary. +func (g GithubClient) LastReleases(owner, repo string) (map[string]string, error) { + lastRelease := make(map[string]string) + + r, err := g.ListAllReleases(owner, repo) + if err != nil { + return nil, err + } + + for _, release := range r { + // Skip draft releases + if *release.Draft { + continue + } + // Alpha release goes only on master branch + if strings.Contains(*release.TagName, "-alpha") && lastRelease["master"] == "" { + lastRelease["master"] = *release.TagName + } else { + re, _ := regexp.Compile("v([0-9]+\\.[0-9]+)\\.([0-9]+(-.+)?)") + version := re.FindStringSubmatch(*release.TagName) + + if version != nil { + // Lastest vx.y.0 release goes on both master and release-vx.y branch + if version[2] == "0" && lastRelease["master"] == "" { + lastRelease["master"] = *release.TagName + } + + branchName := "release-" + version[1] + if lastRelease[branchName] == "" { + lastRelease[branchName] = *release.TagName + } + } + } + } + + return lastRelease, nil +} + +// ListAllReleases lists all releases for given owner and repo. +func (g GithubClient) ListAllReleases(owner, repo string) ([]*github.RepositoryRelease, error) { + lo := &github.ListOptions{ + Page: 1, + PerPage: 100, + } + + releases, resp, err := g.client.Repositories.ListReleases(context.Background(), owner, repo, lo) + if err != nil { + return nil, err + } + lo.Page++ + + for lo.Page <= resp.LastPage { + re, _, err := g.client.Repositories.ListReleases(context.Background(), owner, repo, lo) + if err != nil { + return nil, err + } + for _, r := range re { + releases = append(releases, r) + } + lo.Page++ + } + return releases, nil +} + +// ListAllIssues lists all issues and PRs for given owner and repo. +func (g GithubClient) ListAllIssues(owner, repo string) ([]*github.Issue, error) { + // Because gathering all issues from large Github repo is time-consuming, we add a progress bar + // rendering for more user-helpful output. + log.Printf("Gathering all issues from Github for %s/%s. This may take a while...", owner, repo) + + start := time.Now().Round(time.Second) + + lo := &github.ListOptions{ + Page: 1, + PerPage: 100, + } + ilo := &github.IssueListByRepoOptions{ + State: "all", + ListOptions: *lo, + } + + issues, resp, err := g.client.Issues.ListByRepo(context.Background(), owner, repo, ilo) + if err != nil { + return nil, err + } + RenderProgressBar(ilo.ListOptions.Page, resp.LastPage, time.Now().Round(time.Second).Sub(start).String(), true) + + ilo.ListOptions.Page++ + + for ilo.ListOptions.Page <= resp.LastPage { + is, _, err := g.client.Issues.ListByRepo(context.Background(), owner, repo, ilo) + if err != nil { + // New line following the progress bar + fmt.Print("\n") + return nil, err + } + RenderProgressBar(ilo.ListOptions.Page, resp.LastPage, time.Now().Round(time.Second).Sub(start).String(), false) + + for _, i := range is { + issues = append(issues, i) + } + ilo.ListOptions.Page++ + } + // New line following the progress bar + fmt.Print("\n") + log.Print("All issues fetched.") + return issues, nil +} + +// ListAllTags lists all tags for given owner and repo. +func (g GithubClient) ListAllTags(owner, repo string) ([]*github.RepositoryTag, error) { + lo := &github.ListOptions{ + Page: 1, + PerPage: 100, + } + + tags, resp, err := g.client.Repositories.ListTags(context.Background(), owner, repo, lo) + if err != nil { + return nil, err + } + lo.Page++ + + for lo.Page <= resp.LastPage { + ta, _, err := g.client.Repositories.ListTags(context.Background(), owner, repo, lo) + if err != nil { + return nil, err + } + for _, t := range ta { + tags = append(tags, t) + } + lo.Page++ + } + return tags, nil +} + +// ListAllCommits lists all commits for given owner, repo, branch and time range. +func (g GithubClient) ListAllCommits(owner, repo, branch string, start, end time.Time) ([]*github.RepositoryCommit, error) { + lo := &github.ListOptions{ + Page: 1, + PerPage: 100, + } + + clo := &github.CommitsListOptions{ + SHA: branch, + Since: start, + Until: end, + ListOptions: *lo, + } + + commits, resp, err := g.client.Repositories.ListCommits(context.Background(), owner, repo, clo) + if err != nil { + return nil, err + } + clo.ListOptions.Page++ + + for clo.ListOptions.Page <= resp.LastPage { + co, _, err := g.client.Repositories.ListCommits(context.Background(), owner, repo, clo) + if err != nil { + return nil, err + } + for _, commit := range co { + commits = append(commits, commit) + } + clo.ListOptions.Page++ + } + return commits, nil +} + +// GetCommitDate gets commit time for given tag/commit, provided with repository tags and commits. +// The function returns non-nil error if input tag/commit cannot be found in the repository. +func (g GithubClient) GetCommitDate(owner, repo, tagCommit string, tags []*github.RepositoryTag) (time.Time, error) { + sha := tagCommit + // If input string is a tag, convert it into SHA + for _, t := range tags { + if tagCommit == *t.Name { + sha = *t.Commit.SHA + break + } + } + commit, _, err := g.client.Git.GetCommit(context.Background(), owner, repo, sha) + if err != nil { + return time.Time{}, fmt.Errorf("failed to get commit date for SHA %s (original tag/commit %s): %v", sha, tagCommit, err) + } + return *commit.Committer.Date, nil +} + +// HasLabel checks if input github issue contains input label. +func HasLabel(i *github.Issue, label string) bool { + for _, l := range i.Labels { + if *l.Name == label { + return true + } + } + + return false +} + +// SearchIssues gets all issues matching search query. +// NOTE: Github Search API has tight rate limit (30 requests per minute) and only returns the first 1,000 results. +// The function waits if it hits the rate limit, and reconstruct the search query with "created:<=YYYY-MM-DD" to +// search for issues out of the first 1,000 results. +func (g GithubClient) SearchIssues(query string) ([]github.Issue, error) { + issues := make([]github.Issue, 0) + issuesGot := make(map[int]bool) + lastDateGot := "" + totalIssueNumber := 0 + + lo := &github.ListOptions{ + Page: 1, + PerPage: 100, + } + // In order to search for >1,000 result, we use created time as a metric of searching progress. Therefore we + // enforce "Sort:created" and "Order:desc". + so := &github.SearchOptions{ + Sort: "created", + ListOptions: *lo, + } + + for { + r, _, err := g.client.Search.Issues(context.Background(), query, so) + if err != nil { + if _, ok := err.(*github.RateLimitError); ok { + log.Printf("Hitting Github search API rate limit, sleeping for 30 seconds... error message: %v", err) + time.Sleep(30 * time.Second) + continue + } + return nil, err + } + totalIssueNumber = *r.Total + break + } + + for len(issues) < totalIssueNumber { + q := query + lastDateGot + // Get total number of pages in resp.LastPage + result, resp, err := g.client.Search.Issues(context.Background(), q, so) + if err != nil { + if _, ok := err.(*github.RateLimitError); ok { + log.Printf("Hitting Github search API rate limit, sleeping for 30 seconds... error message: %v", err) + time.Sleep(30 * time.Second) + continue + } + return nil, err + } + for _, i := range result.Issues { + if issuesGot[*i.Number] == false { + issues = append(issues, i) + issuesGot[*i.Number] = true + lastDateGot = fmt.Sprintf(" created:<=%s", i.CreatedAt.Format("2006-01-02")) + } + } + so.ListOptions.Page++ + + for so.ListOptions.Page <= resp.LastPage { + result, _, err = g.client.Search.Issues(context.Background(), q, so) + if err != nil { + if _, ok := err.(*github.RateLimitError); ok { + log.Printf("Hitting Github search API rate limit, sleeping for 30 seconds... error message: %v", err) + time.Sleep(30 * time.Second) + continue + } + return nil, err + } + for _, i := range result.Issues { + if issuesGot[*i.Number] == false { + issues = append(issues, i) + issuesGot[*i.Number] = true + lastDateGot = fmt.Sprintf(" created:<=%s", i.CreatedAt.Format("2006-01-02")) + } + } + so.ListOptions.Page++ + } + // Reset page number + so.ListOptions.Page = 1 + } + return issues, nil +} + +// AddQuery forms a Github query by appending new query parts to input query +func AddQuery(query []string, queryParts ...string) []string { + if len(queryParts) < 2 { + log.Printf("not enough parts to form a query: %v", queryParts) + return query + } + for _, part := range queryParts { + if part == "" { + return query + } + } + + return append(query, fmt.Sprintf("%s:%s", queryParts[0], strings.Join(queryParts[1:], ""))) +} + +// GetBranch is a wrapper of Github GetBranch function. +func (g GithubClient) GetBranch(ctx context.Context, owner, repo, branch string) (*github.Branch, *github.Response, error) { + return g.client.Repositories.GetBranch(ctx, owner, repo, branch) +} diff --git a/toolbox/util/github_test.go b/toolbox/util/github_test.go new file mode 100644 index 00000000000..ccf162e3c11 --- /dev/null +++ b/toolbox/util/github_test.go @@ -0,0 +1,229 @@ +package util + +import ( + "os" + "strings" + "testing" + "time" +) + +func TestLastReleases(t *testing.T) { + tables := []struct { + owner string + repo string + lastRelease map[string]string + }{ + // NOTE: Github webpage doesn't show correct release order. Use Github + // API to get the releases. + {"kubernetes", "kubernetes", map[string]string{ + "master": "v1.8.0", + "release-1.9": "v1.9.0-alpha.1", + "release-1.8": "v1.8.0", + "release-1.7": "v1.7.7", + "release-1.6": "v1.6.11", + "release-1.5": "v1.5.8", + }}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + + for _, table := range tables { + r, err := c.LastReleases(table.owner, table.repo) + if err != nil { + t.Errorf("%v %v: Unexpected error: %v", table.owner, table.repo, err) + } + for k, v := range table.lastRelease { + if r[k] != v { + t.Errorf("%v %v %v: Last release was incorrect, want: %v, got: %v", + table.owner, table.repo, k, v, r[k]) + } + } + } +} + +func TestListAllReleases(t *testing.T) { + tables := []struct { + owner string + repo string + numReleases int + }{ + // NOTE: Github webpage doesn't show number of releases directly. Use Github + // API to get the numbers. + {"kubernetes", "kubernetes", 191}, + {"kubernetes", "helm", 30}, + {"kubernetes", "dashboard", 23}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + + for _, table := range tables { + r, err := c.ListAllReleases(table.owner, table.repo) + if err != nil { + t.Errorf("%v %v: Unexpected error: %v", table.owner, table.repo, err) + } + if len(r) != table.numReleases { + t.Errorf("%v %v: Number of releases was incorrect, want: %d, got: %d", + table.owner, table.repo, table.numReleases, len(r)) + } + } +} + +func TestListAllTags(t *testing.T) { + tables := []struct { + owner string + repo string + numTags int + }{ + {"kubernetes", "kubernetes", 295}, + {"kubernetes", "helm", 35}, + {"roycaihw", "kubernetes", 267}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + + for _, table := range tables { + tags, err := c.ListAllTags(table.owner, table.repo) + if err != nil { + t.Errorf("%v %v: Unexpected error: %v", table.owner, table.repo, err) + } + if len(tags) != table.numTags { + t.Errorf("%v %v: Number of tags was incorrect, want: %d, got: %d", + table.owner, table.repo, table.numTags, len(tags)) + } + } +} + +func TestListAllIssues(t *testing.T) { + tables := []struct { + owner string + repo string + numIssues int + }{ + // NOTE: including open and closed issues and PRs. + {"kubernetes", "features", 164 + 67 + 1 + 253}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + + for _, table := range tables { + i, err := c.ListAllIssues(table.owner, table.repo) + if err != nil { + t.Errorf("%v %v: Unexpected error: %v", table.owner, table.repo, err) + } + if len(i) != table.numIssues { + t.Errorf("%v %v: Number of issues was incorrect, want: %d, got: %d", + table.owner, table.repo, table.numIssues, len(i)) + } + } +} + +func TestListAllCommits(t *testing.T) { + te, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", "2017-09-28 20:17:32 +0000 UTC") + ts, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", "2017-03-30 20:44:26 +0000 UTC") + + tables := []struct { + owner string + repo string + branch string + start time.Time + end time.Time + numCommits int + }{ + {"kubernetes", "features", "58315cc33c51f8f4d05364d80f0b66f5d980bad7", time.Time{}, time.Time{}, 705}, + {"kubernetes", "features", "", time.Time{}, time.Time{}, 705}, + {"kubernetes", "helm", "release-v1.2.1", time.Time{}, time.Time{}, 373}, + {"kubernetes", "kubectl", "master", time.Time{}, time.Time{}, 9}, + {"kubernetes", "kubectl", "master", ts, te, 7}, + } + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + + for _, table := range tables { + commits, err := c.ListAllCommits(table.owner, table.repo, table.branch, table.start, table.end) + if err != nil { + t.Errorf("%v %v %v: Unexpected error: %v", table.owner, table.repo, table.branch, err) + } + if len(commits) != table.numCommits { + t.Errorf("%v %v %v: Number of commits was incorrect, want: %d, got: %d", + table.owner, table.repo, table.branch, table.numCommits, len(commits)) + } + } +} + +func TestGetCommitDate(t *testing.T) { + tables := []struct { + owner string + repo string + tagCommit string + date string + exist bool + }{ + {"kubernetes", "helm", "v2.6.0", "2017-08-16 18:56:09 +0000 UTC", true}, + {"kubernetes", "helm", "018ef2426f4932b2d8b9a772176acb548810a222", "2017-10-03 05:14:25 +0000 UTC", true}, + {"kubernetes", "helm", "018ef2426f4932b2d8b9a772176acb548810a221", "", false}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + tags, _ := c.ListAllTags("kubernetes", "helm") + + for _, table := range tables { + d, err := c.GetCommitDate(table.owner, table.repo, table.tagCommit, tags) + var ok bool + if err == nil { + ok = true + } + if table.exist != ok { + t.Errorf("%v: Existence check failed, want: %v, got: %v", table.tagCommit, table.exist, ok) + } + if table.exist && table.date != d.String() { + t.Errorf("%v: Date was incorrect, want: %v, got: %v", table.tagCommit, table.date, d.String()) + } + } +} + +func TestAddQuerySearchIssues(t *testing.T) { + tables := []struct { + q map[string]string + num int + }{ + {map[string]string{ + "repo": "kubernetes/kubernetes", + "is": "open", + "type": "pr", + "base": "release-1.7", + }, 9}, + {map[string]string{ + "repo": "kubernetes/kubernetes", + "is": "open", + "type": "pr", + "base": "release-1.5", + }, 2}, + {map[string]string{ + "repo": "kubernetes/kubernetes", + "type": "pr", + "label": "release-note", + }, 3259}, + } + + githubToken := os.Getenv("GITHUB_TOKEN") + c := NewClient(githubToken) + + for _, table := range tables { + var query []string + for k, v := range table.q { + query = AddQuery(query, k, v) + } + result, err := c.SearchIssues(strings.Join(query, " ")) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if len(result) != table.num { + t.Errorf("Result number was incorrect, want: %d, got %d", table.num, len(result)) + } + } +} diff --git a/toolbox/util/gitlib.go b/toolbox/util/gitlib.go new file mode 100644 index 00000000000..bc6f1933c65 --- /dev/null +++ b/toolbox/util/gitlib.go @@ -0,0 +1,52 @@ +// Copyright 2017 The Kubernetes Authors All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "regexp" + "strings" +) + +// GetCurrentBranch gets the branch name where the program is called. +func GetCurrentBranch() (string, error) { + branch, err := Shell("git", "rev-parse", "--abbrev-ref", "HEAD") + if err != nil { + return "", err + } + + // Remove trailing newline + branch = strings.TrimSpace(branch) + return branch, nil +} + +// IsVer checks if input version number matches input version type among: "dotzero" +// The function returns true if the version number matches the version type; returns false +// otherwise. +func IsVer(version string, t string) bool { + m := make(map[string]string) + // Regexp Example: + // This regexp matches + // "v1.5.0" + // "v2.3.0" + // + // Doesn't match + // "v1.5.00" + // "v1.5.1" + // "v2.0" + m["dotzero"] = "v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.0$" + + re, _ := regexp.Compile(m[t]) + return re.MatchString(version) +} diff --git a/toolbox/util/gitlib_test.go b/toolbox/util/gitlib_test.go new file mode 100644 index 00000000000..782976a7bcb --- /dev/null +++ b/toolbox/util/gitlib_test.go @@ -0,0 +1,25 @@ +package util + +import "testing" + +func TestIsVer(t *testing.T) { + tables := []struct { + v string + t string + isVer bool + }{ + {"v1.8", "dotzero", false}, + {"v1.8.0", "dotzero", true}, + {"v1.8.00", "dotzero", false}, + {"v1.8.0.0", "dotzero", false}, + {"v1.8.1", "dotzero", false}, + {"v1.8.1.0", "dotzero", false}, + } + + for _, table := range tables { + result := IsVer(table.v, table.t) + if result != table.isVer { + t.Errorf("%v %v: IsVer check failed, want: %v, got: %v", table.v, table.t, table.isVer, result) + } + } +}