Skip to content

Commit

Permalink
Harvest the "without changes" runs from Taskcluster (#865)
Browse files Browse the repository at this point in the history 
This change extends the set of valid task names to accept the
"*-results-without-changes" tasks that are being added in
web-platform-tests/wpt#14382 . The change also
adds a "pr_base" label to these special runs, and "pr_head" to the PR
runs with the changes to prepare for the next step: using these runs
to calculate PR regressions (not implemented in this change).

One more step towards #708 .
  • Loading branch information
@Hexcles
Hexcles committed Dec 6, 2018
1 parent 165ac91 commit 4d96621
Show file tree
Hide file tree
Showing 3 changed files with 143 additions and 50 deletions.
70 changes: 43 additions & 27 deletions api/taskcluster/webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ import (
const flagTaskclusterAllBranches = "taskclusterAllBranches"

var (
taskNameRegex = regexp.MustCompile(`^wpt-(\w+)-(\w+)-(testharness|reftest|wdspec|results)(?:-\d+)?$`)
// This should follow https://github.com/web-platform-tests/wpt/blob/master/.taskcluster.yml
// with a notable exception that "*-stability" runs are not included at the moment.
taskNameRegex = regexp.MustCompile(`^wpt-(\w+-\w+)-(testharness|reftest|wdspec|results|results-without-changes)(?:-\d+)?$`)
resultsReceiverTimeout = time.Minute
)

Expand Down Expand Up @@ -142,7 +144,7 @@ func handleStatusEvent(ctx context.Context, payload []byte) (bool, error) {
return false, err
}

urlsByBrowser, err := extractResultURLs(log, taskGroup)
urlsByProduct, err := extractResultURLs(log, taskGroup)
if err != nil {
return false, err
}
Expand Down Expand Up @@ -172,7 +174,7 @@ func handleStatusEvent(ctx context.Context, payload []byte) (bool, error) {
*status.SHA,
username,
password,
urlsByBrowser,
urlsByProduct,
labels)
if err != nil {
return false, err
Expand Down Expand Up @@ -250,20 +252,26 @@ func extractResultURLs(log shared.Logger, group *taskGroupInfo) (map[string][]st
}

matches := taskNameRegex.FindStringSubmatch(task.Task.Metadata.Name)
if len(matches) != 4 { // full match, browser, channel, test type
if len(matches) != 3 { // full match, browser-channel, test type
log.Debugf("Ignoring unrecognized task: %s", task.Task.Metadata.Name)
continue
}
browser := fmt.Sprintf("%s-%s", matches[1], matches[2])
product := matches[1]
switch matches[2] {
case "results":
product += "-" + shared.PRHeadLabel
case "results-without-changes":
product += "-" + shared.PRBaseLabel
}

if task.Status.State != "completed" {
log.Errorf("Task group %s has an unfinished task: %s; %s will be ignored in this group.",
group.TaskGroupID, taskID, browser)
failures.Add(browser)
group.TaskGroupID, taskID, product)
failures.Add(product)
continue
}

resultURLs[browser] = append(resultURLs[browser],
resultURLs[product] = append(resultURLs[product],
// https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#get-artifact-from-latest-run
fmt.Sprintf(
"https://queue.taskcluster.net/v1/task/%s/artifacts/public/results/wpt_report.json.gz", taskID,
Expand Down Expand Up @@ -296,33 +304,41 @@ func createAllRuns(
sha,
username,
password string,
urlsByBrowser map[string][]string,
urlsByProduct map[string][]string,
labels []string) error {
errors := make(chan error, len(urlsByBrowser))
errors := make(chan error, len(urlsByProduct))
var wg sync.WaitGroup
wg.Add(len(urlsByBrowser))
wg.Add(len(urlsByProduct))
suites, _ := checksAPI.GetSuitesForSHA(sha)
for browser, urls := range urlsByBrowser {
go func(browser string, urls []string) {
for product, urls := range urlsByProduct {
go func(product string, urls []string) {
defer wg.Done()
log.Infof("Reports for %s: %v", browser, urls)
err := createRun(log, client, aeAPI, sha, uploadURL, username, password, urls, labels)
log.Infof("Reports for %s: %v", product, urls)

// chrome-dev-pr_head => [chrome, dev, pr_head]
bits := strings.Split(product, "-")
labelsForRun := labels
if bits[len(bits)-1] == shared.PRBaseLabel || bits[len(bits)-1] == shared.PRHeadLabel {
labelsForRun = append(labelsForRun, bits[len(bits)-1])
}
err := createRun(log, client, aeAPI, sha, uploadURL, username, password, urls, labelsForRun)
if err != nil {
errors <- err
} else {
spec := shared.ProductSpec{}
bits := strings.Split(browser, "-") // chrome-dev => [chrome, dev]
spec.BrowserName = bits[0]
if len(bits) > 1 {
if label := shared.ProductChannelToLabel(bits[1]); label != "" {
spec.Labels = mapset.NewSetWith(label)
}
}
for _, suite := range suites {
checksAPI.PendingCheckRun(suite, spec)
return
}

spec := shared.ProductSpec{}
spec.BrowserName = bits[0]
spec.Labels = shared.NewSetFromStringSlice(labelsForRun)
if len(bits) > 1 {
if label := shared.ProductChannelToLabel(bits[1]); label != "" {
spec.Labels.Add(label)
}
}
}(browser, urls)
for _, suite := range suites {
checksAPI.PendingCheckRun(suite, spec)
}
}(product, urls)
}
wg.Wait()
close(errors)
Expand Down
115 changes: 92 additions & 23 deletions api/taskcluster/webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package taskcluster

import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
Expand All @@ -16,7 +17,6 @@ import (

"github.com/golang/mock/gomock"
"github.com/google/go-github/github"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/web-platform-tests/wpt.fyi/api/checks"
"github.com/web-platform-tests/wpt.fyi/shared"
Expand Down Expand Up @@ -92,27 +92,49 @@ func TestExtractTaskGroupID(t *testing.T) {
extractTaskGroupID("https://tools.taskcluster.net/task-group-inspector/#/Y4rnZeqDRXGiRNiqxT5Qeg"))
}

func TestExtractResultURLs_all_success(t *testing.T) {
group := &taskGroupInfo{Tasks: make([]taskInfo, 3)}
group.Tasks[0].Status.State = "completed"
group.Tasks[0].Status.TaskID = "foo"
func TestExtractResultURLs_all_success_master(t *testing.T) {
group := &taskGroupInfo{Tasks: make([]taskInfo, 4)}
group.Tasks[0].Task.Metadata.Name = "wpt-firefox-nightly-testharness-1"
group.Tasks[1].Status.State = "completed"
group.Tasks[1].Status.TaskID = "bar"
group.Tasks[1].Task.Metadata.Name = "wpt-firefox-nightly-testharness-2"
group.Tasks[2].Status.State = "completed"
group.Tasks[2].Status.TaskID = "baz"
group.Tasks[2].Task.Metadata.Name = "wpt-chrome-dev-testharness-1"
group.Tasks[3].Task.Metadata.Name = "wpt-chrome-dev-reftest-1"
for i := 0; i < len(group.Tasks); i++ {
group.Tasks[i].Status.State = "completed"
group.Tasks[i].Status.TaskID = fmt.Sprint(i)
}

urls, err := extractResultURLs(shared.NewNilLogger(), group)
assert.Nil(t, err)
assert.Equal(t, map[string][]string{
"firefox-nightly": {
"https://queue.taskcluster.net/v1/task/foo/artifacts/public/results/wpt_report.json.gz",
"https://queue.taskcluster.net/v1/task/bar/artifacts/public/results/wpt_report.json.gz",
"https://queue.taskcluster.net/v1/task/0/artifacts/public/results/wpt_report.json.gz",
"https://queue.taskcluster.net/v1/task/1/artifacts/public/results/wpt_report.json.gz",
},
"chrome-dev": {
"https://queue.taskcluster.net/v1/task/baz/artifacts/public/results/wpt_report.json.gz",
"https://queue.taskcluster.net/v1/task/2/artifacts/public/results/wpt_report.json.gz",
"https://queue.taskcluster.net/v1/task/3/artifacts/public/results/wpt_report.json.gz",
},
}, urls)
}

func TestExtractResultURLs_all_success_pr(t *testing.T) {
group := &taskGroupInfo{Tasks: make([]taskInfo, 3)}
group.Tasks[0].Task.Metadata.Name = "wpt-chrome-dev-results"
group.Tasks[1].Task.Metadata.Name = "wpt-chrome-dev-stability"
group.Tasks[2].Task.Metadata.Name = "wpt-chrome-dev-results-without-changes"
for i := 0; i < len(group.Tasks); i++ {
group.Tasks[i].Status.State = "completed"
group.Tasks[i].Status.TaskID = fmt.Sprint(i)
}

urls, err := extractResultURLs(shared.NewNilLogger(), group)
assert.Nil(t, err)
assert.Equal(t, map[string][]string{
"chrome-dev-pr_head": {
"https://queue.taskcluster.net/v1/task/0/artifacts/public/results/wpt_report.json.gz",
},
"chrome-dev-pr_base": {
"https://queue.taskcluster.net/v1/task/2/artifacts/public/results/wpt_report.json.gz",
},
}, urls)
}
Expand All @@ -138,7 +160,7 @@ func TestExtractResultURLs_with_failures(t *testing.T) {
}, urls)
}

func TestCreateAllRuns_success(t *testing.T) {
func TestCreateAllRuns_success_master(t *testing.T) {
var requested uint32
requested = 0
handler := func(w http.ResponseWriter, r *http.Request) {
Expand All @@ -155,14 +177,14 @@ func TestCreateAllRuns_success(t *testing.T) {
checksAPI := checks.NewMockAPI(mockC)
suite := shared.CheckSuite{SHA: sha}
checksAPI.EXPECT().GetSuitesForSHA(sha).Return([]shared.CheckSuite{suite}, nil)
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("safari[experimental]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("chrome[experimental]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("firefox"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("safari[experimental,master]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("chrome[experimental,master]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("firefox[master]"))
aeAPI := sharedtest.NewMockAppEngineAPI(mockC)
aeAPI.EXPECT().GetHostname().MinTimes(1).Return("localhost:8080")

err := createAllRuns(
logrus.New(),
shared.NewNilLogger(),
&http.Client{},
aeAPI,
checksAPI,
Expand All @@ -181,6 +203,51 @@ func TestCreateAllRuns_success(t *testing.T) {
assert.Equal(t, uint32(3), requested)
}

func TestCreateAllRuns_success_pr(t *testing.T) {
var requested uint32
requested = 0
handler := func(w http.ResponseWriter, r *http.Request) {
atomic.AddUint32(&requested, 1)
w.Write([]byte("OK"))
}
server := httptest.NewServer(http.HandlerFunc(handler))
defer server.Close()
mockC := gomock.NewController(t)
defer mockC.Finish()

sha := "abcdef1234abcdef1234abcdef1234abcdef1234"

checksAPI := checks.NewMockAPI(mockC)
suite := shared.CheckSuite{SHA: sha}
checksAPI.EXPECT().GetSuitesForSHA(sha).Return([]shared.CheckSuite{suite}, nil)
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("chrome[experimental,pr_base]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("chrome[experimental,pr_head]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("firefox[pr_base]"))
checksAPI.EXPECT().PendingCheckRun(suite, sharedtest.SameProductSpec("firefox[pr_head]"))
aeAPI := sharedtest.NewMockAppEngineAPI(mockC)
aeAPI.EXPECT().GetHostname().MinTimes(1).Return("localhost:8080")

err := createAllRuns(
shared.NewNilLogger(),
&http.Client{},
aeAPI,
checksAPI,
server.URL,
sha,
"username",
"password",
map[string][]string{
"chrome-dev-pr_head": []string{"1"},
"chrome-dev-pr_base": []string{"1"},
"firefox-pr_head": []string{"1"},
"firefox-pr_base": []string{"1"},
},
nil,
)
assert.Nil(t, err)
assert.Equal(t, uint32(4), requested)
}

func TestCreateAllRuns_one_error(t *testing.T) {
var requested uint32
requested = 0
Expand Down Expand Up @@ -208,7 +275,7 @@ func TestCreateAllRuns_one_error(t *testing.T) {
aeAPI.EXPECT().GetHostname().MinTimes(1).Return("localhost:8080")

err := createAllRuns(
logrus.New(),
shared.NewNilLogger(),
&http.Client{},
aeAPI,
checksAPI,
Expand Down Expand Up @@ -243,7 +310,7 @@ func TestCreateAllRuns_all_errors(t *testing.T) {
aeAPI.EXPECT().GetHostname().MinTimes(1).Return("localhost:8080")

err := createAllRuns(
logrus.New(),
shared.NewNilLogger(),
&http.Client{Timeout: time.Second},
aeAPI,
checksAPI,
Expand All @@ -259,8 +326,10 @@ func TestCreateAllRuns_all_errors(t *testing.T) {
}

func TestTaskNameRegex(t *testing.T) {
assert.Len(t, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-results"), 4)
assert.Len(t, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-reftest-1"), 4)
assert.Len(t, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-testharness-5"), 4)
assert.Len(t, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-wdspec-1"), 4)
assert.Equal(t, []string{"chrome-dev", "results"}, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-results")[1:])
assert.Equal(t, []string{"chrome-stable", "reftest"}, taskNameRegex.FindStringSubmatch("wpt-chrome-stable-reftest-1")[1:])
assert.Equal(t, []string{"firefox-nightly", "testharness"}, taskNameRegex.FindStringSubmatch("wpt-firefox-nightly-testharness-5")[1:])
assert.Equal(t, []string{"firefox-stable", "wdspec"}, taskNameRegex.FindStringSubmatch("wpt-firefox-stable-wdspec-1")[1:])
assert.Equal(t, []string{"chrome-dev", "results-without-changes"}, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-results-without-changes")[1:])
assert.Nil(t, taskNameRegex.FindStringSubmatch("wpt-chrome-dev-stability"))
}
8 changes: 8 additions & 0 deletions shared/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ const BetaLabel = "beta"
// i.e. run from the master branch.
const MasterLabel = "master"

// PRBaseLabel is the implicit label for running just the affected tests on a
// PR but without the changes (i.e. against the base branch).
const PRBaseLabel = "pr_base"

// PRHeadLabel is the implicit label for running just the affected tests on the
// head of a PR (with the changes).
const PRHeadLabel = "pr_head"

// ProductChannelToLabel maps known product-specific channel names
// to the wpt.fyi model's equivalent.
func ProductChannelToLabel(channel string) string {
Expand Down

0 comments on commit 4d96621

Please sign in to comment.