Skip to content

Commit

Permalink
cmd/coordinator: render partial build dashboard
Browse files Browse the repository at this point in the history
This change adds a build dashboard handler to the Coordinator. As part
of the effort to remove the app/appengine build dashboard, this moves a
large part of the template and logic into cmd/coordinator.

As part of this change, cmd/coordinator/internal/dashboard has been
created. I originally developed this in the main package, but the main
package is very crowded in the coordinator. Giving the dashboard its own
package also made testing easier.

Currently, this implementation only supports rendering part of the build
dashboard for the Go repository on master. It does not yet link to test
logs, and only shows successful state.

Updates golang/go#34744

Change-Id: I6ffe064b9fc5e4a3271eadfd5ac45d5baf4ebd37
Reviewed-on: https://go-review.googlesource.com/c/build/+/221920
Run-TryBot: Alexander Rakoczy <alex@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Carlos Amedee <carlos@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
  • Loading branch information
toothrot committed Mar 5, 2020
1 parent ef9e68d commit 236dbea
Show file tree
Hide file tree
Showing 11 changed files with 994 additions and 86 deletions.
17 changes: 12 additions & 5 deletions buildenv/envs.go
Expand Up @@ -54,11 +54,15 @@ type Environment struct {
// This field may be overridden as necessary without impacting other fields.
ProjectName string

// ProjectNumber is the GCP project's number, as visible in the admin console.
// This is used for things such as constructing the "email" of the default
// service account.
// ProjectNumber is the GCP build infrastructure project's number, as visible
// in the admin console. This is used for things such as constructing the
// "email" of the default service account.
ProjectNumber int64

// The GCP project name for the Go project, where build status is stored.
// This field may be overridden as necessary without impacting other fields.
GoProjectName string

// The IsProd flag indicates whether production functionality should be
// enabled. When true, GCE and Kubernetes builders are enabled and the
// coordinator serves on 443. Otherwise, GCE and Kubernetes builders are
Expand Down Expand Up @@ -231,6 +235,7 @@ func ByProjectID(projectID string) *Environment {
var Staging = &Environment{
ProjectName: "go-dashboard-dev",
ProjectNumber: 302018677728,
GoProjectName: "go-dashboard-dev",
IsProd: true,
ControlZone: "us-central1-f",
VMZones: []string{"us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"},
Expand Down Expand Up @@ -263,6 +268,7 @@ var Staging = &Environment{
var Production = &Environment{
ProjectName: "symbolic-datum-552",
ProjectNumber: 872405196845,
GoProjectName: "golang-org",
IsProd: true,
ControlZone: "us-central1-f",
VMZones: []string{"us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"},
Expand Down Expand Up @@ -292,8 +298,9 @@ var Production = &Environment{
}

var Development = &Environment{
IsProd: false,
StaticIP: "127.0.0.1",
GoProjectName: "golang-org",
IsProd: false,
StaticIP: "127.0.0.1",
}

// possibleEnvs enumerate the known buildenv.Environment definitions.
Expand Down
2 changes: 2 additions & 0 deletions cmd/coordinator/Dockerfile
Expand Up @@ -86,6 +86,8 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*


COPY --from=build /go/src/golang.org/x/build/cmd/coordinator/internal/dashboard/dashboard.html /dashboard.html
COPY --from=build /go/src/golang.org/x/build/cmd/coordinator/style.css /style.css
COPY --from=build /go/bin/coordinator /
COPY --from=build_drawterm /usr/local/bin/drawterm /usr/local/bin/

Expand Down
7 changes: 7 additions & 0 deletions cmd/coordinator/coordinator.go
Expand Up @@ -45,6 +45,7 @@ import (
"unicode"

"go4.org/syncutil"
builddash "golang.org/x/build/cmd/coordinator/internal/dashboard"
"golang.org/x/build/cmd/coordinator/protos"
"google.golang.org/grpc"
grpc4 "grpc.go4.org"
Expand Down Expand Up @@ -305,6 +306,11 @@ func main() {
}
maintnerClient = apipb.NewMaintnerServiceClient(cc)

if err := loadStatic(); err != nil {
log.Printf("Failed to load static resources: %v", err)
}

dh := &builddash.Handler{Datastore: goDSClient, Maintner: maintnerClient}
gs := &gRPCServer{dashboardURL: "https://build.golang.org"}
protos.RegisterCoordinatorServer(grpcServer, gs)
http.HandleFunc("/", handleStatus)
Expand All @@ -319,6 +325,7 @@ func main() {
http.HandleFunc("/try.json", serveTryStatus(true))
http.HandleFunc("/status/reverse.json", reversePool.ServeReverseStatusJSON)
http.HandleFunc("/status/post-submit-active.json", handlePostSubmitActiveJSON)
http.Handle("/dashboard", dh)
http.Handle("/buildlet/create", requireBuildletProxyAuth(http.HandlerFunc(handleBuildletCreate)))
http.Handle("/buildlet/list", requireBuildletProxyAuth(http.HandlerFunc(handleBuildletList)))
go func() {
Expand Down
21 changes: 16 additions & 5 deletions cmd/coordinator/gce.go
Expand Up @@ -42,7 +42,7 @@ import (
"golang.org/x/build/internal/lru"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
)

Expand All @@ -63,7 +63,10 @@ func gceAPIGate() {
var (
buildEnv *buildenv.Environment

dsClient *datastore.Client
// dsClient is a datastore client for the build project (symbolic-datum-552), where build progress is stored.
dsClient *datastore.Client
// goDSClient is a datastore client for golang-org, where build status is stored.
goDSClient *datastore.Client
computeService *compute.Service
gcpCreds *google.Credentials
errTryDeps error // non-nil if try bots are disabled
Expand Down Expand Up @@ -100,7 +103,7 @@ func initGCE() error {
}

buildEnv = buildenv.ByProjectID(*buildEnvName)
inStaging = (buildEnv == buildenv.Staging)
inStaging = buildEnv == buildenv.Staging

// If running on GCE, override the zone and static IP, and check service account permissions.
if metadata.OnGCE() {
Expand Down Expand Up @@ -153,9 +156,17 @@ func initGCE() error {
dsClient, err = datastore.NewClient(ctx, buildEnv.ProjectName)
if err != nil {
if *mode == "dev" {
log.Printf("Error creating datastore client: %v", err)
log.Printf("Error creating datastore client for %q: %v", buildEnv.ProjectName, err)
} else {
log.Fatalf("Error creating datastore client: %v", err)
log.Fatalf("Error creating datastore client for %q: %v", buildEnv.ProjectName, err)
}
}
goDSClient, err = datastore.NewClient(ctx, buildEnv.GoProjectName)
if err != nil {
if *mode == "dev" {
log.Printf("Error creating datastore client for %q: %v", buildEnv.GoProjectName, err)
} else {
log.Fatalf("Error creating datastore client for %q: %v", buildEnv.GoProjectName, err)
}
}

Expand Down
188 changes: 188 additions & 0 deletions cmd/coordinator/internal/dashboard/dashboard.html
@@ -0,0 +1,188 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>{{$.Dashboard.Name}} Build Dashboard</title>
<link rel="stylesheet" href="/style.css"/>
<script async>
let showUnsupported = window.location.hash.substr(1) !== 'short';

function redraw() {
showUnsupported = !document.querySelector('#showshort').checked;
document.querySelectorAll('.unsupported').forEach(el => {
el.hidden = !showUnsupported;
});
window.location.hash = showUnsupported ? '' : 'short';
document.querySelectorAll('.Build-builderOS').forEach(el => {
el.setAttribute(
'colspan',
el.getAttribute(
showUnsupported ? 'data-archs' : 'data-firstClassArchs'
)
);
});
document.querySelectorAll('.Build-osColumn').forEach(el => {
el.setAttribute(
'span',
el.getAttribute(
showUnsupported ? 'data-archs' : 'data-firstClassArchs'
)
);
});
}

window.addEventListener('load', () => {
document.querySelector('#showshort').checked = !showUnsupported;
document.querySelector('#showshort').addEventListener('change', redraw);
redraw();
});
</script>
</head>

<body class="Dashboard">
<header class="Dashboard-topbar">
<h1>Go Dashboard</h1>
</header>

<form action="../.." method="GET">
<input type="hidden" name="repo" value="{{.Package.Path}}"/>
<nav class="Dashboard-controls">
{{if not (eq .Branch "")}}
<label>
<select name="branch" onchange="this.form.submit()">
{{range $.Branches}}
<option value="{{.}}" {{if eq $.Branch .}} selected{{end}}>
{{.}}
</option>
{{end}}
</select>
</label>
{{end}}
<label>
<input type="checkbox" id="showshort"/>
show only
<a href="http://golang.org/wiki/PortingPolicy">first-class ports</a>
</label>
</nav>
</form>
<h2 class="Dashboard-packageName">{{$.Package.Name}}</h2>

<div class="page Build-scrollTable">
{{if $.Commits}}
<table class="Build">
<colgroup class="col-hash" {{if $.Package.Path}} span="2" {{end}}></colgroup>
<colgroup class="col-user"></colgroup>
<colgroup class="col-time"></colgroup>
<colgroup class="Build-descriptionColumn col-desc"></colgroup>
{{range $.Builders}}
<colgroup class="Build-osColumn col-result{{if .Unsupported}} unsupported{{end}}" span="{{.Archs | len}}"
data-archs="{{.Archs | len}}" data-firstClassArchs="{{.FirstClassArchs | len}}"></colgroup>
{{end}}
<tr class="Build-builderOSRow">
{{if $.Package.Path}}
<th colspan="2">revision</th>
{{else}}
<th>&nbsp;</th>
{{end}}
<th></th>
<th></th>
<th></th>
{{range $.Builders}}
<th class="Build-builderOS{{if not .FirstClass}} unsupported{{end}}" colspan="{{.Archs | len}}"
data-archs="{{.Archs | len}}" data-firstClassArchs="{{.FirstClassArchs | len}}">
{{.OS}}
</th>
{{end}}
</tr>

<tr class="Build-builderArchRow">
{{if $.Package.Path}}
<th class="result arch">repo</th>
<th class="result arch">{{$.Dashboard.Name}}</th>
{{else}}
<th>&nbsp;</th>
{{end}}
<th></th>
<th></th>
<th></th>
{{range $.Builders}}
{{range.Archs}}
<th class="result arch{{if not (.FirstClass)}} unsupported{{end}}" title="{{.Name}}">
{{.Arch}}
</th>
{{end}}
{{end}}
</tr>

<tr class="Build-builderTagRow">
<th {{if $.Package.Path}}colspan="2" {{end}}>&nbsp;</th>
<th></th>
<th></th>
<th></th>
{{range $.Builders}}
{{range.Archs}}
<th class="Build-resultArch result arch{{if not (.FirstClass)}} unsupported{{end}}" title="{{.Name}}">
{{.Tag}}
</th>
{{end}}
{{end}}
</tr>
{{range $c := $.Commits}}
<tr class="commit">
<td class="hash">
<span class="ShortHash">
<a href="https://go-review.googlesource.com/q/{{$c.Hash}}">
{{$c.Hash}}
</a>
</span>
</td>
<td class="Build-user" title="{{$c.User}}">{{$c.ShortUser}}</td>
<td class="Build-commitTime">
{{$c.Time}}
</td>
<td class="Build-desc desc" title="{{$c.Desc}}">{{$c.Desc}}</td>
{{range $b := $.Builders}}
{{range $a := .Archs}}
<td class="{{if not $a.FirstClass}} unsupported{{end}}" data-builder="{{$a.Name}}">
{{with $c.ResultForBuilder $a.Name}}
{{if .OK}} ok {{end}}
{{end}}
</td>
{{end}}
{{end}}
</tr>
{{end}}
</table>

{{with $.Pagination}}
<div class="paginate">
<nav>
{{if .HasPrev}}
<a href="?repo={{$.Package.Path}}&page={{.Prev}}&branch={{$.Branch}}">
newer
</a>
{{else}}
newer
{{end}}
{{if .Next}}
<a href="?repo={{$.Package.Path}}&page={{.Next}}branch={{$.Branch}}">
older
</a>
{{else}}
older
{{end}}
{{if .HasPrev}}
<a href="?branch={{$.Branch}}">
latest
</a>
{{else}}
latest
{{end}}
</nav>
</div>
{{end}}
{{else}}
<p>No commits to display. Hm.</p>
{{end}}
</div>
</body>
</html>
44 changes: 44 additions & 0 deletions cmd/coordinator/internal/dashboard/datastore.go
@@ -0,0 +1,44 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.13
// +build linux darwin

package dashboard

import (
"context"
"log"

"cloud.google.com/go/datastore"
)

// getDatastoreResults populates result data on commits, fetched from Datastore.
func getDatastoreResults(ctx context.Context, cl *datastore.Client, commits []*commit, pkg string) {
var keys []*datastore.Key
for _, c := range commits {
pkey := datastore.NameKey("Package", pkg, nil)
pkey.Namespace = "Git"
key := datastore.NameKey("Commit", "|"+c.Hash, pkey)
key.Namespace = "Git"
keys = append(keys, key)
}
out := make([]*Commit, len(keys))
if err := cl.GetMulti(ctx, keys, out); err != nil {
log.Printf("getResults: error fetching %d results: %v", len(keys), err)
return
}
hashOut := make(map[string]*Commit)
for _, o := range out {
if o != nil && o.Hash != "" {
hashOut[o.Hash] = o
}
}
for _, c := range commits {
if result, ok := hashOut[c.Hash]; ok {
c.ResultData = result.ResultData
}
}
return
}

0 comments on commit 236dbea

Please sign in to comment.