Skip to content

Commit

Permalink
cmd/coordinator: add health item for gitmirror
Browse files Browse the repository at this point in the history 
Change-Id: I2e58f30a635bad22df8d7ec0d7b4e515c471aa05
Reviewed-on: https://go-review.googlesource.com/c/build/+/179877
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
  • Loading branch information
@bradfitz
bradfitz committed Jun 10, 2019
1 parent c2ec9e4 commit 4c1c063
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 12 deletions.
1 change: 1 addition & 0 deletions cmd/coordinator/kube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func initKube() error {
sourcecache.RegisterGitMirrorDial(func(ctx context.Context) (net.Conn, error) {
return goKubeClient.DialServicePort(ctx, "gitmirror", "")
})
go monitorGitMirror() // requires goKubeClient

go kubePool.pollCapacityLoop()
return nil
Expand Down
95 changes: 83 additions & 12 deletions cmd/coordinator/status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,17 @@
package main

import (
"bufio"
"bytes"
"context"
"fmt"
"html"
"html/template"
"io"
"net/http"
"os"
"os/exec"
"regexp"
"runtime"
"sort"
"strings"
Expand Down Expand Up @@ -92,7 +95,7 @@ func (w *checkWriter) hasErrors() bool {
type healthChecker struct {
ID string
Title string
EnvURL string
DocURL string
Check func(*checkWriter)
}

Expand Down Expand Up @@ -130,13 +133,14 @@ func init() {
addHealthChecker(newJoyentSolarisChecker())
addHealthChecker(newJoyentIllumosChecker())
addHealthChecker(newBasepinChecker())
addHealthChecker(newGitMirrorChecker())
}

func newBasepinChecker() *healthChecker {
return &healthChecker{
ID: "basepin",
Title: "VM snapshots",
EnvURL: "https://golang.org/issue/21305",
DocURL: "https://golang.org/issue/21305",
Check: func(w *checkWriter) {
v := basePinErr.Load()
if v == nil {
Expand All @@ -151,6 +155,73 @@ func newBasepinChecker() *healthChecker {
}
}

var lastGitMirrorErrors atomic.Value // of []string

func monitorGitMirror() {
for {
lastGitMirrorErrors.Store(gitMirrorErrors())
time.Sleep(30 * time.Second)
}
}

// $1 is repo; $2 is error message
var gitMirrorLineRx = regexp.MustCompile(`/debug/watcher/([\w-]+).?>.+</a> - (.*)`)

func gitMirrorErrors() (errs []string) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req, _ := http.NewRequest("GET", "http://gitmirror/", nil)
req = req.WithContext(ctx)
res, err := watcherProxy.Transport.RoundTrip(req)
if err != nil {
return []string{err.Error()}
}
defer res.Body.Close()
if res.StatusCode != 200 {
return []string{res.Status}
}
// TODO: add a JSON mode to gitmirror so we don't need to parse HTML.
// This works for now. We control its output.
bs := bufio.NewScanner(res.Body)
for bs.Scan() {
// Lines look like:
// <html><body><pre><a href='/debug/watcher/arch'>arch</a> - ok
// or:
// <a href='/debug/watcher/arch'>arch</a> - ok
// (See https://farmer.golang.org/debug/watcher/)
line := bs.Text()
if strings.HasSuffix(line, " - ok") {
continue
}
m := gitMirrorLineRx.FindStringSubmatch(line)
if m == nil {
if strings.Contains(line, "</html>") {
break
}
return []string{fmt.Sprintf("error parsing line %q", line)}
}
errs = append(errs, fmt.Sprintf("repo %s: %s"))
}
if err := bs.Err(); err != nil {
errs = append(errs, err.Error())
}
return errs
}

func newGitMirrorChecker() *healthChecker {
return &healthChecker{
ID: "gitmirror",
Title: "Git mirroring",
DocURL: "https://github.com/golang/build/tree/master/cmd/gitmirror",
Check: func(w *checkWriter) {
ee, _ := lastGitMirrorErrors.Load().([]string)
for _, v := range ee {
w.error(v)
}
},
}
}

func newMacHealthChecker() *healthChecker {
var hosts []string
const numMacHosts = 10 // physical Mac minis, not reverse buildlet connections
Expand Down Expand Up @@ -196,7 +267,7 @@ func newMacHealthChecker() *healthChecker {
return &healthChecker{
ID: "macs",
Title: "MacStadium Mac VMs",
EnvURL: "https://github.com/golang/build/tree/master/env/darwin/macstadium",
DocURL: "https://github.com/golang/build/tree/master/env/darwin/macstadium",
Check: func(w *checkWriter) {
// Check hosts.
checkHosts(w)
Expand All @@ -214,7 +285,7 @@ func newJoyentSolarisChecker() *healthChecker {
return &healthChecker{
ID: "joyent-solaris",
Title: "Joyent solaris/amd64 machines",
EnvURL: "https://github.com/golang/build/tree/master/env/solaris-amd64/joyent",
DocURL: "https://github.com/golang/build/tree/master/env/solaris-amd64/joyent",
Check: hostTypeChecker("host-solaris-amd64"),
}
}
Expand All @@ -223,7 +294,7 @@ func newJoyentIllumosChecker() *healthChecker {
return &healthChecker{
ID: "joyent-illumos",
Title: "Joyent illumos/amd64 machines",
EnvURL: "https://github.com/golang/build/tree/master/env/illumos-amd64-joyent",
DocURL: "https://github.com/golang/build/tree/master/env/illumos-amd64-joyent",
Check: hostTypeChecker("host-illumos-amd64-joyent"),
}
}
Expand Down Expand Up @@ -263,7 +334,7 @@ func newScalewayHealthChecker() *healthChecker {
return &healthChecker{
ID: "scaleway",
Title: "Scaleway linux/arm machines",
EnvURL: "https://github.com/golang/build/tree/master/env/linux-arm/scaleway",
DocURL: "https://github.com/golang/build/tree/master/env/linux-arm/scaleway",
Check: reverseHostChecker(hosts),
}
}
Expand All @@ -277,7 +348,7 @@ func newPacketHealthChecker() *healthChecker {
return &healthChecker{
ID: "packet",
Title: "Packet linux/arm64 machines",
EnvURL: "https://github.com/golang/build/tree/master/env/linux-arm64/packet",
DocURL: "https://github.com/golang/build/tree/master/env/linux-arm64/packet",
Check: reverseHostChecker(hosts),
}
}
Expand All @@ -291,7 +362,7 @@ func newOSUPPC64Checker() *healthChecker {
return &healthChecker{
ID: "osuppc64",
Title: "OSU linux/ppc64 machines",
EnvURL: "https://github.com/golang/build/tree/master/env/linux-ppc64/osuosl",
DocURL: "https://github.com/golang/build/tree/master/env/linux-ppc64/osuosl",
Check: reverseHostChecker(hosts),
}
}
Expand All @@ -305,7 +376,7 @@ func newOSUPPC64leChecker() *healthChecker {
return &healthChecker{
ID: "osuppc64le",
Title: "OSU linux/ppc64le machines",
EnvURL: "https://github.com/golang/build/tree/master/env/linux-ppc64le/osuosl",
DocURL: "https://github.com/golang/build/tree/master/env/linux-ppc64le/osuosl",
Check: reverseHostChecker(hosts),
}
}
Expand Down Expand Up @@ -388,8 +459,8 @@ func healthCheckerHandler(hc *healthChecker) http.Handler {
return
}
fmt.Fprintf(w, "# %q status: %s\n", hc.ID, hc.Title)
if hc.EnvURL != "" {
fmt.Fprintf(w, "# Notes: %v\n", hc.EnvURL)
if hc.DocURL != "" {
fmt.Fprintf(w, "# Notes: %v\n", hc.DocURL)
}
for _, v := range cw.Out {
fmt.Fprintf(w, "%s: %s\n", v.Level, v.Text)
Expand Down Expand Up @@ -556,7 +627,7 @@ var statusTmpl = template.Must(template.New("status").Parse(`
<h2 id=health>Health <a href='#health'>¶</a></h2>
<ul>{{range .HealthCheckers}}
<li><a href="/status/{{.ID}}">{{.Title}}</a>{{if .EnvURL}} [<a href="{{.EnvURL}}">docs</a>]{{end -}}: {{with .DoCheck.Out}}
<li><a href="/status/{{.ID}}">{{.Title}}</a>{{if .DocURL}} [<a href="{{.DocURL}}">docs</a>]{{end -}}: {{with .DoCheck.Out}}
<ul>
{{- range .}}
<li>{{ .AsHTML}}</li>
Expand Down

0 comments on commit 4c1c063

Please sign in to comment.