Add swarming_token app to retrieve a token and write to a file

Retrieves the service account token from metadata and writes to the
location expected by Swarming. Should be run as a cron job.

Also add go_build_all_platforms.sh to compile an app for every platform
we support, and go_build_and_upload_all_platforms.sh to do the same and
upload to the skia-public-binaries bucket.

Bug: skia:5410
Change-Id: Id0ac5bd84ecdf1f66a95eb1864a28d16d5c200dc
Reviewed-on: https://skia-review.googlesource.com/99886
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>

android_compile/go/android_compile/main.go

@@ -246,12 +246,7 @@ func main() {
 		)
 	}
 	defer common.Defer()
-
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatal(err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	reloadTemplates()
 	serverURL = "https://" + *host
@@ -282,6 +277,9 @@ func main() {
 	// of run(s).
 	ctx := context.Background()
 	_, runningTasksAndKeys, err := GetCompileTasksAndKeys()
+	if err != nil {
+		sklog.Fatalf("Failed to retrieve compile tasks and keys: %s", err)
+	}
 	for _, taskAndKey := range runningTasksAndKeys {
 		sklog.Infof("Found orphaned task %d. Retriggering it...", taskAndKey.key.ID)
 		triggerCompileTask(ctx, taskAndKey.task, taskAndKey.key)

autoroll/go/autoroll/main.go

@@ -294,12 +294,7 @@ func main() {
 	defer common.Defer()
 
 	Init()
-
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatal(err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	if *rollIntoGoogle3 {
 		if *cqExtraTrybots != "" {

ct/go/ctfe/main.go

@@ -250,11 +250,7 @@ func main() {
 	}
 
 	common.InitWithMust("ctfe", common.PrometheusOpt(promPort), common.CloudLoggingOpt())
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatal(err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	Init()
 	serverURL := "https://" + *host

ct_pixel_diff/go/ct_pixel_diff/main.go

@@ -81,11 +81,7 @@ func main() {
 	ctx := context.Background()
 
 	// Get the version of the repo.
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatalf("Unable to retrieve version: %s", err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	// Set the resource directory if it's empty.
 	if *resourcesDir == "" {

go/metadata/metadata.go

@@ -1,13 +1,15 @@
 package metadata
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 
 	"go.skia.org/infra/go/httputils"
 	"go.skia.org/infra/go/sklog"
 	"go.skia.org/infra/go/util"
+	"golang.org/x/oauth2"
 )
 
 // GCE project level metadata keys.
@@ -40,19 +42,22 @@ const (
 	// and email address that are allowed to perform admin tasks.
 	ADMIN_WHITE_LIST = "admin_white_list"
 
-	// METADATA_URL_PREFIX_TMPL is the template for the first part of the
+	// METADATA_PATH_PREFIX_TMPL is the template for the first part of the
 	// metadata URL. The placeholder is for the level ("instance" or
 	// "project").
-	METADATA_URL_PREFIX_TMPL = "/computeMetadata/v1/%s"
+	METADATA_PATH_PREFIX_TMPL = "/computeMetadata/v1/%s"
 
 	// METADATA_SUB_URL_TMPL is the URL template for metadata. The
 	// placeholders are for the level ("instance" or "project") and the
 	// metadata key.
-	METADATA_SUB_URL_TMPL = METADATA_URL_PREFIX_TMPL + "/attributes/%s"
+	METADATA_SUB_URL_TMPL = METADATA_PATH_PREFIX_TMPL + "/attributes/%s"
+
+	// METADATA_URL_PREFIX is the prefix of the metadata URL.
+	METADATA_URL_PREFIX = "http://metadata"
 
 	// METADATA_URL is the URL template for metadata. The placeholders are
 	// for the level ("instance" or "project") and the metadata key.
-	METADATA_URL = "http://metadata" + METADATA_SUB_URL_TMPL
+	METADATA_URL = METADATA_URL_PREFIX + METADATA_SUB_URL_TMPL
 
 	// WEBHOOK_REQUEST_SALT is used to authenticate webhook requests. The value stored in
 	// Metadata is base64-encoded.
@@ -76,33 +81,46 @@ const (
 	HEADER_MD_FLAVOR_VAL = "Google"
 )
 
-// get retrieves the named value from the Metadata server. See
-// https://developers.google.com/compute/docs/metadata
-//
-// level should be either "instance" or "project" for the kind of
-// metadata to retrieve.
-func get(name string, level string) (string, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf(METADATA_URL, level, name), nil)
+var (
+	// Metadata path for a default service account token.
+	TOKEN_PATH = fmt.Sprintf(METADATA_PATH_PREFIX_TMPL, LEVEL_INSTANCE) + "/service-accounts/default/token"
+
+	// Full metadata URL for a default service account token.
+	TOKEN_URL = METADATA_URL_PREFIX + TOKEN_PATH
+)
+
+// getUrl retrieves the given metadata URL.
+func getUrl(url string) (string, error) {
+	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return "", fmt.Errorf("metadata.Get() failed to build request: %s", err)
 	}
 	c := httputils.NewTimeoutClient()
 	req.Header.Add(HEADER_MD_FLAVOR_KEY, HEADER_MD_FLAVOR_VAL)
 	resp, err := c.Do(req)
 	if err != nil {
-		return "", fmt.Errorf("metadata.Get() failed to make HTTP request for %s: %s", name, err)
+		return "", fmt.Errorf("metadata.Get() failed to make HTTP request for %s: %s", url, err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		return "", fmt.Errorf("HTTP response has status %d", resp.StatusCode)
 	}
 	defer util.Close(resp.Body)
 	value, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return "", fmt.Errorf("Failed to read %s from metadata server: %s", name, err)
+		return "", fmt.Errorf("Failed to read %s from metadata server: %s", url, err)
 	}
 	return string(value), nil
 }
 
+// get retrieves the named value from the Metadata server. See
+// https://developers.google.com/compute/docs/metadata
+//
+// level should be either "instance" or "project" for the kind of
+// metadata to retrieve.
+func get(name string, level string) (string, error) {
+	return getUrl(fmt.Sprintf(METADATA_URL, level, name))
+}
+
 // Get retrieves the named value from the instance Metadata server. See
 // https://developers.google.com/compute/docs/metadata
 func Get(name string) (string, error) {
@@ -158,3 +176,16 @@ func NSQDTestServerAddr() string {
 	sklog.Errorf("Got test NSQ server: %s", server)
 	return fmt.Sprintf("%s:4150", server)
 }
+
+// GetToken returns a default service account token.
+func GetToken() (*oauth2.Token, error) {
+	tokString, err := getUrl(TOKEN_URL)
+	if err != nil {
+		return nil, err
+	}
+	var tok oauth2.Token
+	if err := json.Unmarshal([]byte(tokString), &tok); err != nil {
+		return nil, err
+	}
+	return &tok, nil
+}

go/metadata/server.go

@@ -172,8 +172,7 @@ func SetupServer(r *mux.Router, pm ProjectMetadata, im InstanceMetadata, tok *Se
 
 	// The service account token path does not quite follow the pattern of
 	// the other two metadata types.
-	path := fmt.Sprintf(METADATA_URL_PREFIX_TMPL, LEVEL_INSTANCE) + "/service-accounts/default/token"
-	r.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
+	r.HandleFunc(TOKEN_PATH, func(w http.ResponseWriter, r *http.Request) {
 		t, err := tok.Get()
 		if err != nil {
 			httputils.ReportError(w, r, err, "Failed to obtain key.")
@@ -184,5 +183,4 @@ func SetupServer(r *mux.Router, pm ProjectMetadata, im InstanceMetadata, tok *Se
 			return
 		}
 	})
-	sklog.Infof("service-account: %s", path)
 }

go/skiaversion/skiaversion.go

@@ -31,6 +31,15 @@ func GetVersion() (*Version, error) {
 	return nil, fmt.Errorf("No version was set at compile time! Did you forget to run \"make skiaversion\"?")
 }
 
+// MustLogVersion logs the version info and panics if it is not found.
+func MustLogVersion() {
+	v, err := GetVersion()
+	if err != nil {
+		sklog.Fatal(err)
+	}
+	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+}
+
 // JsonHandler is a pre-built handler for HTTP requests which returns version
 // information in JSON format.
 func JsonHandler(w http.ResponseWriter, r *http.Request) {

golden/go/correctness_migratedb/main.go

@@ -26,12 +26,7 @@ func main() {
 
 	// Global init to initialize logging and parse arguments.
 	common.Init()
-
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatalf("Unable to retrieve version: %s", err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	if *promptPassword {
 		if err := dbConf.PromptForPassword(); err != nil {

golden/go/skia_diff_server/main.go

@@ -58,11 +58,7 @@ func main() {
 	common.InitWithMust(appName, logOpts...)
 
 	// Get the version of the repo.
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatalf("Unable to retrieve version: %s", err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	// Get the client to be used to access GCS.
 	client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")

golden/go/skiacorrectness/main.go

@@ -122,11 +122,7 @@ func main() {
 
 	ctx := context.Background()
 
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatalf("Unable to retrieve version: %s", err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	// Enable the memory profiler if memProfile was set.
 	// TODO(stephana): This should be moved to a HTTP endpoint that

leasing/go/leasing/main.go

@@ -476,11 +476,7 @@ func main() {
 		)
 	}
 
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatal(err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	reloadTemplates()
 	serverURL = "https://" + *host

skolo/Makefile

@@ -1,3 +1,5 @@
+include ../go/skiaversion/skiaversion.mk
+
 .PHONY: all
 all: hotspare backup cloudlogger trooper_tools
 
@@ -111,3 +113,7 @@ get_oauth2_token_push: get_oauth2_token_release
 	pushcli "get-oauth2-token" "jumphost-rpi-01"
 	pushcli "get-oauth2-token" "jumphost-win-02"
 
+.PHONY: refresh_swarming_token_upload
+refresh_swarming_token_upload: skiaversion
+	../tools/go_build_and_upload_all_platforms.sh ./go/refresh_swarming_token
+

skolo/go/metadata_server/main.go

@@ -52,11 +52,7 @@ func main() {
 		//common.CloudLoggingOpt(),
 	)
 
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatal(err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	// TODO(borenet): Load these from a file?
 	var pm projectMetadataMap = map[string]string{

skolo/go/refresh_swarming_token/main.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"runtime"
+
+	"go.skia.org/infra/go/common"
+	"go.skia.org/infra/go/metadata"
+	"go.skia.org/infra/go/skiaversion"
+	"go.skia.org/infra/go/sklog"
+	"go.skia.org/infra/go/util"
+)
+
+/*
+	Obtain an OAuth token from metadata and write it to the expected location.
+
+	Run this program every 4 minutes to ensure that the token is always valid.
+*/
+
+const (
+	TOKEN_DEST     = "/var/lib/swarming/oauth_bot_token.json"
+	TOKEN_DEST_WIN = "C:\\swarming\\oauth_bot_token.json"
+)
+
+func main() {
+	common.Init()
+	skiaversion.MustLogVersion()
+
+	// Obtain the token.
+	tok, err := metadata.GetToken()
+	if err != nil {
+		sklog.Fatal(err)
+	}
+
+	// Swarming expects tokens in a slightly different format.
+	token := struct {
+		Token     string `json:"token"`
+		TokenType string `json:"token_type"`
+		Expiry    int64  `json:"expiry"`
+	}{
+		Token:     tok.AccessToken,
+		TokenType: tok.TokenType,
+		Expiry:    tok.Expiry.Unix(),
+	}
+
+	// Write the token to the expected location.
+	dest := TOKEN_DEST
+	if runtime.GOOS == "windows" {
+		dest = TOKEN_DEST_WIN
+	}
+	if err := util.WithWriteFile(dest, func(w io.Writer) error {
+		return json.NewEncoder(w).Encode(token)
+	}); err != nil {
+		sklog.Fatal(err)
+	}
+}

status/go/status/main.go

@@ -649,11 +649,7 @@ func main() {
 		common.CloudLoggingOpt(),
 	)
 
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatalf("Unable to get Skia version: %s", err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	Init()
 	if *testing {
@@ -666,6 +662,7 @@ func main() {
 	ctx := context.Background()
 
 	// Create remote Tasks DB.
+	var err error
 	if *testing {
 		taskDb, err = local_db.NewDB("status-testing", path.Join(*workdir, "status-testing.bdb"))
 		if err != nil {

task_scheduler/go/task_scheduler/main.go

@@ -582,11 +582,7 @@ func main() {
 
 	reloadTemplates()
 
-	v, err := skiaversion.GetVersion()
-	if err != nil {
-		sklog.Fatal(err)
-	}
-	sklog.Infof("Version %s, built at %s", v.Commit, v.Date)
+	skiaversion.MustLogVersion()
 
 	ctx, cancelFn := context.WithCancel(context.Background())
 	defer cancelFn()