Skip to content

Commit

Permalink
Record repo version from main configured refs
Browse files Browse the repository at this point in the history 
When the repo version is exposed in the started record, we need to
expose the resolved SHA for the main configured ref under test, which
may or may not be the first to be cloned in the clone records.

Signed-off-by: Steve Kuznetsov <skuznets@redhat.com>
  • Loading branch information
@stevekuznetsov
stevekuznetsov committed May 31, 2019
1 parent 36fcb35 commit 6c3a878
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 28 deletions.
2 changes: 2 additions & 0 deletions prow/initupload/BUILD.bazel
View file
Expand Up @@ -10,6 +10,7 @@ go_library(
importpath = "k8s.io/test-infra/prow/initupload",
visibility = ["//visibility:public"],
deps = [
"//prow/apis/prowjobs/v1:go_default_library",
"//prow/gcsupload:go_default_library",
"//prow/pod-utils/clone:go_default_library",
"//prow/pod-utils/downwardapi:go_default_library",
Expand All @@ -27,6 +28,7 @@ go_test(
deps = [
"//prow/apis/prowjobs/v1:go_default_library",
"//prow/gcsupload:go_default_library",
"//prow/pod-utils/clone:go_default_library",
"//prow/pod-utils/downwardapi:go_default_library",
"//prow/pod-utils/gcs:go_default_library",
],
Expand Down
51 changes: 30 additions & 21 deletions prow/initupload/run.go
View file
Expand Up @@ -22,24 +22,28 @@ import (
"errors"
"fmt"
"io/ioutil"
"reflect"
"strconv"
"time"

prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
"k8s.io/test-infra/prow/pod-utils/clone"
"k8s.io/test-infra/prow/pod-utils/downwardapi"
"k8s.io/test-infra/prow/pod-utils/gcs"
)

// specToStarted translate a jobspec into a started struct
// optionally overwrite RepoVersion with provided mainRefSHA
func specToStarted(spec *downwardapi.JobSpec, mainRefSHA string) gcs.Started {
// optionally overwrite RepoVersion with provided cloneRecords
func specToStarted(spec *downwardapi.JobSpec, cloneRecords []clone.Record) gcs.Started {
started := gcs.Started{
Timestamp: time.Now().Unix(),
RepoVersion: downwardapi.GetRevisionFromSpec(spec),
Timestamp: time.Now().Unix(),
}

if mainRefSHA != "" {
started.RepoVersion = mainRefSHA
if mainRefs := spec.MainRefs(); mainRefs != nil {
started.RepoVersion = shaForRefs(*mainRefs, cloneRecords)
}
if started.RepoVersion == "" {
started.RepoVersion = downwardapi.GetRevisionFromSpec(spec)
}

// TODO(fejta): VM name
Expand All @@ -60,6 +64,16 @@ func specToStarted(spec *downwardapi.JobSpec, mainRefSHA string) gcs.Started {
return started
}

// shaForRefs finds the resolved SHA after cloning and merging for the given refs
func shaForRefs(refs prowapi.Refs, cloneRecords []clone.Record) string {
for _, record := range cloneRecords {
if reflect.DeepEqual(refs, record.Refs) {
return record.FinalSHA
}
}
return ""
}

// Run will start the initupload job to upload the artifacts, logs and clone status.
func (o Options) Run() error {
spec, err := downwardapi.ResolveSpecFromEnv()
Expand All @@ -70,14 +84,14 @@ func (o Options) Run() error {
uploadTargets := map[string]gcs.UploadFunc{}

var failed bool
var mainRefSHA string
var cloneRecords []clone.Record
if o.Log != "" {
if failed, mainRefSHA, err = processCloneLog(o.Log, uploadTargets); err != nil {
if failed, cloneRecords, err = processCloneLog(o.Log, uploadTargets); err != nil {
return err
}
}

started := specToStarted(spec, mainRefSHA)
started := specToStarted(spec, cloneRecords)

startedData, err := json.Marshal(&started)
if err != nil {
Expand All @@ -100,29 +114,24 @@ func (o Options) Run() error {
// processCloneLog checks if clone operation successed or failed for a ref
// and upload clone logs as build log upon failures.
// returns: bool - clone status
// string - final main ref SHA on a successful clone
// []Record - containing final SHA on successful clones
// error - when unexpected file operation happens
func processCloneLog(logfile string, uploadTargets map[string]gcs.UploadFunc) (bool, string, error) {
func processCloneLog(logfile string, uploadTargets map[string]gcs.UploadFunc) (bool, []clone.Record, error) {
var cloneRecords []clone.Record
data, err := ioutil.ReadFile(logfile)
if err != nil {
return true, "", fmt.Errorf("could not read clone log: %v", err)
return true, cloneRecords, fmt.Errorf("could not read clone log: %v", err)
}
if err = json.Unmarshal(data, &cloneRecords); err != nil {
return true, "", fmt.Errorf("could not unmarshal clone records: %v", err)
return true, cloneRecords, fmt.Errorf("could not unmarshal clone records: %v", err)
}
// Do not read from cloneLog directly. Instead create multiple readers from cloneLog so it can
// be uploaded to both clone-log.txt and build-log.txt on failure.
cloneLog := bytes.Buffer{}
var failed bool
var mainRefSHA string
for idx, record := range cloneRecords {
for _, record := range cloneRecords {
cloneLog.WriteString(clone.FormatRecord(record))
failed = failed || record.Failed
// fill in mainRefSHA with FinalSHA from the first record
if idx == 0 {
mainRefSHA = record.FinalSHA
}

}
uploadTargets["clone-log.txt"] = gcs.DataUpload(bytes.NewReader(cloneLog.Bytes()))
Expand All @@ -140,9 +149,9 @@ func processCloneLog(logfile string, uploadTargets map[string]gcs.UploadFunc) (b
}
finishedData, err := json.Marshal(&finished)
if err != nil {
return true, mainRefSHA, fmt.Errorf("could not marshal finishing data: %v", err)
return true, cloneRecords, fmt.Errorf("could not marshal finishing data: %v", err)
}
uploadTargets["finished.json"] = gcs.DataUpload(bytes.NewReader(finishedData))
}
return failed, mainRefSHA, nil
return failed, cloneRecords, nil
}
48 changes: 41 additions & 7 deletions prow/initupload/run_test.go
View file
Expand Up @@ -21,16 +21,17 @@ import (
"testing"

prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
"k8s.io/test-infra/prow/pod-utils/clone"
"k8s.io/test-infra/prow/pod-utils/downwardapi"
"k8s.io/test-infra/prow/pod-utils/gcs"
)

func TestSpecToStarted(t *testing.T) {
var tests = []struct {
name string
spec downwardapi.JobSpec
mainRefSHA string
expected gcs.Started
name string
spec downwardapi.JobSpec
cloneRecords []clone.Record
expected gcs.Started
}{
{
name: "Refs with Pull",
Expand Down Expand Up @@ -98,7 +99,7 @@ func TestSpecToStarted(t *testing.T) {
},
},
{
name: "Refs with ExtraRef and mainRefSHA override",
name: "Refs with ExtraRef and cloneRecords containing a final SHA",
spec: downwardapi.JobSpec{
Refs: &prowapi.Refs{
Org: "kubernetes",
Expand All @@ -113,7 +114,14 @@ func TestSpecToStarted(t *testing.T) {
},
},
},
mainRefSHA: "aaaaaaaa",
cloneRecords: []clone.Record{{
Refs: prowapi.Refs{
Org: "kubernetes",
Repo: "test-infra",
BaseRef: "master",
},
FinalSHA: "aaaaaaaa",
}},
expected: gcs.Started{
RepoVersion: "aaaaaaaa",
Repos: map[string]string{
Expand All @@ -122,10 +130,36 @@ func TestSpecToStarted(t *testing.T) {
},
},
},
{
name: "Refs with only ExtraRef and cloneRecords containing a final SHA",
spec: downwardapi.JobSpec{
ExtraRefs: []prowapi.Refs{
{
Org: "kubernetes",
Repo: "release",
BaseRef: "v1.10",
},
},
},
cloneRecords: []clone.Record{{
Refs: prowapi.Refs{
Org: "kubernetes",
Repo: "release",
BaseRef: "v1.10",
},
FinalSHA: "aaaaaaaa",
}},
expected: gcs.Started{
RepoVersion: "aaaaaaaa",
Repos: map[string]string{
"kubernetes/release": "v1.10",
},
},
},
}

for _, test := range tests {
actual, expected := specToStarted(&test.spec, test.mainRefSHA), test.expected
actual, expected := specToStarted(&test.spec, test.cloneRecords), test.expected
expected.Timestamp = actual.Timestamp
if !reflect.DeepEqual(actual, expected) {
t.Errorf("%s: got started: %#v, but expected: %#v", test.name, actual, expected)
Expand Down
11 changes: 11 additions & 0 deletions prow/pod-utils/downwardapi/jobspec.go
View file
Expand Up @@ -177,3 +177,14 @@ func GetRevisionFromSpec(spec *JobSpec) string {
}
return ""
}

// MainRefs determines the main refs under test, if there are any
func (s *JobSpec) MainRefs() *prowapi.Refs {
if s.Refs != nil {
return s.Refs
}
if len(s.ExtraRefs) > 0 {
return &s.ExtraRefs[0]
}
return nil
}

0 comments on commit 6c3a878

Please sign in to comment.