diff --git a/prow/Makefile b/prow/Makefile index 8f38e070a2db..09ab03294774 100644 --- a/prow/Makefile +++ b/prow/Makefile @@ -79,7 +79,7 @@ update-cluster: get-cluster-credentials @make splice-deployment --no-print-directory update-jobs: get-cluster-credentials - kubectl create configmap job-configs --from-file=presubmit=presubmit.yaml --from-file=postsubmit=postsubmit.yaml --dry-run -o yaml | kubectl replace configmap job-configs -f - + kubectl create configmap job-configs --from-file=presubmit=presubmit.yaml --from-file=postsubmit=postsubmit.yaml --from-file=periodic=periodic.yaml --dry-run -o yaml | kubectl replace configmap job-configs -f - update-plugins: get-cluster-credentials kubectl create configmap plugins --from-file=plugins=plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f - diff --git a/prow/cmd/hook/main.go b/prow/cmd/hook/main.go index 9a91ed231a78..53d720787934 100644 --- a/prow/cmd/hook/main.go +++ b/prow/cmd/hook/main.go @@ -46,6 +46,7 @@ var ( presubmit = flag.String("presubmits", "/etc/jobs/presubmit", "Path to presubmit.yaml.") postsubmit = flag.String("postsubmits", "/etc/jobs/postsubmit", "Path to postsubmit.yaml.") + periodic = flag.String("periodic", "/etc/jobs/periodic", "Path to periodic.yaml.") pluginConfig = flag.String("plugin-config", "/etc/plugins/plugins", "Path to plugin config file.") local = flag.Bool("local", false, "Run locally for testing purposes only. Does not require secret files.") @@ -116,7 +117,7 @@ func main() { } jobAgent := &jobs.JobAgent{} - if err := jobAgent.Start(*presubmit, *postsubmit); err != nil { + if err := jobAgent.Start(*presubmit, *postsubmit, *periodic); err != nil { logrus.WithError(err).Fatal("Error starting job agent.") } diff --git a/prow/cmd/line/main.go b/prow/cmd/line/main.go index a424ea9f2675..883df1473e94 100644 --- a/prow/cmd/line/main.go +++ b/prow/cmd/line/main.go @@ -57,6 +57,7 @@ var ( presubmit = flag.String("presubmit", "/etc/jobs/presubmit", "Where is presubmit.yaml.") postsubmit = flag.String("postsubmit", "/etc/jobs/postsubmit", "Where is postsubmit.yaml.") + periodic = flag.String("periodic", "/etc/jobs/periodic", "Where is periodic.yaml.") labelsPath = flag.String("labels-path", "/etc/labels/labels", "Where our metadata.labels are mounted.") jenkinsURL = flag.String("jenkins-url", "http://pull-jenkins-master:8080", "Jenkins URL") jenkinsUserName = flag.String("jenkins-user", "jenkins-trigger", "Jenkins username") @@ -124,7 +125,7 @@ func main() { } ja := jobs.JobAgent{} - if err := ja.LoadOnce(*presubmit, *postsubmit); err != nil { + if err := ja.LoadOnce(*presubmit, *postsubmit, *periodic); err != nil { logrus.WithError(err).Fatal("Error loading job config.") } fullRepoName := fmt.Sprintf("%s/%s", *repoOwner, *repoName) diff --git a/prow/cmd/splice/main.go b/prow/cmd/splice/main.go index 4a38eb038060..82e714a8f7dd 100644 --- a/prow/cmd/splice/main.go +++ b/prow/cmd/splice/main.go @@ -42,6 +42,7 @@ var ( logJson = flag.Bool("log-json", false, "output log in JSON format") presubmit = flag.String("presubmit", "/etc/jobs/presubmit", "Where is presubmit.yaml.") postsubmit = flag.String("postsubmit", "/etc/jobs/postsubmit", "Where is postsubmit.yaml.") + periodic = flag.String("periodic", "/etc/jobs/periodic", "Where is periodic.yaml.") maxBatchSize = flag.Int("batch-size", 5, "Maximum batch size") ) @@ -218,7 +219,7 @@ func main() { defer splicer.cleanup() ja := &jobs.JobAgent{} - if err := ja.Start(*presubmit, *postsubmit); err != nil { + if err := ja.Start(*presubmit, *postsubmit, *periodic); err != nil { log.WithError(err).Fatal("Could not start job agent.") } diff --git a/prow/jobs/jobs.go b/prow/jobs/jobs.go index a482bc5016a3..8c0f2398881d 100644 --- a/prow/jobs/jobs.go +++ b/prow/jobs/jobs.go @@ -17,6 +17,7 @@ limitations under the License. package jobs import ( + "fmt" "io/ioutil" "regexp" "sync" @@ -65,6 +66,25 @@ type Presubmit struct { reChanges *regexp.Regexp // from RunIfChanged } +// Postsubmit runs on push events. +type Postsubmit struct { + Name string `json:"name"` + Spec *kube.PodSpec `json:"spec,omitempty"` + + Brancher + + RunAfterSuccess []Postsubmit `json:"run_after_success"` +} + +// Periodic runs on a timer. +type Periodic struct { + Name string `json:"name"` + Spec *kube.PodSpec `json:"spec,omitempty"` + Interval string `json:"interval"` + + interval time.Duration +} + func (br Brancher) RunsAgainstBranch(branch string) bool { // Favor SkipBranches over Branches if len(br.SkipBranches) == 0 && len(br.Branches) == 0 { @@ -96,31 +116,22 @@ func (ps Presubmit) RunsAgainstChanges(changes []string) bool { return false } -// Postsubmit runs on push events. -type Postsubmit struct { - Name string `json:"name"` - Spec *kube.PodSpec `json:"spec,omitempty"` - - Brancher - - RunAfterSuccess []Postsubmit `json:"run_after_success"` -} - type JobAgent struct { mut sync.Mutex // Repo FullName (eg "kubernetes/kubernetes") -> []Job presubmits map[string][]Presubmit postsubmits map[string][]Postsubmit + periodics map[string][]Periodic } -func (ja *JobAgent) Start(pre, post string) error { - if err := ja.LoadOnce(pre, post); err != nil { +func (ja *JobAgent) Start(pre, post, periodic string) error { + if err := ja.LoadOnce(pre, post, periodic); err != nil { return err } ticker := time.Tick(1 * time.Minute) go func() { for range ticker { - ja.tryLoad(pre, post) + ja.tryLoad(pre, post, periodic) } }() return nil @@ -154,6 +165,11 @@ func (ja *JobAgent) AllJobNames() []string { for _, v := range ja.postsubmits { res = append(res, listPost(v)...) } + for _, v := range ja.periodics { + for _, j := range v { + res = append(res, j.Name) + } + } return res } @@ -176,13 +192,16 @@ func (ja *JobAgent) SetPresubmits(jobs map[string][]Presubmit) error { return nil } -func (ja *JobAgent) LoadOnce(pre, post string) error { +func (ja *JobAgent) LoadOnce(pre, post, periodic string) error { ja.mut.Lock() defer ja.mut.Unlock() if err := ja.loadPresubmits(pre); err != nil { return err } - return ja.loadPostsubmits(post) + if err := ja.loadPostsubmits(post); err != nil { + return err + } + return ja.loadPeriodics(periodic) } func (ja *JobAgent) MatchingPresubmits(fullRepoName, body string, testAll *regexp.Regexp) []Presubmit { @@ -306,7 +325,30 @@ func (ja *JobAgent) loadPostsubmits(path string) error { return nil } -func (ja *JobAgent) tryLoad(pre, post string) { +// Hold the lock. +func (ja *JobAgent) loadPeriodics(path string) error { + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + nj := map[string][]Periodic{} + if err := yaml.Unmarshal(b, &nj); err != nil { + return err + } + for _, js := range nj { + for j := range js { + d, err := time.ParseDuration(js[j].Interval) + if err != nil { + return fmt.Errorf("cannot parse duration for %s: %v", js[j].Name, err) + } + js[j].interval = d + } + } + ja.periodics = nj + return nil +} + +func (ja *JobAgent) tryLoad(pre, post, periodic string) { ja.mut.Lock() defer ja.mut.Unlock() if err := ja.loadPresubmits(pre); err != nil { @@ -315,4 +357,7 @@ func (ja *JobAgent) tryLoad(pre, post string) { if err := ja.loadPostsubmits(post); err != nil { logrus.WithField("path", post).WithError(err).Error("Error loading postsubmits.") } + if err := ja.loadPeriodics(periodic); err != nil { + logrus.WithField("path", periodic).WithError(err).Error("Error loading periodics.") + } } diff --git a/prow/jobs/jobs_test.go b/prow/jobs/jobs_test.go index 1fea10abca07..62141ee22b27 100644 --- a/prow/jobs/jobs_test.go +++ b/prow/jobs/jobs_test.go @@ -256,6 +256,13 @@ func TestPostsubmits(t *testing.T) { } } +func TestPeriodics(t *testing.T) { + ja := &JobAgent{} + if err := ja.loadPeriodics("../periodic.yaml"); err != nil { + t.Fatalf("Could not load job configs: %v", err) + } +} + func TestGetPresubmits(t *testing.T) { pres := []Presubmit{ { diff --git a/prow/periodic.yaml b/prow/periodic.yaml new file mode 100644 index 000000000000..ed97d539c095 --- /dev/null +++ b/prow/periodic.yaml @@ -0,0 +1 @@ +--- diff --git a/testgrid/jenkins_verify/jenkins_validate.go b/testgrid/jenkins_verify/jenkins_validate.go index 69f0f61f2476..b7e14214f690 100644 --- a/testgrid/jenkins_verify/jenkins_validate.go +++ b/testgrid/jenkins_verify/jenkins_validate.go @@ -67,7 +67,7 @@ func main() { } ja := jobs.JobAgent{} - if err := ja.LoadOnce(prowPath+"/presubmit.yaml", prowPath+"/postsubmit.yaml"); err != nil { + if err := ja.LoadOnce(prowPath+"/presubmit.yaml", prowPath+"/postsubmit.yaml", prowPath+"/periodic.yaml"); err != nil { fmt.Printf("Could not load prow configs: %v\n", err) os.Exit(1) }