Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Run integration tests in parallel when possible.
- Loading branch information
Tom Wilkie
committed
Jul 29, 2015
1 parent
4da999b
commit 669a0db
Showing
12 changed files
with
260 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,246 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "fmt" | ||
| "net/http" | ||
| "net/url" | ||
| "os" | ||
| "os/exec" | ||
| "sort" | ||
| "strconv" | ||
| "strings" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/docker/docker/pkg/mflag" | ||
| "github.com/mgutz/ansi" | ||
| ) | ||
|
|
||
| const ( | ||
| schedulerHost = "positive-cocoa-90213.appspot.com" | ||
| JSON = "application/json" | ||
| ) | ||
|
|
||
| var ( | ||
| start = ansi.ColorCode("white+ub") | ||
| fail = ansi.ColorCode("red+b") | ||
| succ = ansi.ColorCode("green+b") | ||
| reset = ansi.ColorCode("reset") | ||
|
|
||
| useScheduler = false | ||
| runParallel = false | ||
|
|
||
| consoleLock = sync.Mutex{} | ||
| ) | ||
|
|
||
| type test struct { | ||
| name string | ||
| hosts int | ||
| } | ||
|
|
||
| type schedule struct { | ||
| Tests []string `json:"tests"` | ||
| } | ||
|
|
||
| type result struct { | ||
| errored bool | ||
| hosts []string | ||
| } | ||
|
|
||
| type tests []test | ||
|
|
||
| func (ts tests) Len() int { return len(ts) } | ||
| func (ts tests) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } | ||
| func (ts tests) Less(i, j int) bool { | ||
| if ts[i].hosts != ts[j].hosts { | ||
| return ts[i].hosts < ts[j].hosts | ||
| } | ||
| return ts[i].name < ts[j].name | ||
| } | ||
|
|
||
| func (ts *tests) pick(availible int) (test, bool) { | ||
| // pick the first test that fits in the availible hosts | ||
| for i, test := range *ts { | ||
| if test.hosts <= availible { | ||
| *ts = append((*ts)[:i], (*ts)[i+1:]...) | ||
| return test, true | ||
| } | ||
| } | ||
|
|
||
| return test{}, false | ||
| } | ||
|
|
||
| func (t test) run(hosts []string) bool { | ||
| consoleLock.Lock() | ||
| fmt.Printf("%s>>> Running %s on %s%s\n", start, t.name, hosts, reset) | ||
| consoleLock.Unlock() | ||
|
|
||
| var out bytes.Buffer | ||
|
|
||
| cmd := exec.Command(t.name) | ||
| cmd.Env = os.Environ() | ||
| cmd.Stdout = &out | ||
| cmd.Stderr = &out | ||
|
|
||
| // replace HOSTS in env | ||
| for i, env := range cmd.Env { | ||
| if strings.HasPrefix(env, "HOSTS") { | ||
| cmd.Env[i] = fmt.Sprintf("HOSTS=%s", strings.Join(hosts, " ")) | ||
| break | ||
| } | ||
| } | ||
|
|
||
| start := time.Now() | ||
| err := cmd.Run() | ||
| duration := float64(time.Now().Sub(start)) / float64(time.Second) | ||
|
|
||
| consoleLock.Lock() | ||
| if err != nil { | ||
| fmt.Printf("%s>>> Test %s finished after %0.1f secs with error: %v%s\n", fail, t.name, duration, err, reset) | ||
| } else { | ||
| fmt.Printf("%s>>> Test %s finished with success after %0.1f secs%s\n", succ, t.name, duration, reset) | ||
| } | ||
| fmt.Print(out.String()) | ||
| fmt.Println() | ||
| consoleLock.Unlock() | ||
|
|
||
| if err != nil && useScheduler { | ||
| updateScheduler(t.name, duration) | ||
| } | ||
|
|
||
| return err != nil | ||
| } | ||
|
|
||
| func updateScheduler(test string, duration float64) { | ||
| req := &http.Request{ | ||
| Method: "POST", | ||
| Host: schedulerHost, | ||
| URL: &url.URL{ | ||
| Opaque: fmt.Sprintf("/record/%s/%0.2f", url.QueryEscape(test), duration), | ||
| Scheme: "http", | ||
| Host: schedulerHost, | ||
| }, | ||
| Close: true, | ||
| } | ||
| if resp, err := http.DefaultClient.Do(req); err != nil { | ||
| fmt.Printf("Error updating scheduler: %v\n", err) | ||
| } else { | ||
| resp.Body.Close() | ||
| } | ||
| } | ||
|
|
||
| func getSchedule(tests []string) ([]string, error) { | ||
| var ( | ||
| testRun = "integration-" + os.Getenv("CIRCLE_BUILD_NUM") | ||
| shardCount = os.Getenv("CIRCLE_NODE_TOTAL") | ||
| shardID = os.Getenv("CIRCLE_NODE_INDEX") | ||
| requestBody = &bytes.Buffer{} | ||
| ) | ||
| if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { | ||
| return []string{}, err | ||
| } | ||
| url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) | ||
| resp, err := http.Post(url, JSON, requestBody) | ||
| if err != nil { | ||
| return []string{}, err | ||
| } | ||
| var sched schedule | ||
| if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { | ||
| return []string{}, err | ||
| } | ||
| return sched.Tests, nil | ||
| } | ||
|
|
||
| func getTests(testNames []string) (tests, error) { | ||
| var err error | ||
| if useScheduler { | ||
| testNames, err = getSchedule(testNames) | ||
| if err != nil { | ||
| return tests{}, err | ||
| } | ||
| } | ||
| tests := tests{} | ||
| for _, name := range testNames { | ||
| parts := strings.Split(strings.TrimSuffix(name, "_test.sh"), "_") | ||
| numHosts, err := strconv.Atoi(parts[len(parts)-1]) | ||
| if err != nil { | ||
| numHosts = 1 | ||
| } | ||
| tests = append(tests, test{name, numHosts}) | ||
| fmt.Printf("Test %s needs %d hosts\n", name, numHosts) | ||
| } | ||
| sort.Sort(sort.Reverse(tests)) | ||
| return tests, nil | ||
| } | ||
|
|
||
| func parallel(tests tests, hosts []string) bool { | ||
| resultsChan := make(chan result) | ||
| outstanding := 0 | ||
| errored := false | ||
| for len(tests) > 0 || outstanding > 0 { | ||
| // While we have some free hosts, try and schedule | ||
| // a test on them | ||
| for len(hosts) > 0 { | ||
| test, ok := tests.pick(len(hosts)) | ||
| if !ok { | ||
| break | ||
| } | ||
| testHosts := hosts[:test.hosts] | ||
| hosts = hosts[test.hosts:] | ||
|
|
||
| go func() { | ||
| errored := test.run(testHosts) | ||
| resultsChan <- result{errored, testHosts} | ||
| }() | ||
| outstanding++ | ||
| } | ||
|
|
||
| // Otherwise, wait for the test to finish and return | ||
| // the hosts to the pool | ||
| result := <-resultsChan | ||
| hosts = append(hosts, result.hosts...) | ||
| outstanding-- | ||
| errored = result.errored || errored | ||
| } | ||
| return errored | ||
| } | ||
|
|
||
| func sequential(tests tests, hosts []string) bool { | ||
| errored := false | ||
| for _, test := range tests { | ||
| errored = test.run(hosts) || errored | ||
| } | ||
| return errored | ||
| } | ||
|
|
||
| func main() { | ||
| mflag.BoolVar(&useScheduler, []string{"scheduler"}, false, "Use scheduler to distribute tests across shards") | ||
| mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") | ||
| mflag.Parse() | ||
|
|
||
| tests, err := getTests(mflag.Args()) | ||
| if err != nil { | ||
| fmt.Printf("Error parsing tests: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| hosts := strings.Fields(os.Getenv("HOSTS")) | ||
| maxHosts := len(hosts) | ||
| if maxHosts == 0 { | ||
| fmt.Print("No HOSTS specified.\n") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| var errored bool | ||
| if runParallel { | ||
| errored = parallel(tests, hosts) | ||
| } else { | ||
| errored = sequential(tests, hosts) | ||
| } | ||
|
|
||
| if errored { | ||
| os.Exit(1) | ||
| } | ||
| } |