Adding capability to write xunit report with the task suites and results #25

Open
wants to merge 8 commits into
from
View
@@ -32,6 +32,7 @@ var (
discard = flag.Bool("discard", false, "Discard reused servers without running")
residue = flag.String("residue", "", "Where to store residual data from tasks")
seed = flag.Int64("seed", 0, "Seed for job order permutation")
+ xunit = flag.Bool("xunit", false, "Create a XUnit report")
)
func main() {
@@ -92,6 +93,7 @@ func run() error {
Discard: *discard,
Residue: *residue,
Seed: *seed,
+ XUnit: *xunit,
}
project, err := spread.Load(".")
View
@@ -212,24 +212,27 @@ const (
)
func (c *Client) Run(script string, dir string, env *Environment) error {
- _, err := c.run(script, dir, env, combinedOutput)
+ _, err, _ := c.run(script, dir, env, combinedOutput)
return err
}
func (c *Client) Output(script string, dir string, env *Environment) (output []byte, err error) {
- return c.run(script, dir, env, splitOutput)
+ output, err, _ = c.run(script, dir, env, splitOutput)
+ return output, err
}
func (c *Client) CombinedOutput(script string, dir string, env *Environment) (output []byte, err error) {
- return c.run(script, dir, env, combinedOutput)
+ output, err, _ = c.run(script, dir, env, combinedOutput)
+ return output, err
}
-func (c *Client) Trace(script string, dir string, env *Environment) (output []byte, err error) {
- return c.run(script, dir, env, traceOutput)
+func (c *Client) Trace(script string, dir string, env *Environment) (output []byte, err error, duration time.Duration) {
+ output, err, duration = c.run(script, dir, env, traceOutput)
+ return output, err, duration
}
func (c *Client) Shell(script string, dir string, env *Environment) error {
- _, err := c.run(script, dir, env, shellOutput)
+ _, err, _ := c.run(script, dir, env, shellOutput)
return err
}
@@ -241,7 +244,7 @@ func (e *rebootError) Error() string { return "reboot requested" }
const maxReboots = 10
-func (c *Client) run(script string, dir string, env *Environment, mode outputMode) (output []byte, err error) {
+func (c *Client) run(script string, dir string, env *Environment, mode outputMode) (output []byte, err error, duration time.Duration) {
if env == nil {
env = NewEnvironment()
}
@@ -251,13 +254,13 @@ func (c *Client) run(script string, dir string, env *Environment, mode outputMod
rebootKey = strconv.Itoa(reboot)
}
env.Set("SPREAD_REBOOT", rebootKey)
- output, err = c.runPart(script, dir, env, mode, output)
+ output, err, duration = c.runPart(script, dir, env, mode, output)
rerr, ok := err.(*rebootError)
if !ok {
- return output, err
+ return output, err, duration
}
if reboot > maxReboots {
- return nil, fmt.Errorf("%s rebooted more than %d times", c.server, maxReboots)
+ return nil, fmt.Errorf("%s rebooted more than %d times", c.server, maxReboots), duration
}
printf("Rebooting %s as requested...", c.server)
@@ -273,13 +276,13 @@ func (c *Client) run(script string, dir string, env *Environment, mode outputMod
if err == nil {
select {
case <-timedout:
- return nil, fmt.Errorf("kill-timeout reached while waiting for %s to reboot", c.server)
+ return nil, fmt.Errorf("kill-timeout reached while waiting for %s to reboot", c.server), duration
default:
}
- return nil, fmt.Errorf("reboot request on %s failed", c.server)
+ return nil, fmt.Errorf("reboot request on %s failed", c.server), duration
}
if err := c.dialOnReboot(); err != nil {
- return nil, err
+ return nil, err, duration
}
}
panic("unreachable")
@@ -292,15 +295,15 @@ var toBashRC = map[string]bool{
"SPREAD_SYSTEM": true,
}
-func (c *Client) runPart(script string, dir string, env *Environment, mode outputMode, previous []byte) (output []byte, err error) {
+func (c *Client) runPart(script string, dir string, env *Environment, mode outputMode, previous []byte) (output []byte, err error, duration time.Duration) {
script = strings.TrimSpace(script)
if len(script) == 0 && mode != shellOutput {
- return nil, nil
+ return nil, nil, 0
}
script += "\n"
session, err := c.sshc.NewSession()
if err != nil {
- return nil, err
+ return nil, err, 0
}
defer session.Close()
@@ -366,7 +369,7 @@ func (c *Client) runPart(script string, dir string, env *Environment, mode outpu
} else {
stdin, err := session.StdinPipe()
if err != nil {
- return nil, err
+ return nil, err, 0
}
defer stdin.Close()
@@ -397,27 +400,29 @@ func (c *Client) runPart(script string, dir string, env *Environment, mode outpu
session.Stderr = os.Stderr
w, h, err := terminal.GetSize(0)
if err != nil {
- return nil, fmt.Errorf("cannot get local terminal size: %v", err)
+ return nil, fmt.Errorf("cannot get local terminal size: %v", err), 0
}
if err := session.RequestPty(getenv("TERM", "vt100"), h, w, nil); err != nil {
- return nil, fmt.Errorf("cannot get remote pseudo terminal: %v", err)
+ return nil, fmt.Errorf("cannot get remote pseudo terminal: %v", err), 0
}
default:
panic("internal error: invalid output mode")
}
+ startTime := time.Now()
if mode == shellOutput {
termLock()
tstate, terr := terminal.MakeRaw(0)
if terr != nil {
- return nil, fmt.Errorf("cannot put local terminal in raw mode: %v", terr)
+ return nil, fmt.Errorf("cannot put local terminal in raw mode: %v", terr), 0
}
err = session.Run(cmd)
terminal.Restore(0, tstate)
termUnlock()
} else {
err = c.runCommand(session, cmd, &stdout, &stderr)
}
+ elapsedTime := time.Since(startTime)
if stdout.Len() > 0 {
debugf("Output from running script on %s:\n-----\n%s\n-----", c.server, stdout.Bytes())
@@ -430,10 +435,10 @@ func (c *Client) runPart(script string, dir string, env *Environment, mode outpu
lines := bytes.Split(bytes.TrimSpace(stdout.Bytes()), []byte{'\n'})
m := commandExp.FindSubmatch(lines[len(lines)-1])
if len(m) > 0 && string(m[1]) == "REBOOT" {
- return append(previous, stdout.Bytes()...), &rebootError{string(m[2])}
+ return append(previous, stdout.Bytes()...), &rebootError{string(m[2])}, elapsedTime
}
if len(m) > 0 && string(m[1]) == "ERROR" {
- return nil, fmt.Errorf("%s", m[2])
+ return nil, fmt.Errorf("%s", m[2]), elapsedTime
}
}
@@ -450,12 +455,12 @@ func (c *Client) runPart(script string, dir string, env *Environment, mode outpu
output = append(previous, output...)
if err != nil {
- return nil, outputErr(output, err)
+ return nil, outputErr(output, err), elapsedTime
}
if err := <-errch; err != nil {
printf("Error writing script to %s: %v", c.server, err)
}
- return output, nil
+ return output, nil, elapsedTime
}
func (c *Client) sudo() string {
View
@@ -350,6 +350,7 @@ type Job struct {
Variant string
Environment *Environment
+ Duration time.Duration
}
func (job *Job) String() string {
View
@@ -0,0 +1,140 @@
+package spread
+
+import (
+ "strconv"
+ "time"
+
+ "encoding/xml"
+ "io/ioutil"
+)
+
+type XUnitTestSuites struct {
+ XMLName xml.Name `xml:"testsuites"`
+ Suites []*XUnitTestSuite
+}
+
+func (tss *XUnitTestSuites) addSuite(suite *XUnitTestSuite) {
+ tss.Suites = append(tss.Suites, suite)
+}
+
+func (tss *XUnitTestSuites) getSuite(suiteName string) *XUnitTestSuite {
+ for _, s := range tss.Suites {
+ if s.Name == suiteName {
+ return s
+ }
+ }
+ suite := NewTestSuite(suiteName)
+
+ tss.addSuite(suite)
+ return suite
+}
+
+type XUnitTestSuite struct {
+ XMLName xml.Name `xml:"testsuite"`
+ Tests int `xml:"tests,attr"`
+ Failures int `xml:"failures,attr"`
+ Time string `xml:"time,attr"`
+ Name string `xml:"name,attr"`
+ Properties []*XUnitProperty `xml:"properties>property,omitempty"`
+ TestCases []*XUnitTestCase
+}
+
+func NewTestSuite(suiteName string) *XUnitTestSuite {
+ return &XUnitTestSuite{
+ Tests: 0,
+ Failures: 0,
+ Time: "",
+ Name: suiteName,
+ Properties: []*XUnitProperty{},
+ TestCases: []*XUnitTestCase{},
+ }
+}
+
+func (ts *XUnitTestSuite) addTest(test *XUnitTestCase) {
+ if test.Failure != nil {
+ ts.Failures += 1
+ }
+ ts.Tests += 1
+ ts.TestCases = append(ts.TestCases, test)
+}
+
+type XUnitTestCase struct {
+ XMLName xml.Name `xml:"testcase"`
+ Classname string `xml:"classname,attr"`
+ Name string `xml:"name,attr"`
+ Time string `xml:"time,attr"`
+ SkipMessage *XUnitSkipMessage `xml:"skipped,omitempty"`
+ Failure *XUnitFailure `xml:"failure,omitempty"`
+}
+
+func NewTestCase(testName string, className string, duration time.Duration) *XUnitTestCase {
+ return &XUnitTestCase{
+ Classname: className,
+ Name: testName,
+ Time: strconv.FormatFloat(float64(duration/time.Millisecond)/1000, 'f' , 3, 64),
+ Failure: nil,
+ }
+}
+
+type XUnitSkipMessage struct {
+ Message string `xml:"message,attr"`
+}
+
+type XUnitProperty struct {
+ Name string `xml:"name,attr"`
+ Value string `xml:"value,attr"`
+}
+
+type XUnitFailure struct {
+ Message string `xml:"message,attr"`
+ Type string `xml:"type,attr"`
+ Contents string `xml:",chardata"`
+}
+
+type XUnitReport struct {
+ FileName string
+ TestSuites *XUnitTestSuites
+}
+
+func NewXUnitReport(name string) XUnitReport {
+ report := XUnitReport{}
+ report.FileName = name
+ report.TestSuites = &XUnitTestSuites{}
+ return report
+}
+
+func (r XUnitReport) finish() error {
+ bytes, err := xml.MarshalIndent(r.TestSuites, "", "\t")
+ check(err)
+
+ err = ioutil.WriteFile(r.FileName, bytes, 0644)
+ check(err)
+
+ return nil
+}
+
+func (r XUnitReport) addTest(suiteName string, test *XUnitTestCase) {
+ suite := r.TestSuites.getSuite(suiteName)
+ suite.addTest(test)
+}
+
+func (r XUnitReport) addFailedTest(suiteName string, className string, testName string, duration time.Duration) {
+ testcase := NewTestCase(testName, className, duration)
+ testcase.Failure = &XUnitFailure{
+ Message: "Failed",
+ Type: "",
+ Contents: "",
+ }
+ r.addTest(suiteName, testcase)
+}
+
+func (r XUnitReport) addPassedTest(suiteName string, className string, testName string, duration time.Duration) {
+ testcase := NewTestCase(testName, className, duration)
+ r.addTest(suiteName, testcase)
+}
+
+func check(e error) {
+ if e != nil {
+ panic(e)
+ }
+}
View
@@ -33,6 +33,7 @@ type Options struct {
Discard bool
Residue string
Seed int64
+ XUnit bool
}
type Runner struct {
@@ -143,6 +144,9 @@ func (r *Runner) loop() (err error) {
}
}
r.stats.log()
+ if r.options.XUnit {
+ r.stats.createXUnitReport()
+ }
}
if !r.options.Reuse || r.options.Discard {
for len(r.servers) > 0 {
@@ -450,11 +454,12 @@ func (r *Runner) run(client *Client, job *Job, verb string, context interface{},
}
client.SetWarnTimeout(job.WarnTimeoutFor(context))
client.SetKillTimeout(job.KillTimeoutFor(context))
- _, err := client.Trace(script, dir, job.Environment)
+ _, err, dur := client.Trace(script, dir, job.Environment)
+ job.Duration = dur
if err != nil {
printf("Error %s %s : %v", verb, contextStr, err)
if debug != "" {
- output, err := client.Trace(debug, dir, job.Environment)
+ output, err, _ := client.Trace(debug, dir, job.Environment)
if err != nil {
printf("Error debugging %s : %v", contextStr, err)
} else if len(output) > 0 {
@@ -996,6 +1001,31 @@ func (s *stats) log() {
logNames(printf, "Failed project restore", s.ProjectRestoreError, projectName)
}
+func (s *stats) addTestsToXUnitReport(report XUnitReport, testsList []*Job, failed bool) {
+ for _, job := range testsList {
+ splittedName := strings.Split(taskName(job), "/")
+ suiteName := strings.Join(splittedName[:len(splittedName)-1], ".")
+ testName := splittedName[len(splittedName)-1]
+ className := strings.Join([]string{job.Project.Name, job.Backend.Name, job.System.Name, suiteName}, ".")
+
+ if failed {
+ report.addFailedTest(suiteName, className, testName, job.Duration)
+ } else {
+ report.addPassedTest(suiteName, className, testName, job.Duration)
+ }
+
+ }
+}
+
+func (s *stats) createXUnitReport() {
+ printf("Creating XUnit report")
+ report := NewXUnitReport("report.xml")
+ s.addTestsToXUnitReport(report, s.TaskError, true)
+ s.addTestsToXUnitReport(report, s.TaskAbort, true)
+ s.addTestsToXUnitReport(report, s.TaskDone, false)
+ report.finish()
+}
+
func projectName(job *Job) string { return "project" }
func backendName(job *Job) string { return job.Backend.Name }
func suiteName(job *Job) string { return job.Suite.Name }