The measurement feature allows to use a script to measure the testbed
(in some meaningful way) and raise a red flag (hopefully with a useful
error message) when something changes. This can be invaluable to look
for tests that don't clean up properly or change the system in some
unexpected way.

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>

README.md

@@ -16,6 +16,7 @@ Spread
 [Fast iterations with reuse](#reuse)  
 [Debugging](#debugging)  
 [Repeating tasks](#repeating)
+[Measuring system deviation](#measuring)
 [Passwords and usernames](#passwords)  
 [Including, excluding, and renaming files](#including)  
 [Selecting which tasks to run](#selecting)  
@@ -516,6 +517,20 @@ In a similar way to prepare and restore scripts, these can also be defined
 as a `debug-each` script at the project, backend, and suite levels, so they
 are aggregated and repeated for every task under them.
 
+<a name="measuring"/>
+
+## Measuring
+
+Tests can modify the system in unexpected ways. Using project-wide
+`measure-each` stanza you can execute a command that can measure deviations of
+the test environment.
+
+As a simple advice, collect useful information that should be invariant (file
+list, package list, mount table, etc) and store that in a set of files. Those
+files should be (ideally) removed in project-wide prepare script.
+
+The measurement command is executed before preparing and after restoring each
+job.
 
 <a name="repeating"/>
 

spread/project.go

@@ -30,6 +30,7 @@ type Project struct {
 	PrepareEach string `yaml:"prepare-each"`
 	RestoreEach string `yaml:"restore-each"`
 	DebugEach   string `yaml:"debug-each"`
+	MeasureEach string `yaml:"measure-each"`
 
 	Suites map[string]*Suite
 
@@ -487,6 +488,7 @@ func Load(path string) (*Project, error) {
 	project.PrepareEach = strings.TrimSpace(project.PrepareEach)
 	project.RestoreEach = strings.TrimSpace(project.RestoreEach)
 	project.DebugEach = strings.TrimSpace(project.DebugEach)
+	project.MeasureEach = strings.TrimSpace(project.MeasureEach)
 
 	if err := checkEnv(project, &project.Environment); err != nil {
 		return nil, err

spread/runner.go

@@ -411,6 +411,7 @@ func (r *Runner) waitContent() (io.Reader, error) {
 }
 
 const (
+	measuring = "measuring"
 	preparing = "preparing"
 	executing = "executing"
 	restoring = "restoring"
@@ -596,6 +597,14 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) {
 
 		debug := job.Debug()
 		for repeat := r.options.Repeat; repeat >= 0; repeat-- {
+
+			if !r.run(client, job, measuring, r.project, r.project.MeasureEach, r.project.Debug, &abend) {
+				r.add(&stats.ProjectMeasureError, job)
+				r.add(&stats.TaskAbort, job)
+				badProject = true
+				repeat = -1
+			}
+
 			if r.options.Restore {
 				// Do not prepare or execute, and don't repeat.
 				repeat = -1
@@ -622,6 +631,13 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) {
 				badProject = true
 				repeat = -1
 			}
+
+			if !r.run(client, job, measuring, r.project, r.project.MeasureEach, r.project.Debug, &abend) {
+				r.add(&stats.ProjectMeasureError, job)
+				r.add(&stats.TaskAbort, job)
+				badProject = true
+				repeat = -1
+			}
 		}
 	}
 
@@ -1008,6 +1024,7 @@ type stats struct {
 	BackendRestoreError []*Job
 	ProjectPrepareError []*Job
 	ProjectRestoreError []*Job
+	ProjectMeasureError []*Job
 }
 
 func (s *stats) errorCount() int {
@@ -1021,6 +1038,7 @@ func (s *stats) errorCount() int {
 		s.BackendRestoreError,
 		s.ProjectPrepareError,
 		s.ProjectRestoreError,
+		s.ProjectMeasureError,
 	}
 	count := 0
 	for _, jobs := range errors {
@@ -1042,6 +1060,7 @@ func (s *stats) log() {
 	logNames(printf, "Failed backend restore", s.BackendRestoreError, backendName)
 	logNames(printf, "Failed project prepare", s.ProjectPrepareError, projectName)
 	logNames(printf, "Failed project restore", s.ProjectRestoreError, projectName)
+	logNames(printf, "Failed project measure", s.ProjectMeasureError, projectName)
 }
 
 func projectName(job *Job) string { return "project" }